├── .gitignore
├── CODEOWNERS
├── README.md
├── images
└── console-icon.png
├── light-get-started.yaml
├── quick-start-m4.yml
├── quick-start.yml
├── simple.yml
└── sql
├── README.md
├── clickhouse.xml
└── docker-compose.yml
/.gitignore:
--------------------------------------------------------------------------------
1 | platform_data/
2 |
3 | # standard IDE files ignore
4 | .idea/
5 | *.iml
6 | .vscode
--------------------------------------------------------------------------------
/CODEOWNERS:
--------------------------------------------------------------------------------
1 | @conduktor/conduktor-platform
2 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | 
5 |
6 |
7 | Conduktor Console
8 |
9 |
10 |
11 | Explore the docs »
12 |
13 |
14 | 
15 | ·
16 | 
17 | ·
18 | 
19 | ·
20 | 
21 |
22 |
23 | We take the complexity out of Kafka. Console gives you visibility into your Kafka ecosystem and concentrates all of the Kafka APIs into a single interface. Troubleshoot and debug Kafka, drill-down into topic data, and continuously monitor your streaming applications.
24 |
25 | Conduktor supports all Kafka providers (Apache Kafka, MSK, Confluent, Aiven, Redpanda, Strimzi etc.)
26 |
27 | 
28 |
29 |
30 | # 👩💻 Get Started
31 |
32 | You have two options for getting started:
33 |
34 | 1. Quick-Start: Preconfigured with embedded Redpanda + Datagen
35 | ```` shell
36 | # Start Conduktor in seconds, powered with Redpanda
37 | curl -L https://releases.conduktor.io/quick-start -o docker-compose.yml && docker compose up
38 | ````
39 |
40 | 2. Console Only: Start Conduktor and connect your own Kafka
41 | ```` shell
42 | # Start Conduktor and connect it to your own Kafka
43 | curl -L https://releases.conduktor.io/console -o docker-compose.yml && docker compose up
44 | ````
45 |
46 | Once started, you can access the console at **http://localhost:8080**
47 |
48 | See our [docs](https://docs.conduktor.io/) for more information on
49 | - [Configuring your Kafka cluster](https://docs.conduktor.io/platform/installation/get-started/docker/#step-3-configure-your-existing-kafka-cluster)
50 | - [User authentication (SSO, LDAP)](https://docs.conduktor.io/platform/category/user-authentication/)
51 | - [Environment variables](https://docs.conduktor.io/platform/configuration/env-variables/)
52 | - [Deployment methods](https://docs.conduktor.io/platform/category/get-started/)
53 |
54 | ---
55 |
56 | # 📢 Our Roadmap
57 |
58 | We really want our community to be part of our evolution.
59 |
60 | Please upvote or comment everything we're doing. WE. ARE. LISTENING.
61 |
62 | [View roadmap and provide feedback](https://product.conduktor.help/tabs/1-in-development)
63 |
64 | ---
65 |
66 | # 🆘 Looking for help?
67 |
68 | * [Console Changelog](https://www.conduktor.io/changelog)
69 | * [Console Documentation](https://docs.conduktor.io/)
70 | * Contact us for anything else: support@conduktor.io
71 |
72 | ---
73 |
74 | # Get Involved
75 |
76 | * [Our Roadmap](https://product.conduktor.help/tabs/1-in-development)
77 | * Follow @getconduktor on Twitter
78 | * Read the Conduktor blog
79 | * Looking for a job? careers
80 | * Learning Apache Kafka? We have a whole learning website dedicated for you!
81 |
--------------------------------------------------------------------------------
/images/console-icon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/conduktor/conduktor-platform/68dc5fec4d9c2ff76edb8c4d3b4fa7a6a83b738c/images/console-icon.png
--------------------------------------------------------------------------------
/light-get-started.yaml:
--------------------------------------------------------------------------------
1 | # This docker-compose is only to get started with Conduktor.
2 | # It is starting Conduktor Console and a Redpanda cluster (Kafka).
3 |
4 | # Go to http://localhost:8080 when started
5 |
6 | ####################################################################################################
7 | # DO NOT USE IT IN PRODUCTION
8 | #
9 | # For production, please check: https://docs.conduktor.io/platform/category/deployment-options/
10 | # to deploy on Kubernetes via Helm, or AWS using Cloudformation.
11 | ####################################################################################################
12 |
13 |
14 | services:
15 | # Conduktor Console, the enterprise UI.
16 | # It depends on PostgreSQL. Here, we depend on Redpanda only for our get-started.
17 | # https://docs.conduktor.io/platform/get-started/configuration/introduction/
18 | conduktor-console:
19 | image: conduktor/conduktor-console:1.34.3
20 | ports:
21 | - "8080:8080"
22 | volumes:
23 | - conduktor_data:/var/conduktor
24 | environment:
25 | CDK_DATABASE_URL: "postgresql://conduktor:change_me@postgresql:5432/conduktor-console"
26 | CDK_CLUSTERS_0_ID: "local-kafka"
27 | CDK_CLUSTERS_0_NAME: "local-kafka"
28 | CDK_CLUSTERS_0_BOOTSTRAPSERVERS: "redpanda-0:9092"
29 | CDK_CLUSTERS_0_SCHEMAREGISTRY_URL: "http://redpanda-0:18081"
30 | CDK_CLUSTERS_0_COLOR: "#6A57C8"
31 | CDK_CLUSTERS_0_ICON: "kafka"
32 | CDK_MONITORING_CORTEX-URL: http://conduktor-monitoring:9009/
33 | CDK_MONITORING_ALERT-MANAGER-URL: http://conduktor-monitoring:9010/
34 | CDK_MONITORING_CALLBACK-URL: http://conduktor-console:8080/monitoring/api/
35 | CDK_MONITORING_NOTIFICATIONS-CALLBACK-URL: http://localhost:8080
36 | depends_on:
37 | redpanda-0:
38 | condition: service_healthy
39 | postgresql:
40 | condition: service_healthy
41 | # Conduktor stores its metadata in PostgreSQL.
42 | # Consider using an external managed database for production usage.
43 | # https://docs.conduktor.io/platform/get-started/configuration/database/
44 | postgresql:
45 | image: postgres:14
46 | hostname: postgresql
47 | volumes:
48 | - pg_data:/var/lib/postgresql/data
49 | environment:
50 | PGDATA: "/var/lib/postgresql/data"
51 | POSTGRES_DB: "conduktor-console"
52 | POSTGRES_USER: "conduktor"
53 | POSTGRES_PASSWORD: "change_me"
54 | POSTGRES_HOST_AUTH_METHOD: "scram-sha-256"
55 | healthcheck:
56 | test: ["CMD-SHELL", "pg_isready"]
57 | interval: 10s
58 | timeout: 5s
59 | retries: 5
60 | # Conduktor uses Cortex to store Kafka and applications metrics as well as alerting.
61 | # It is optional.
62 | # https://docs.conduktor.io/platform/get-started/configuration/cortex/
63 | conduktor-monitoring:
64 | image: conduktor/conduktor-console-cortex:1.34.3
65 | environment:
66 | CDK_CONSOLE-URL: "http://conduktor-console:8080"
67 | # We use Redpanda to start with Kafka as it's small and efficient.
68 | # This is an example here. For production, connect Conduktor Console to your own Kafka clusters.
69 | redpanda-0:
70 | command:
71 | - redpanda
72 | - start
73 | - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
74 | - --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
75 | - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
76 | # Address the broker advertises to clients that connect to the HTTP Proxy.
77 | - --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
78 | - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
79 | # Redpanda brokers use the RPC API to communicate with eachother internally.
80 | - --rpc-addr redpanda-0:33145
81 | - --advertise-rpc-addr redpanda-0:33145
82 | - --smp 1
83 | - --memory 1G
84 | - --mode dev-container
85 | - --default-log-level=info
86 | image: docker.redpanda.com/redpandadata/redpanda:v24.1.6
87 | container_name: redpanda-0
88 | volumes:
89 | - redpanda-0:/var/lib/redpanda/data
90 | ports:
91 | - 18081:18081
92 | - 18082:18082
93 | - 19092:19092
94 | - 19644:9644
95 | healthcheck:
96 | test: ["CMD-SHELL", "rpk cluster health | grep -E 'Healthy:.+true' || exit 1"]
97 | interval: 15s
98 | timeout: 3s
99 | retries: 5
100 | start_period: 5s
101 | volumes:
102 | pg_data: {}
103 | conduktor_data: {}
104 | redpanda-0: {}
105 |
--------------------------------------------------------------------------------
/quick-start-m4.yml:
--------------------------------------------------------------------------------
1 | # This docker-compose is only to get started with Conduktor.
2 | # It is starting Conduktor, a Redpanda cluster (Kafka), and a fake app to generate traffic.
3 |
4 | # Go to http://localhost:8080 when started
5 |
6 | ####################################################################################################
7 | # DO NOT USE IT IN PRODUCTION
8 | #
9 | # For production, please check: https://docs.conduktor.io/platform/category/deployment-options/
10 | # to deploy on Kubernetes via Helm, or AWS using Cloudformation.
11 | ####################################################################################################
12 |
13 |
14 | services:
15 | # Conduktor Console, the enterprise UI.
16 | # It depends on PostgreSQL. Here, we depend on Redpanda only for our get-started.
17 | # https://docs.conduktor.io/platform/get-started/configuration/introduction/
18 | conduktor-console:
19 | image: conduktor/conduktor-console:1.34.3
20 | ports:
21 | - "8080:8080"
22 | volumes:
23 | - conduktor_data:/var/conduktor
24 | environment:
25 | CDK_DATABASE_URL: "postgresql://conduktor:change_me@postgresql:5432/conduktor-console"
26 | CDK_KAFKASQL_DATABASE_URL: "postgresql://conduktor:change_me@postgresql-sql:5432/conduktor-sql"
27 | CDK_ORGANIZATION_NAME: "getting-started"
28 | CDK_CLUSTERS_0_ID: "local-kafka"
29 | CDK_CLUSTERS_0_NAME: "local-kafka"
30 | CDK_CLUSTERS_0_BOOTSTRAPSERVERS: "redpanda-0:9092"
31 | CDK_CLUSTERS_0_SCHEMAREGISTRY_URL: "http://redpanda-0:18081"
32 | CDK_CLUSTERS_0_COLOR: "#6A57C8"
33 | CDK_CLUSTERS_0_ICON: "kafka"
34 | CDK_CLUSTERS_1_ID: "cdk-gateway"
35 | CDK_CLUSTERS_1_NAME: "cdk-gateway"
36 | CDK_CLUSTERS_1_BOOTSTRAPSERVERS: "conduktor-gateway:6969"
37 | CDK_CLUSTERS_1_SCHEMAREGISTRY_URL: "http://redpanda-0:18081"
38 | CDK_CLUSTERS_1_KAFKAFLAVOR_URL: "http://conduktor-gateway:8888"
39 | CDK_CLUSTERS_1_KAFKAFLAVOR_USER: "admin"
40 | CDK_CLUSTERS_1_KAFKAFLAVOR_PASSWORD: "conduktor"
41 | CDK_CLUSTERS_1_KAFKAFLAVOR_VIRTUALCLUSTER: "passthrough"
42 | CDK_CLUSTERS_1_KAFKAFLAVOR_TYPE: "Gateway"
43 | CDK_CLUSTERS_1_COLOR: "#6A57C8"
44 | CDK_CLUSTERS_1_ICON: "dog"
45 | CDK_MONITORING_CORTEX-URL: http://conduktor-monitoring:9009/
46 | CDK_MONITORING_ALERT-MANAGER-URL: http://conduktor-monitoring:9010/
47 | CDK_MONITORING_CALLBACK-URL: http://conduktor-console:8080/monitoring/api/
48 | CDK_MONITORING_NOTIFICATIONS-CALLBACK-URL: http://localhost:8080
49 | CONSOLE_JAVA_OPTS: "-XX:UseSVE=0"
50 | depends_on:
51 | redpanda-0:
52 | condition: service_healthy
53 | postgresql:
54 | condition: service_healthy
55 | postgresql-2:
56 | condition: service_healthy
57 | # Conduktor stores its metadata in PostgreSQL.
58 | # Consider using an external managed database for production usage.
59 | # https://docs.conduktor.io/platform/get-started/configuration/database/
60 | postgresql:
61 | image: postgres:14
62 | hostname: postgresql
63 | volumes:
64 | - pg_data:/var/lib/postgresql/data
65 | environment:
66 | PGDATA: "/var/lib/postgresql/data"
67 | POSTGRES_DB: "conduktor-console"
68 | POSTGRES_USER: "conduktor"
69 | POSTGRES_PASSWORD: "change_me"
70 | POSTGRES_HOST_AUTH_METHOD: "scram-sha-256"
71 | healthcheck:
72 | test: ["CMD-SHELL", "pg_isready"]
73 | interval: 10s
74 | timeout: 5s
75 | retries: 5
76 | # Conduktor depends on a separate db for storing Kafka data for SQL querying
77 | # It is optional, but required if you wish to use SQL functionality
78 | # Separate db ensures continued operation of the core Console experience if the SQL db becomes unavailable
79 | # https://docs.conduktor.io/platform/guides/configure-sql/
80 | postgresql-2:
81 | image: postgres:14
82 | hostname: postgresql-sql
83 | volumes:
84 | - pg_data_sql:/var/lib/postgresql/data
85 | environment:
86 | PGDATA: "/var/lib/postgresql/data"
87 | POSTGRES_DB: "conduktor-sql"
88 | POSTGRES_USER: "conduktor"
89 | POSTGRES_PASSWORD: "change_me"
90 | POSTGRES_HOST_AUTH_METHOD: "scram-sha-256"
91 | healthcheck:
92 | test: ["CMD-SHELL", "pg_isready"]
93 | interval: 10s
94 | timeout: 5s
95 | retries: 5
96 | # Conduktor uses Cortex to store Kafka and applications metrics as well as alerting.
97 | # It is optional.
98 | # https://docs.conduktor.io/platform/get-started/configuration/cortex/
99 | conduktor-monitoring:
100 | image: conduktor/conduktor-console-cortex:1.34.3
101 | environment:
102 | CDK_CONSOLE-URL: "http://conduktor-console:8080"
103 | # We use Redpanda to get started with Kafka as it's small and efficient.
104 | # This is an example here. For production, connect Conduktor to your own Kafka clusters.
105 | redpanda-0:
106 | command:
107 | - redpanda
108 | - start
109 | - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
110 | - --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
111 | - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
112 | # Address the broker advertises to clients that connect to the HTTP Proxy.
113 | - --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
114 | - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
115 | # Redpanda brokers use the RPC API to communicate with eachother internally.
116 | - --rpc-addr redpanda-0:33145
117 | - --advertise-rpc-addr redpanda-0:33145
118 | - --smp 1
119 | - --memory 1G
120 | - --mode dev-container
121 | - --default-log-level=info
122 | image: docker.redpanda.com/redpandadata/redpanda:v24.1.6
123 | container_name: redpanda-0
124 | volumes:
125 | - redpanda-0:/var/lib/redpanda/data
126 | ports:
127 | - 18081:18081
128 | - 18082:18082
129 | - 19092:19092
130 | - 19644:9644
131 | healthcheck:
132 | test: ["CMD-SHELL", "rpk cluster health | grep -E 'Healthy:.+true' || exit 1"]
133 | interval: 15s
134 | timeout: 3s
135 | retries: 5
136 | start_period: 5s
137 | # Conduktor comes with its Gateway, a Kafka proxy bringing many security and governance features.
138 | # In this get started, Gateway uses Redpanda as its backend Kafka cluster.
139 | # https://docs.conduktor.io/gateway/
140 | conduktor-gateway:
141 | image: conduktor/conduktor-gateway:3.9.0
142 | hostname: conduktor-gateway
143 | container_name: conduktor-gateway
144 | environment:
145 | KAFKA_BOOTSTRAP_SERVERS: redpanda-0:9092
146 | JAVA_OPTS: "-XX:UseSVE=0"
147 | ports:
148 | - "8888:8888"
149 | healthcheck:
150 | test: curl localhost:8888/health
151 | interval: 5s
152 | retries: 25
153 | depends_on:
154 | redpanda-0:
155 | condition: service_healthy
156 | # As this is a get started, we want to bring some life to the cluster to demonstrate the value of Conduktor.
157 | # This is totally optional and only used for this purpose. Do not use it in production.
158 | conduktor-data-generator:
159 | image: conduktor/conduktor-data-generator:0.9
160 | container_name: conduktor-data-generator
161 | environment:
162 | KAFKA_BOOTSTRAP_SERVERS: conduktor-gateway:6969
163 | KAFKA_SCHEMA_REGISTRY_URL: http://redpanda-0:8081
164 | GATEWAY_ADMIN_API: http://conduktor-gateway:8888
165 | JAVA_OPTS: "-XX:UseSVE=0"
166 | restart: on-failure
167 | depends_on:
168 | redpanda-0:
169 | condition: service_healthy
170 | conduktor-gateway:
171 | condition: service_healthy
172 | volumes:
173 | pg_data: {}
174 | pg_data_sql: {}
175 | conduktor_data: {}
176 | redpanda-0: {}
177 |
--------------------------------------------------------------------------------
/quick-start.yml:
--------------------------------------------------------------------------------
1 | # This docker-compose is only to get started with Conduktor.
2 | # It is starting Conduktor, a Redpanda cluster (Kafka), and a fake app to generate traffic.
3 |
4 | # Go to http://localhost:8080 when started
5 |
6 | ####################################################################################################
7 | # DO NOT USE IT IN PRODUCTION
8 | #
9 | # For production, please check: https://docs.conduktor.io/platform/category/deployment-options/
10 | # to deploy on Kubernetes via Helm, or AWS using Cloudformation.
11 | ####################################################################################################
12 |
13 |
14 | services:
15 | # Conduktor Console, the enterprise UI.
16 | # It depends on PostgreSQL. Here, we depend on Redpanda only for our get-started.
17 | # https://docs.conduktor.io/platform/get-started/configuration/introduction/
18 | conduktor-console:
19 | image: conduktor/conduktor-console:1.34.3
20 | ports:
21 | - "8080:8080"
22 | volumes:
23 | - conduktor_data:/var/conduktor
24 | environment:
25 | CDK_DATABASE_URL: "postgresql://conduktor:change_me@postgresql:5432/conduktor-console"
26 | CDK_KAFKASQL_DATABASE_URL: "postgresql://conduktor:change_me@postgresql-sql:5432/conduktor-sql"
27 | CDK_ORGANIZATION_NAME: "getting-started"
28 | CDK_CLUSTERS_0_ID: "local-kafka"
29 | CDK_CLUSTERS_0_NAME: "local-kafka"
30 | CDK_CLUSTERS_0_BOOTSTRAPSERVERS: "redpanda-0:9092"
31 | CDK_CLUSTERS_0_SCHEMAREGISTRY_URL: "http://redpanda-0:18081"
32 | CDK_CLUSTERS_0_COLOR: "#6A57C8"
33 | CDK_CLUSTERS_0_ICON: "kafka"
34 | CDK_CLUSTERS_1_ID: "cdk-gateway"
35 | CDK_CLUSTERS_1_NAME: "cdk-gateway"
36 | CDK_CLUSTERS_1_BOOTSTRAPSERVERS: "conduktor-gateway:6969"
37 | CDK_CLUSTERS_1_SCHEMAREGISTRY_URL: "http://redpanda-0:18081"
38 | CDK_CLUSTERS_1_KAFKAFLAVOR_URL: "http://conduktor-gateway:8888"
39 | CDK_CLUSTERS_1_KAFKAFLAVOR_USER: "admin"
40 | CDK_CLUSTERS_1_KAFKAFLAVOR_PASSWORD: "conduktor"
41 | CDK_CLUSTERS_1_KAFKAFLAVOR_VIRTUALCLUSTER: "passthrough"
42 | CDK_CLUSTERS_1_KAFKAFLAVOR_TYPE: "Gateway"
43 | CDK_CLUSTERS_1_COLOR: "#6A57C8"
44 | CDK_CLUSTERS_1_ICON: "dog"
45 | CDK_MONITORING_CORTEX-URL: http://conduktor-monitoring:9009/
46 | CDK_MONITORING_ALERT-MANAGER-URL: http://conduktor-monitoring:9010/
47 | CDK_MONITORING_CALLBACK-URL: http://conduktor-console:8080/monitoring/api/
48 | CDK_MONITORING_NOTIFICATIONS-CALLBACK-URL: http://localhost:8080
49 | depends_on:
50 | redpanda-0:
51 | condition: service_healthy
52 | postgresql:
53 | condition: service_healthy
54 | postgresql-2:
55 | condition: service_healthy
56 | # Conduktor stores its metadata in PostgreSQL.
57 | # Consider using an external managed database for production usage.
58 | # https://docs.conduktor.io/platform/get-started/configuration/database/
59 | postgresql:
60 | image: postgres:14
61 | hostname: postgresql
62 | volumes:
63 | - pg_data:/var/lib/postgresql/data
64 | environment:
65 | PGDATA: "/var/lib/postgresql/data"
66 | POSTGRES_DB: "conduktor-console"
67 | POSTGRES_USER: "conduktor"
68 | POSTGRES_PASSWORD: "change_me"
69 | POSTGRES_HOST_AUTH_METHOD: "scram-sha-256"
70 | healthcheck:
71 | test: ["CMD-SHELL", "pg_isready"]
72 | interval: 10s
73 | timeout: 5s
74 | retries: 5
75 | # Conduktor depends on a separate db for storing Kafka data for SQL querying
76 | # It is optional, but required if you wish to use SQL functionality
77 | # Separate db ensures continued operation of the core Console experience if the SQL db becomes unavailable
78 | # https://docs.conduktor.io/platform/guides/configure-sql/
79 | postgresql-2:
80 | image: postgres:14
81 | hostname: postgresql-sql
82 | volumes:
83 | - pg_data_sql:/var/lib/postgresql/data
84 | environment:
85 | PGDATA: "/var/lib/postgresql/data"
86 | POSTGRES_DB: "conduktor-sql"
87 | POSTGRES_USER: "conduktor"
88 | POSTGRES_PASSWORD: "change_me"
89 | POSTGRES_HOST_AUTH_METHOD: "scram-sha-256"
90 | healthcheck:
91 | test: ["CMD-SHELL", "pg_isready"]
92 | interval: 10s
93 | timeout: 5s
94 | retries: 5
95 | # Conduktor uses Cortex to store Kafka and applications metrics as well as alerting.
96 | # It is optional.
97 | # https://docs.conduktor.io/platform/get-started/configuration/cortex/
98 | conduktor-monitoring:
99 | image: conduktor/conduktor-console-cortex:1.34.3
100 | environment:
101 | CDK_CONSOLE-URL: "http://conduktor-console:8080"
102 | # We use Redpanda to get started with Kafka as it's small and efficient.
103 | # This is an example here. For production, connect Conduktor to your own Kafka clusters.
104 | redpanda-0:
105 | command:
106 | - redpanda
107 | - start
108 | - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
109 | - --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
110 | - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
111 | # Address the broker advertises to clients that connect to the HTTP Proxy.
112 | - --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
113 | - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
114 | # Redpanda brokers use the RPC API to communicate with eachother internally.
115 | - --rpc-addr redpanda-0:33145
116 | - --advertise-rpc-addr redpanda-0:33145
117 | - --smp 1
118 | - --memory 1G
119 | - --mode dev-container
120 | - --default-log-level=info
121 | image: docker.redpanda.com/redpandadata/redpanda:v24.1.6
122 | container_name: redpanda-0
123 | volumes:
124 | - redpanda-0:/var/lib/redpanda/data
125 | ports:
126 | - 18081:18081
127 | - 18082:18082
128 | - 19092:19092
129 | - 19644:9644
130 | healthcheck:
131 | test: ["CMD-SHELL", "rpk cluster health | grep -E 'Healthy:.+true' || exit 1"]
132 | interval: 15s
133 | timeout: 3s
134 | retries: 5
135 | start_period: 5s
136 | # Conduktor comes with its Gateway, a Kafka proxy bringing many security and governance features.
137 | # In this get started, Gateway uses Redpanda as its backend Kafka cluster.
138 | # https://docs.conduktor.io/gateway/
139 | conduktor-gateway:
140 | image: conduktor/conduktor-gateway:3.9.0
141 | hostname: conduktor-gateway
142 | container_name: conduktor-gateway
143 | environment:
144 | KAFKA_BOOTSTRAP_SERVERS: redpanda-0:9092
145 | ports:
146 | - "8888:8888"
147 | healthcheck:
148 | test: curl localhost:8888/health
149 | interval: 5s
150 | retries: 25
151 | depends_on:
152 | redpanda-0:
153 | condition: service_healthy
154 | # As this is a get started, we want to bring some life to the cluster to demonstrate the value of Conduktor.
155 | # This is totally optional and only used for this purpose. Do not use it in production.
156 | conduktor-data-generator:
157 | image: conduktor/conduktor-data-generator:0.9
158 | container_name: conduktor-data-generator
159 | environment:
160 | KAFKA_BOOTSTRAP_SERVERS: conduktor-gateway:6969
161 | KAFKA_SCHEMA_REGISTRY_URL: http://redpanda-0:8081
162 | GATEWAY_ADMIN_API: http://conduktor-gateway:8888
163 | restart: on-failure
164 | depends_on:
165 | redpanda-0:
166 | condition: service_healthy
167 | conduktor-gateway:
168 | condition: service_healthy
169 | volumes:
170 | pg_data: {}
171 | pg_data_sql: {}
172 | conduktor_data: {}
173 | redpanda-0: {}
174 |
--------------------------------------------------------------------------------
/simple.yml:
--------------------------------------------------------------------------------
1 | services:
2 | postgresql:
3 | image: postgres:14
4 | hostname: postgresql
5 | volumes:
6 | - pg_data:/var/lib/postgresql/data
7 | environment:
8 | POSTGRES_DB: "conduktor-console"
9 | POSTGRES_USER: "conduktor"
10 | POSTGRES_PASSWORD: "change_me"
11 | POSTGRES_HOST_AUTH_METHOD: "scram-sha-256"
12 | healthcheck:
13 | test: ["CMD-SHELL", "pg_isready"]
14 | interval: 10s
15 | timeout: 5s
16 | retries: 5
17 | conduktor-console:
18 | image: conduktor/conduktor-console:1.34.3
19 | ports:
20 | - "8080:8080"
21 | volumes:
22 | - conduktor_data:/var/conduktor
23 | environment:
24 | CDK_ORGANIZATION_NAME: "getting-started"
25 | CDK_DATABASE_URL: "postgresql://conduktor:change_me@postgresql:5432/conduktor-console"
26 | CDK_MONITORING_CORTEX-URL: http://conduktor-monitoring:9009/
27 | CDK_MONITORING_ALERT-MANAGER-URL: http://conduktor-monitoring:9010/
28 | CDK_MONITORING_CALLBACK-URL: http://conduktor-platform:8080/monitoring/api/
29 | CDK_MONITORING_NOTIFICATIONS-CALLBACK-URL: http://localhost:8080
30 | depends_on:
31 | postgresql:
32 | condition: service_healthy
33 | conduktor-monitoring:
34 | image: conduktor/conduktor-console-cortex:1.34.3
35 | environment:
36 | CDK_CONSOLE-URL: "http://conduktor-console:8080"
37 | volumes:
38 | pg_data: {}
39 | conduktor_data: {}
40 |
--------------------------------------------------------------------------------
/sql/README.md:
--------------------------------------------------------------------------------
1 | # 🚀 Conduktor SQL Alpha
2 |
3 | This alpha introduces SQL as a first-class citizen in Conduktor for exploration, troubleshooting and quality purposes. It also makes your Kafka data accessible via API to integrate with your existing dashboards and pipelines (e.g. Grafana, Apache Superset, dbt).
4 |
5 | Note that all SQL queries provides an ad-hoc view of the data inside Kafka at query execution time, it is not a replacement for SQL processing, i.e. push queries (ksqlDB, Flink, Spark).
6 |
7 | # Why
8 |
9 | - Our intention is to reconcile Kafka data with those using SQL (everyone knows SQL, right?)
10 | - Converge Kafka data at source with Analytics data, eliminating unnecessary data pipelines
11 | - Simplify access to data while also facilitating the development towards 'data as a product'
12 | - Faster resolution of Kafka troubleshooting (message data and metadata) through SQL
13 |
14 | # How
15 |
16 | ## Run
17 | ```
18 | docker compose up
19 | ```
20 |
21 | ## Query data inside Conduktor
22 |
23 | When the container is up, you will find a new SQL tab inside the Console interface:
24 |
25 | 
26 |
27 | Run a `DESCRIBE ` query to view the underlying schema, note that you can also query message metadata such as:
28 | - partition, offset, key
29 | - batch compression type, batch size
30 | - schemaId
31 |
32 |
33 | ## Query data outside Conduktor (Clickhouse, MySQL, PostgreSQL)
34 |
35 | To access your Kafka data via SQL outside Conduktor, you can use either clickhouse, mysql or postgres connections.
36 |
37 | 
38 |
39 | ### Connection details
40 |
41 | **Host**: `conduktor-sql`
42 |
43 | **Ports**:
44 | | Connection | Port |
45 | |----------|----------|
46 | | Clickhouse HTTP | `8123` |
47 | | Clickhouse native | `9000` |
48 | | mysql | `9004` |
49 | | postgres | `9005` |
50 |
51 | Note that it's recommended to use Clickhouse or mysql for the best experience today.
52 |
53 | **Database**: ``
54 |
55 | **Authentication**: For Clickhouse, you can use the same credentials used to log into Conduktor. For mysql and postgres, you should use the environment variables from the docker compose (i.e. default values of `conduktor` and `change_me`).
56 |
57 | ## Want to discuss this Alpha? Need support?
58 |
59 | We'd love to chat, get in [contact](https://www.conduktor.io/contact/demo/) with us.
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
--------------------------------------------------------------------------------
/sql/clickhouse.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | 1000
6 | 1000
7 | 1000
8 | 3
9 | 50
10 | 1000
11 |
12 |
13 |
18 |
19 |
--------------------------------------------------------------------------------
/sql/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '3.8'
2 |
3 | services:
4 |
5 | postgresql:
6 | image: postgres:14
7 | hostname: postgresql
8 | volumes:
9 | - pg_data:/var/lib/postgresql/data
10 | environment:
11 | PGDATA: "/var/lib/postgresql/data"
12 | POSTGRES_DB: "conduktor-console"
13 | POSTGRES_USER: "conduktor"
14 | POSTGRES_PASSWORD: "change_me"
15 | POSTGRES_HOST_AUTH_METHOD: "scram-sha-256"
16 |
17 | conduktor-console:
18 | image: harbor.cdkt.dev/public/conduktor-console:1.22.0-sql0
19 | depends_on:
20 | - postgresql
21 | - redpanda-0
22 | ports:
23 | - "8080:8080"
24 | volumes:
25 | - conduktor_data:/var/conduktor
26 | environment:
27 | CDK_DATABASE_URL: "postgresql://conduktor:change_me@postgresql:5432/conduktor-console"
28 | CDK_CLUSTERS_0_ID: "local-kafka"
29 | CDK_CLUSTERS_0_NAME: "local-kafka"
30 | CDK_CLUSTERS_0_BOOTSTRAPSERVERS: "redpanda-0:9092"
31 | CDK_CLUSTERS_0_SCHEMAREGISTRY_URL: "http://redpanda-0:18081"
32 | CDK_CLUSTERS_0_COLOR: "#6A57C8"
33 | CDK_CLUSTERS_0_ICON: "kafka"
34 | CDK_CLUSTERS_1_ID: "cdk-gateway"
35 | CDK_CLUSTERS_1_NAME: "cdk-gateway"
36 | CDK_CLUSTERS_1_BOOTSTRAPSERVERS: "conduktor-gateway:6969"
37 | CDK_CLUSTERS_1_SCHEMAREGISTRY_URL: "http://redpanda-0:18081"
38 | CDK_CLUSTERS_1_KAFKAFLAVOR_URL: "http://conduktor-gateway:8888"
39 | CDK_CLUSTERS_1_KAFKAFLAVOR_USER: "admin"
40 | CDK_CLUSTERS_1_KAFKAFLAVOR_PASSWORD: "conduktor"
41 | CDK_CLUSTERS_1_KAFKAFLAVOR_VIRTUALCLUSTER: "passthrough"
42 | CDK_CLUSTERS_1_KAFKAFLAVOR_TYPE: "Gateway"
43 | CDK_CLUSTERS_1_COLOR: "#6A57C8"
44 | CDK_CLUSTERS_1_ICON: "dog"
45 | CDK_MONITORING_CORTEX-URL: http://conduktor-monitoring:9009/
46 | CDK_MONITORING_ALERT-MANAGER-URL: http://conduktor-monitoring:9010/
47 | CDK_MONITORING_CALLBACK-URL: http://conduktor-console:8080/monitoring/api/
48 | CDK_MONITORING_NOTIFICATIONS-CALLBACK-URL: http://localhost:8080
49 | CDK_SQL_CLICKHOUSE-URL: "http://conduktor:change_me@clickhouse:8123"
50 | CDK_SQL_MARKDOWN-URL: "https://raw.githubusercontent.com/haianh1233/markdown-test/main/"
51 |
52 | conduktor-monitoring:
53 | image: conduktor/conduktor-console-cortex:1.22.0
54 | environment:
55 | CDK_CONSOLE-URL: "http://conduktor-console:8080"
56 |
57 | redpanda-0:
58 | command:
59 | - redpanda
60 | - start
61 | - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
62 | - --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
63 | - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
64 | # Address the broker advertises to clients that connect to the HTTP Proxy.
65 | - --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
66 | - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
67 | # Redpanda brokers use the RPC API to communicate with eachother internally.
68 | - --rpc-addr redpanda-0:33145
69 | - --advertise-rpc-addr redpanda-0:33145
70 | - --smp 1
71 | - --memory 1G
72 | - --mode dev-container
73 | - --default-log-level=info
74 | image: docker.redpanda.com/redpandadata/redpanda:v23.1.11
75 | container_name: redpanda-0
76 | volumes:
77 | - redpanda-0:/var/lib/redpanda/data
78 | ports:
79 | - 18081:18081
80 | - 18082:18082
81 | - 19092:19092
82 | - 19644:9644
83 | healthcheck:
84 | test: ["CMD-SHELL", "rpk cluster health | grep -E 'Healthy:.+true' || exit 1"]
85 | interval: 15s
86 | timeout: 3s
87 | retries: 5
88 | start_period: 5s
89 |
90 | conduktor-gateway:
91 | image: conduktor/conduktor-gateway:3.0.1
92 | hostname: conduktor-gateway
93 | container_name: conduktor-gateway
94 | environment:
95 | KAFKA_BOOTSTRAP_SERVERS: redpanda-0:9092
96 | ports:
97 | - "8888:8888"
98 | healthcheck:
99 | test: curl localhost:8888/health
100 | interval: 5s
101 | retries: 25
102 | depends_on:
103 | redpanda-0:
104 | condition: service_healthy
105 |
106 | conduktor-data-generator:
107 | image: conduktor/conduktor-data-generator:0.3
108 | container_name: conduktor-data-generator
109 | environment:
110 | KAFKA_BOOTSTRAP_SERVERS: conduktor-gateway:6969
111 | KAFKA_SCHEMA_REGISTRY_URL: http://redpanda-0:8081
112 | GATEWAY_ADMIN_API: http://conduktor-gateway:8888
113 | restart: on-failure
114 | depends_on:
115 | redpanda-0:
116 | condition: service_healthy
117 | conduktor-gateway:
118 | condition: service_healthy
119 |
120 | conduktor-sql:
121 | image: clickhouse/clickhouse-server
122 | hostname: clickhouse
123 | ports:
124 | - "8123:8123" # http
125 | - "9000:9000" # native
126 | - "9004:9004" # MySQL
127 | - "9005:9005" # PostgreSQL
128 | environment:
129 | CLICKHOUSE_USER: "conduktor"
130 | CLICKHOUSE_PASSWORD: "change_me"
131 | CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
132 | CDK_LOGIN_URL: http://conduktor-console:8080/auth/login
133 | volumes:
134 | - ./clickhouse.xml:/etc/clickhouse-server/config.d/config.xml
135 | - conduktor_sql_data:/var/lib/clickhouse
136 |
137 | volumes:
138 | pg_data: {}
139 | conduktor_data: {}
140 | redpanda-0: {}
141 | conduktor_sql_data: {}
142 |
--------------------------------------------------------------------------------