├── .babelrc.js
├── .gitignore
├── LICENSE
├── README.md
├── app
├── LaunchWindow.jsx
├── MainWindow.jsx
└── MetricTray.jsx
├── build
├── 2e0f6edb704eeb6183ad.jpg
├── bundle.js
└── bundle.js.LICENSE.txt
├── configs
├── alertmanager
│ └── alertmanager_template.yml
├── configGenerator.js
├── docker
│ ├── docker_multiple_nodes_template.yml
│ └── docker_single_node_template.yml
├── grafana
│ ├── provisioning
│ │ ├── dashboards
│ │ │ └── kafka.yaml
│ │ └── datasources
│ │ │ └── datasource.yaml
│ └── templates
│ │ ├── broker_hard_disk_usage.json
│ │ ├── broker_jvm_os.json
│ │ ├── broker_performance.json
│ │ ├── broker_zookeeper.json
│ │ ├── client_consumers_fetch_lag.json
│ │ ├── cluster_healthcheck.json
│ │ ├── cluster_replication.json
│ │ ├── panels_template.js
│ │ └── topics_logs.json
├── jmx_exporter
│ ├── config_kafka_template.yml
│ └── metric_list.js
└── prometheus
│ ├── alert_rules.yml
│ └── prometheus_template.yml
├── electron.js
├── package-lock.json
├── package.json
├── src
├── assets
│ ├── app-logo.png
│ ├── github-logo.jpg
│ ├── icon-mac.png
│ ├── icon-windows.png
│ ├── launch-demo.gif
│ ├── main-demo.gif
│ └── white-icon.png
├── components
│ ├── App.jsx
│ ├── BrokerHardDiskUsage.jsx
│ ├── BrokerJVMAndOS.jsx
│ ├── BrokerPerformance.jsx
│ ├── BrokerZookeeper.jsx
│ ├── ClusterHealthCheck.jsx
│ ├── ClusterReplication.jsx
│ ├── GrafanaDash.jsx
│ ├── HelpTab.jsx
│ ├── Home.jsx
│ ├── Launch.jsx
│ ├── MetricCard.jsx
│ ├── Sidebar.jsx
│ ├── TopicsLogs.jsx
│ ├── _utils
│ │ ├── displayMetrics.js
│ │ └── renderMetricPanels.js
│ └── alerts.jsx
├── index.html
├── index.js
├── launch.html
├── models
│ └── metricURLs.js
└── styles
│ ├── styles.css
│ ├── styles.css.map
│ └── styles.scss
└── webpack.config.js
/.babelrc.js:
--------------------------------------------------------------------------------
1 | // only bundles the necessary material UI elements to reduce bundle size / compilation time
2 |
3 | const plugins = [
4 | [
5 | 'babel-plugin-import',
6 | {
7 | libraryName: '@mui/material',
8 | libraryDirectory: '',
9 | camel2DashComponentName: false,
10 | },
11 | 'core',
12 | ],
13 | [
14 | 'babel-plugin-import',
15 | {
16 | libraryName: '@mui/icons-material',
17 | libraryDirectory: '',
18 | camel2DashComponentName: false,
19 | },
20 | 'icons',
21 | ],
22 | ];
23 |
24 | module.exports = { plugins };
25 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
2 |
3 | # auto-generated config files
4 | configs/grafana/dashboards/*
5 | configs/docker/docker_multiple_nodes.yml
6 | configs/prometheus/prometheus.yml
7 | configs/jmx_exporter/config_kafka1*
8 | configs/alertmanager/alertmanager.yml
9 | configs/docker/docker_single_node.yml
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2022 OSLabs Beta
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
Intuitive, customizable Kafka cluster metrics
6 |
7 | Product Website »
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 | ---
18 |
19 | ## Table of Contents 🗺️
20 |
21 | - [About Kaffia](#about)
22 | - [Features](#features)
23 | - [Getting Started](#getting-started)
24 | - [Contributors](#contributors)
25 | - [Become a contributor](#contribute)
26 | - [License](#license)
27 |
28 | ---
29 |
30 | ## About Kaffia 📙
31 |
32 | Apache Kafka is one of the most widely used stream-processing platforms, yet it lacks a free, easy-to-use GUI to monitor key Kafka cluster metrics. Without a comprehensive monitoring tool, the barrier to entry of Apache Kafka remains high, and engineers working with Kafka clusters may miss key issues as they come up.
33 |
34 | That's why we created Kaffia. Kaffia is an open-source, intuitive GUI for Kafka clusters that allows you to tailor Kafka cluster monitoring to your needs and experience level. All you have to do is finish some quick setup, launch the app, input your monitoring and Kafka broker preferences, and let Kaffia handle the rest!
35 |
36 | Read on to see some of our core features and learn how you can get Kaffia up and running on your machine.
37 |
38 | ---
39 |
40 | ## Core Features 😊
41 |
42 | ### Easy setup ✅
43 |
44 | - Easily configure your Kafka cluster broker count, monitor certain metrics, and sign up for email alerts
45 | - One button launches your cluster, metrics scrapers, visualization service, and alert manager—automatically!
46 |
47 | ### Comprehensive visualization 🔎
48 |
49 | - View key Kafka metrics (broker count, throughput, topic size, etc.)
50 | - Live updates as you produce to and consume from your cluster
51 | - Intuitive GUI to make parsing complex data less cumbersome
52 |
53 | ### Quickly launch and stop your cluster ⚡️
54 |
55 | - Containerize your cluster in Docker Desktop with the click of a button
56 | - Easily shut down your cluster from the Kaffia dashboard
57 |
58 | ---
59 |
60 | ## Getting Started with Kaffia ⬆️
61 |
62 | Kaffia automates cluster configuration and launching by creating a Docker application that containerizes everything from Zookeeper to the metrics scraper. Make sure you have [Docker Desktop](https://www.docker.com/products/docker-desktop/) up and running before you launch Kaffia.
63 |
64 | To get started, fork our repository and clone it to your local machine. To install all dependencies, run the following:
65 |
66 | ```sh
67 | npm install
68 | ```
69 |
70 | After the dependencies install, you're good to go! Just spin up the app by running this command inside of the Kaffia directory:
71 |
72 | ```sh
73 | npm start
74 | ```
75 |
76 | Once Kaffia is up and running, configuring and monitoring your cluster is simple! Just choose your broker count and metrics from the launch screen, hit submit, and watch your cluster launch automatically in Docker Desktop!
77 |
78 |
79 |
80 | After the cluster launches, you'll be able to navigate throughout the app and view different key metrics that will help you monitor your cluster's health without having to do any setup on your end. Enjoy!
81 |
82 |
83 |
84 | ---
85 |
86 | ## Contributors 👋
87 |
88 | - Liz Blackledge
89 | - Aiden Blinn
90 | - Ritchie Cervantes
91 | - Jonathan Oh
92 |
93 | ### Contribute to Kaffia 💪
94 |
95 | We welcome any and all contributions to Kaffia! You can reach out to one of us on LinkedIn if you have any ideas, or you can fork the repository, make some changes, and submit a pull request.
96 |
97 | ---
98 |
99 | ## License 🧐
100 |
101 | MIT License
102 |
--------------------------------------------------------------------------------
/app/LaunchWindow.jsx:
--------------------------------------------------------------------------------
1 | /**
2 | * LaunchWindow takes advantage of Electron's built-in
3 | * BrowserWindow to create a window with consistent sizing
4 | * and settings tailored to its intended use.
5 | */
6 |
7 | const electron = require('electron');
8 |
9 | const { BrowserWindow, app } = electron;
10 |
11 | class LaunchWindow extends BrowserWindow {
12 | constructor(url) {
13 | super({
14 | webPreferences: {
15 | nodeIntegration: true,
16 | contextIsolation: false,
17 | backgroundThrottling: false,
18 | },
19 | height: 700,
20 | width: 500,
21 | frame: true,
22 | autoHideMenuBar: false,
23 | resizable: process.env.NODE_ENV === 'development',
24 | show: true,
25 | });
26 | this.on('closed', () => app.quit());
27 | this.loadURL(url);
28 | }
29 | }
30 |
31 | module.exports = LaunchWindow;
32 |
--------------------------------------------------------------------------------
/app/MainWindow.jsx:
--------------------------------------------------------------------------------
1 | /**
2 | * MainWindow takes advantage of Electron's built-in
3 | * BrowserWindow to create a window with consistent sizing
4 | * and settings tailored to its intended use.
5 | */
6 |
7 | const electron = require('electron');
8 |
9 | const { BrowserWindow, app } = electron;
10 |
11 | class MainWindow extends BrowserWindow {
12 | constructor(url) {
13 | super({
14 | webPreferences: {
15 | nodeIntegration: true,
16 | contextIsolation: false,
17 | backgroundThrottling: false,
18 | },
19 | height: 800,
20 | minHeight: 500,
21 | width: 1100,
22 | minWidth: 800,
23 | frame: true,
24 | autoHideMenuBar: false,
25 | resizable: true,
26 | show: true,
27 | });
28 | this.on('closed', () => app.quit());
29 | this.loadURL(url);
30 | }
31 | }
32 |
33 | module.exports = MainWindow;
34 |
--------------------------------------------------------------------------------
/app/MetricTray.jsx:
--------------------------------------------------------------------------------
1 | /**
2 | * The Metric Tray is a feature for a future release. This tray
3 | * is designed to provide the user a quick glance at core metrics
4 | * by just clicking on an icon in the user's menu without having
5 | * to open the whole app and look at the entire dashboard.
6 | */
7 |
8 | const electron = require('electron');
9 | const { Tray, Menu } = electron;
10 |
11 | class MetricTray extends Tray {
12 | constructor(iconPath, popupWindow) {
13 | super(iconPath);
14 | this.popupWindow = popupWindow;
15 | this.on('click', this.onClick);
16 | this.on('right-click', this.onRightClick);
17 | this.setToolTip('Kaffia');
18 | }
19 |
20 | onClick = (event, bounds) => {
21 | const { x, y } = bounds;
22 | const { height, width } = this.popupWindow.getBounds();
23 |
24 | if (this.popupWindow.isVisible()) {
25 | this.popupWindow.hide();
26 | } else {
27 | this.popupWindow.setBounds({
28 | x: x - width / 2,
29 | y: process.platform === 'darwin' ? y : y - height,
30 | height,
31 | width,
32 | });
33 | this.popupWindow.show();
34 | }
35 | };
36 |
37 | onRightClick = (event) => {
38 | const menuConfig = Menu.buildFromTemplate([{ role: 'quit' }]);
39 | this.popUpContextMenu(menuConfig);
40 | };
41 | }
42 |
43 | module.exports = MetricTray;
44 |
--------------------------------------------------------------------------------
/build/2e0f6edb704eeb6183ad.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/oslabs-beta/Kaffia/4b585a0b94846f460c865e2f08103d25d42d0020/build/2e0f6edb704eeb6183ad.jpg
--------------------------------------------------------------------------------
/build/bundle.js.LICENSE.txt:
--------------------------------------------------------------------------------
1 | /** @license MUI v5.5.2
2 | *
3 | * This source code is licensed under the MIT license found in the
4 | * LICENSE file in the root directory of this source tree.
5 | */
6 |
7 | /** @license React v17.0.2
8 | * react-jsx-runtime.production.min.js
9 | *
10 | * Copyright (c) Facebook, Inc. and its affiliates.
11 | *
12 | * This source code is licensed under the MIT license found in the
13 | * LICENSE file in the root directory of this source tree.
14 | */
15 |
--------------------------------------------------------------------------------
/configs/alertmanager/alertmanager_template.yml:
--------------------------------------------------------------------------------
1 | global:
2 | resolve_timeout: 1m
3 | route:
4 | receiver: 'gmail-notifications'
5 | receivers:
6 | - name: 'gmail-notifications'
7 | email_configs:
8 | - to:
9 | from: kaffiamonitor@gmail.com
10 | smarthost: smtp.gmail.com:587
11 | auth_username: kaffiamonitor@gmail.com
12 | auth_identity: kaffiamonitor@gmail.com
13 | auth_password: dhfyxdzixrvlhotl
14 | send_resolved: true
15 |
--------------------------------------------------------------------------------
/configs/configGenerator.js:
--------------------------------------------------------------------------------
1 | const yaml = require('js-yaml');
2 | const fs = require('fs');
3 | const path = require('path');
4 |
5 | const grafanaPanels = require('./grafana/templates/panels_template.js');
6 | const jmxMetrics = require('./jmx_exporter/metric_list.js');
7 |
8 | /**
9 | * dockerConfigGenerator creates a yaml file for a multi-container Docker application
10 | * that uses Grafana, Prometheus, jmx-exporter, and Kafka to create a self-monitoring
11 | * cluster.
12 | * @param brokerCount: user-specified number of Kafka brokers
13 | * @returns void
14 | *
15 | */
16 |
17 | const dockerConfigGenerator = (brokerCount, email) => {
18 | if (brokerCount === 1) {
19 | try {
20 | const dockerConfig = yaml.load(
21 | fs.readFileSync(
22 | path.join(__dirname, './docker/docker_single_node_template.yml'),
23 | 'utf8'
24 | )
25 | );
26 | if (email) {
27 | dockerConfig.services.alertManager = {
28 | image: 'prom/alertmanager:v0.23.0',
29 | restart: 'unless-stopped',
30 | ports: ['9096:9096'],
31 | volumes: ['../alertmanager:/config', 'alertmanager-data:/data'],
32 | command: '--config.file=/config/alertmanager.yml --log.level=debug',
33 | depends_on: ['prometheus'],
34 | };
35 | }
36 | return fs.writeFileSync(
37 | path.join(__dirname, 'docker/docker_single_node.yml'),
38 | yaml.dump(dockerConfig, { noRefs: true })
39 | );
40 | } catch (e) {
41 | return console.log(e);
42 | }
43 | }
44 | try {
45 | // load in the multi-container Docker yaml template
46 | const dockerConfig = yaml.load(
47 | fs.readFileSync(
48 | path.join(__dirname, './docker/docker_multiple_nodes_template.yml'),
49 | 'utf8'
50 | )
51 | );
52 |
53 | // define the properties for the jmx-exporter and Kafka cluster services
54 | // that are the same in each cluster
55 | let jmxConfig = {
56 | image: 'sscaling/jmx-prometheus-exporter',
57 | environment: {
58 | CONFIG_YML: '/../jmx_exporter/config.yml',
59 | JVM_OPTS: '-Xmx512M',
60 | },
61 | };
62 | let kafkaConfig = {
63 | image: 'confluentinc/cp-kafka:latest',
64 | depends_on: ['zk1'],
65 | environment: {
66 | KAFKA_ZOOKEEPER_CONNECT: 'zookeeper1:2181',
67 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP:
68 | 'PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT',
69 | KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT',
70 | CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: 'zookeeper1:2181',
71 | CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1,
72 | CONFLUENT_METRICS_ENABLE: 'false',
73 | KAFKA_HEAP_OPTS: '-Xmx512M -Xms512M',
74 | },
75 | };
76 |
77 | // generate the unique, cluster-specific properties for the jmx-exporter
78 | // and Kafka services according to the user's preferred broker count
79 | for (let i = 0; i < brokerCount; i++) {
80 | jmxConfig = {
81 | ...jmxConfig,
82 | ports: [`${5556 + i}:5556`],
83 | volumes: [
84 | `../jmx_exporter/config_kafka10${
85 | i + 1
86 | }.yml:/../jmx_exporter/config.yml`,
87 | ],
88 | container_name: `jmx-kafka${101 + i}`,
89 | depends_on: [`kafka${101 + i}`],
90 | };
91 | dockerConfig.services[`jmx-kafka${101 + i}`] = jmxConfig;
92 |
93 | kafkaConfig = {
94 | ...kafkaConfig,
95 | ports: [`909${i + 1}:909${i + 1}`, `999${i + 1}:999${i + 1}`],
96 | container_name: `kafka${101 + i}`,
97 | environment: {
98 | ...kafkaConfig.environment,
99 | KAFKA_BROKER_ID: 101 + i,
100 | KAFKA_JMX_PORT: 9991 + i,
101 | KAFKA_ADVERTISED_LISTENERS: `PLAINTEXT://kafka10${
102 | i + 1
103 | }:29092,PLAINTEXT_HOST://localhost:909${i + 1}`,
104 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR:
105 | brokerCount < 3 ? brokerCount : 3,
106 | KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR:
107 | brokerCount < 3 ? brokerCount : 3,
108 | CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: `kafka10${i + 1}:29092`,
109 | },
110 | };
111 | dockerConfig.services[`kafka${101 + i}`] = kafkaConfig;
112 | }
113 |
114 | if (email) {
115 | dockerConfig.services.alertManager = {
116 | image: 'prom/alertmanager:v0.23.0',
117 | restart: 'unless-stopped',
118 | ports: ['9096:9096'],
119 | volumes: ['../alertmanager:/config', 'alertmanager-data:/data'],
120 | command: '--config.file=/config/alertmanager.yml --log.level=debug',
121 | depends_on: ['prometheus'],
122 | };
123 | }
124 | // after adding the required services for each broker, save the completed template
125 | // to a filepath that will then be used to launch the Docker app
126 | fs.writeFileSync(
127 | path.join(__dirname, './docker/docker_multiple_nodes.yml'),
128 | yaml.dump(dockerConfig, { noRefs: true })
129 | );
130 | } catch (e) {
131 | return console.log(e);
132 | }
133 | };
134 |
135 | /**
136 | * promConfigGenerator creates a yaml file for a multi-container Docker application
137 | * to enable Prometheus monitoring of the jmx-exporter exposing Kafka metrics
138 | * @param brokerCount: user-specified number of Kafka brokers
139 | * @returns void
140 | *
141 | */
142 |
143 | const promConfigGenerator = (brokerCount, email) => {
144 | try {
145 | // read in prometheus.yml template file and add jmx-exporter ports depending on
146 | // the user's preferred number of Kafka brokers
147 | const promConfig = yaml.load(
148 | fs.readFileSync(
149 | path.join(__dirname, 'prometheus/prometheus_template.yml'),
150 | 'utf8'
151 | )
152 | );
153 | const promTargets = [];
154 | for (let i = 0; i < brokerCount; i++) {
155 | promTargets.push(`jmx-kafka10${i + 1}:5556`);
156 | }
157 |
158 | // add ports to scrape to the yml file and save changes into completed yml file
159 | promConfig.scrape_configs[0].static_configs[0].targets = promTargets;
160 |
161 | if (email) {
162 | promConfig.rule_files[0] = ['alert_rules.yml'];
163 | promConfig.alerting.alertmanagers[0].static_configs[0].targets[0] = [
164 | 'alertmanager:9096',
165 | ];
166 | }
167 |
168 | fs.writeFileSync(
169 | path.join(__dirname, 'prometheus/prometheus.yml'),
170 | yaml.dump(promConfig, { noRefs: true })
171 | );
172 | } catch (e) {
173 | console.log(e);
174 | }
175 | };
176 |
177 | /**
178 | * promConfigGenerator creates multiple yaml files for the jmx-exporters to only
179 | * expose the metrics that the user would like to view. Additionally, the method
180 | * constructs Grafana dashboards using those user-specified metrics.
181 | * @param brokerCount: user-specified number of Kafka brokers
182 | * @returns void
183 | *
184 | */
185 |
186 | const jvmGrafanaConfigGenerator = (brokerCount, userMetrics) => {
187 | // create Grafana dashboard folder if it does not already exist
188 | if (!fs.existsSync(path.join(__dirname, 'grafana/dashboards'))) {
189 | fs.mkdirSync(path.join(__dirname, 'grafana/dashboards'));
190 | }
191 | // delete existing dashboards from previous cluster creation, which may have different settings
192 | const existingDashboards = fs.readdirSync(
193 | path.join(__dirname, 'grafana/dashboards')
194 | );
195 | existingDashboards.forEach((dashboard) => {
196 | fs.unlinkSync(path.join(__dirname, 'grafana/dashboards', dashboard));
197 | });
198 |
199 | // load template for jmx-exporter to add specific metrics to whtielist
200 | const config_kafka_template = yaml.load(
201 | fs.readFileSync(
202 | path.join(__dirname, 'jmx_exporter/config_kafka_template.yml'),
203 | 'utf8'
204 | )
205 | );
206 |
207 | // loop through each user-selected dashboard to add panels to Prometheus file
208 | for (const dashboard in userMetrics) {
209 | // grab Grafana file corresponding to selected dashboard
210 | const grafanaFile = JSON.parse(
211 | fs.readFileSync(
212 | path.join(__dirname, `grafana/templates/${dashboard}.json`),
213 | 'utf-8'
214 | )
215 | );
216 | // add jmx-exporter metric to whitelist to scrape that item
217 | // add panel to Grafana dashboard
218 | for (const panel of userMetrics[dashboard]) {
219 | // jmxMetrics[dashboard][panel].forEach(whitelist.add, whitelist);
220 | grafanaFile.panels.push(...grafanaPanels[dashboard][panel]);
221 | }
222 | fs.writeFileSync(
223 | path.join(__dirname, `./grafana/dashboards/${dashboard}.json`),
224 | JSON.stringify(grafanaFile),
225 | { noRefs: true }
226 | );
227 | }
228 | // config_kafka_template.whitelistObjectNames = [...whitelist];
229 |
230 | // save jmx-exporter config files with correct ports
231 | for (let i = 0; i < brokerCount; i++) {
232 | config_kafka_template.hostPort = `kafka10${i + 1}:999${i + 1}`;
233 | fs.writeFileSync(
234 | path.join(__dirname, `./jmx_exporter/config_kafka10${i + 1}.yml`),
235 | yaml.dump(config_kafka_template, { noRefs: true })
236 | );
237 | }
238 | };
239 |
240 | /**
241 | * alertConfigGenerator creates a yaml file to alert the user
242 | * about critical cluster issues using the email provided on the
243 | * launch page form.
244 | * @param email: user-specified email address
245 | * @returns void
246 | *
247 | */
248 |
249 | const alertConfigGenerator = (email) => {
250 | try {
251 | // read in alert manager template file and update with email
252 | // provided by user
253 | const alertManager = yaml.load(
254 | fs.readFileSync(
255 | path.join(__dirname, 'alertmanager/alertmanager_template.yml'),
256 | 'utf8'
257 | )
258 | );
259 | alertManager.receivers[0].email_configs[0].to = email;
260 | // save updated file to be used when Docker launches
261 | // the necessary containerized services
262 | fs.writeFileSync(
263 | path.join(__dirname, 'alertmanager/alertmanager.yml'),
264 | yaml.dump(alertManager, { noRefs: true })
265 | );
266 | } catch (e) {
267 | console.log(e);
268 | }
269 | };
270 |
271 | module.exports = (brokerCount, metrics, email) => {
272 | // run all three (or four) config methods each time user submits form with preferences
273 | promConfigGenerator(brokerCount, email);
274 | jvmGrafanaConfigGenerator(brokerCount, metrics);
275 | if (email) alertConfigGenerator(email);
276 | dockerConfigGenerator(brokerCount, email);
277 | };
278 |
--------------------------------------------------------------------------------
/configs/docker/docker_multiple_nodes_template.yml:
--------------------------------------------------------------------------------
1 | version: '3'
2 | services:
3 | grafana:
4 | image: 'grafana/grafana'
5 | ports:
6 | - '3000:3000'
7 | environment:
8 | GF_PATHS_DATA: /var/lib/grafana
9 | GF_SECURITY_ALLOW_EMBEDDING: 'true'
10 | GF_AUTH_ANONYMOUS_ENABLED: 'true'
11 | GF_SMTP_ENABLED: 'true'
12 | GF_SECURITY_ADMIN_PASSWORD: kaffia
13 | volumes:
14 | - ../grafana/provisioning:/etc/grafana/provisioning
15 | - ../grafana/dashboards:/var/lib/grafana/dashboards
16 | container_name: grafana
17 | depends_on:
18 | - prometheus
19 | prometheus:
20 | image: 'prom/prometheus'
21 | ports:
22 | - '9090:9090'
23 | volumes:
24 | - ../prometheus/prometheus.yml:/../prometheus/prometheus.yml
25 | - ../prometheus/alert_rules.yml:/../prometheus/alert_rules.yml
26 | command: '--config.file=/../prometheus/prometheus.yml'
27 | container_name: prometheus
28 | zk1:
29 | image: confluentinc/cp-zookeeper:latest
30 | environment:
31 | ZOOKEEPER_CLIENT_PORT: 2181
32 | ZOOKEEPER_TICK_TIME: 2000
33 | ZOOKEEPER_INIT_LIMIT: 5
34 | ZOOKEEPER_SYNC_LIMIT: 2
35 | ports:
36 | - 2181:2181
37 | container_name: zookeeper1
38 | volumes:
39 | alertmanager-data:
40 |
--------------------------------------------------------------------------------
/configs/docker/docker_single_node_template.yml:
--------------------------------------------------------------------------------
1 | version: '3'
2 | services:
3 | grafana:
4 | image: 'grafana/grafana'
5 | ports:
6 | - '3000:3000'
7 | environment:
8 | GF_PATHS_DATA: /var/lib/grafana
9 | GF_SECURITY_ALLOW_EMBEDDING: 'true'
10 | GF_AUTH_ANONYMOUS_ENABLED: 'true'
11 | GF_ALERTING_ENABLED: 'false'
12 | GF_SMTP_ENABLED: 'true'
13 | GF_SECURITY_ADMIN_PASSWORD: kaffia
14 | volumes:
15 | - ../grafana/provisioning:/etc/grafana/provisioning
16 | - ../grafana/dashboards:/var/lib/grafana/dashboards
17 | container_name: grafana
18 | depends_on:
19 | - prometheus
20 | prometheus:
21 | image: 'prom/prometheus'
22 | ports:
23 | - '9090:9090'
24 | volumes:
25 | - ../prometheus/prometheus.yml:/../prometheus/prometheus.yml
26 | - ../prometheus/alert_rules.yml:/../prometheus/alert_rules.yml
27 | command: '--config.file=/../prometheus/prometheus.yml'
28 | container_name: prometheus
29 | zk1:
30 | image: confluentinc/cp-zookeeper:latest
31 | environment:
32 | ZOOKEEPER_CLIENT_PORT: 2181
33 | ZOOKEEPER_TICK_TIME: 2000
34 | ZOOKEEPER_INIT_LIMIT: 5
35 | ZOOKEEPER_SYNC_LIMIT: 2
36 | ports:
37 | - 2181:2181
38 | container_name: zookeeper1
39 | jmx-kafka101:
40 | image: 'sscaling/jmx-prometheus-exporter'
41 | ports:
42 | - '5556:5556'
43 | environment:
44 | CONFIG_YML: '/../jmx_exporter/config.yml'
45 | volumes:
46 | - ./../jmx_exporter/config_kafka101.yml:/../jmx_exporter/config.yml
47 | container_name: jmx-kafka101
48 | depends_on:
49 | - kafka101
50 | kafka101:
51 | image: confluentinc/cp-kafka:latest
52 | depends_on:
53 | - zk1
54 | ports:
55 | - 9092:9092
56 | - 9991:9991
57 | container_name: kafka101
58 | environment:
59 | KAFKA_BROKER_ID: 101
60 | KAFKA_JMX_PORT: 9991
61 | KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181
62 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka101:29092,PLAINTEXT_HOST://localhost:9092
63 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
64 | KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
65 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
66 | KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
67 | KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
68 | CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka101:29092
69 | CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper1:2181
70 | CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
71 | CONFLUENT_METRICS_ENABLE: 'false'
72 | KAFKA_HEAP_OPTS: '-Xmx512M -Xms512M'
73 | volumes:
74 | alertmanager-data:
75 |
--------------------------------------------------------------------------------
/configs/grafana/provisioning/dashboards/kafka.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: 1
2 |
3 | providers:
4 | # an unique provider name. Required
5 | - name: 'default'
6 | # Org id. Default to 1
7 | orgId: 1
8 | # name of the dashboard folder.
9 | folder: ''
10 | # folder UID. will be automatically generated if not specified
11 | folderUid: ''
12 | # provider type. Default to 'file'
13 | type: file
14 | # disable dashboard deletion
15 | disableDeletion: false
16 | # how often Grafana will scan for changed dashboards
17 | updateIntervalSeconds: 5
18 | # allow updating provisioned dashboards from the UI
19 | allowUiUpdates: false
20 | options:
21 | # path to dashboard files on disk. Required when using the 'file' type
22 | path: /var/lib/grafana/dashboards
23 |
--------------------------------------------------------------------------------
/configs/grafana/provisioning/datasources/datasource.yaml:
--------------------------------------------------------------------------------
1 | # config file version
2 | apiVersion: 1
3 |
4 | # list of datasources that should be deleted from the database
5 | deleteDatasources:
6 | - name: Prometheus
7 | orgId: 1
8 |
9 | # list of datasources to insert/update depending
10 | # what's available in the database
11 | datasources:
12 | # name of the datasource. Required
13 | - name: Prometheus
14 | # datasource type. Required
15 | type: prometheus
16 | # access mode. proxy or direct (Server or Browser in the UI). Required
17 | access: direct
18 | # org id. will default to orgId 1 if not specified
19 | orgId: 1
20 | # url
21 | url: http://localhost:9090
22 | # database password, if used
23 | password:
24 | # database user, if used
25 | user:
26 | # database name, if used
27 | database:
28 | # enable/disable basic auth
29 | basicAuth:
30 | # basic auth username
31 | basicAuthUser:
32 | # basic auth password
33 | basicAuthPassword:
34 | # enable/disable with credentials headers
35 | withCredentials:
36 | # mark as default datasource. Max one per org
37 | isDefault: true
38 | #