├── .gitignore
├── client
├── assets
│ ├── beancarry.png
│ ├── Untitled_Artwork.gif
│ ├── brewkafka_logo_closecrop.png
│ └── brewkafka_logo_original.png
├── components
│ ├── pages
│ │ ├── Alerts.jsx
│ │ ├── Brokers.jsx
│ │ ├── Topics.jsx
│ │ ├── ProducersAndConsumers.jsx
│ │ └── Loginpage.jsx
│ ├── app.jsx
│ ├── inputBar.jsx
│ └── Dashboards.jsx
├── index.js
├── index.html
├── services
│ └── KafkaService.js
└── style.css
├── server
├── kafkaInstance.js
├── prometheus.yml
├── controllers
│ └── kafkacontroller.js
├── services
│ └── prometheusConfigWriter.js
└── server.js
├── Dockerfile
├── grafana
├── provisioning
│ ├── datasources
│ │ └── datasource.yml
│ └── dashboards
│ │ └── dashboards.yml
└── dashboards
│ ├── messages in per sec topics-1701184634460.json
│ ├── KMinion Cluster Dashboard - Prometheus Exporter for Apache Kafka-1701184605403.json
│ └── nickdockerdashboards.json
├── webpack.config.js
├── docker-compose.yml
├── package.json
├── README.md
└── grafana.ini
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
2 | dist
3 | .DS_Store
4 | client/.DS_Store
--------------------------------------------------------------------------------
/client/assets/beancarry.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/oslabs-beta/brewkafka/HEAD/client/assets/beancarry.png
--------------------------------------------------------------------------------
/client/assets/Untitled_Artwork.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/oslabs-beta/brewkafka/HEAD/client/assets/Untitled_Artwork.gif
--------------------------------------------------------------------------------
/server/kafkaInstance.js:
--------------------------------------------------------------------------------
1 | const { Kafka } = require('kafkajs');
2 | const express = require('express');
3 | const app = express();
4 |
5 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM node:20.6.0
2 | WORKDIR /app
3 | COPY package*.json ./
4 | RUN npm install
5 | COPY . .
6 | EXPOSE 8080
7 | CMD ["npm", "start"]
--------------------------------------------------------------------------------
/client/assets/brewkafka_logo_closecrop.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/oslabs-beta/brewkafka/HEAD/client/assets/brewkafka_logo_closecrop.png
--------------------------------------------------------------------------------
/client/assets/brewkafka_logo_original.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/oslabs-beta/brewkafka/HEAD/client/assets/brewkafka_logo_original.png
--------------------------------------------------------------------------------
/client/components/pages/Alerts.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | export function Alerts () {
4 | return (
5 | <>
6 | Alerts Dashboard
7 |
8 | )
9 | }
--------------------------------------------------------------------------------
/grafana/provisioning/datasources/datasource.yml:
--------------------------------------------------------------------------------
1 | apiVersion: 1
2 |
3 | datasources:
4 | - name: Prometheus
5 | type: prometheus
6 | access: proxy
7 | orgId: 1
8 | url: http://prometheus:9090
9 | isDefault: true
10 |
--------------------------------------------------------------------------------
/grafana/provisioning/dashboards/dashboards.yml:
--------------------------------------------------------------------------------
1 | apiVersion: 1
2 |
3 | providers:
4 | - name: 'default'
5 | orgId: 1
6 | folder: ''
7 | type: file
8 | disableDeletion: false
9 | editable: true
10 | options:
11 | path: /var/lib/grafana/dashboards
12 |
--------------------------------------------------------------------------------
/server/prometheus.yml:
--------------------------------------------------------------------------------
1 | global:
2 | scrape_interval: 15s
3 | evaluation_interval: 15s
4 | alerting:
5 | alertmanagers:
6 | - static_configs:
7 | - targets: null
8 | rule_files: null
9 | scrape_configs:
10 | - job_name: prometheus
11 | static_configs:
12 | - targets:
13 | - localhost:9090
14 | - job_name: kafka
15 | static_configs:
16 | - targets:
17 | - ''
18 |
--------------------------------------------------------------------------------
/client/index.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import { createRoot } from "react-dom/client";
3 | import ReactDOM from 'react-dom/client';
4 | import { BrowserRouter } from 'react-router-dom'
5 | import App from './components/app'
6 | import './style.css';
7 | import Loginpage from './components/pages/Loginpage';
8 |
9 | const root = ReactDOM.createRoot (document.getElementById('root'))
10 |
11 | root.render(
12 |
13 |
14 |
15 |
16 |
17 | );
--------------------------------------------------------------------------------
/client/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | BrewKafka
7 |
8 |
9 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
--------------------------------------------------------------------------------
/client/components/app.jsx:
--------------------------------------------------------------------------------
1 | import React, { useEffect, useState } from 'react';
2 | import { Routes, Route } from 'react-router-dom';
3 | import { NavLink } from 'react-router-dom';
4 | import { Brokers } from './pages/Brokers';
5 | import { Topics } from './pages/Topics';
6 | import { ProducersAndConsumers } from './pages/ProducersAndConsumers';
7 | import { Alerts } from './pages/Alerts';
8 | import Loginpage from './pages/Loginpage';
9 | import Dashboards from './Dashboards';
10 | import beanCarry from '../assets/beancarry.png';
11 |
12 | const App = () => {
13 | return (
14 |
15 |
16 | } />
17 | } />
18 | } />
19 | } />
20 | } />
21 | } />
22 |
23 |
24 | );
25 | };
26 |
27 | export default App;
28 |
--------------------------------------------------------------------------------
/webpack.config.js:
--------------------------------------------------------------------------------
1 | const path = require('path');
2 | const HTMLWebpackPlugin = require('html-webpack-plugin');
3 |
4 | module.exports = {
5 | entry: '/client/index.js',
6 | mode: process.env.NODE_ENV,
7 | output: {
8 | path: path.join(__dirname, '/dist'),
9 | filename: 'bundle.js',
10 | },
11 | module: {
12 | rules: [
13 | {
14 | test: /\.jsx?$/,
15 | exclude: /node_modules/,
16 | use: {
17 | loader: 'babel-loader',
18 | options: {
19 | presets: ['@babel/preset-env', '@babel/preset-react'],
20 | },
21 | },
22 | },
23 | {
24 | test: /\.s?css$/,
25 | exclude: /node_modules/,
26 | use: ['style-loader', 'css-loader'],
27 | },
28 | {
29 | test: /\.(png|gif)$/,
30 | type: 'asset/resource'
31 | },
32 | ],
33 | },
34 | plugins: [
35 | new HTMLWebpackPlugin({
36 | template: './client/index.html',
37 | }),
38 | ],
39 | devServer: {
40 | port: 8080,
41 | proxy: {
42 | '/': 'http://localhost:1234',
43 | },
44 | },
45 | resolve: {
46 | extensions: ['.js', '.jsx', '.scss'],
47 | },
48 | };
49 |
--------------------------------------------------------------------------------
/client/components/inputBar.jsx:
--------------------------------------------------------------------------------
1 | import React, { useState } from 'react';
2 | import KafkaService from '../services/KafkaService';
3 |
4 | const InputBar = () => {
5 | //handle change
6 | const [inputValue, setInputValue] = useState('');
7 |
8 | const navigate = useNavigate();
9 |
10 | const handleChange = e => {
11 | setInputValue(e.target.value);
12 | };
13 |
14 | // handle submit
15 | const handleSubmit = async () => {
16 | console.log('Submitted: ', inputValue);
17 | try {
18 | await KafkaService.addKafkaServer(inputValue);
19 | console.log('Updated server successfully');
20 | await KafkaService.reloadPrometheusConfig();
21 | console.log('Prometheus reloaded');
22 | } catch (error) {
23 | console.log('Error with Kafka config updating: ' + error);
24 | }
25 | navigate('/dashboards')
26 | };
27 | return (
28 |
29 |
33 |
34 |
35 | );
36 | };
37 |
38 | export default InputBar;
39 |
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '3'
2 | services:
3 | brewkafka:
4 | build: .
5 | volumes:
6 | - prometheus_config:/app/server/
7 | ports:
8 | - '8080:8080'
9 | depends_on:
10 | - grafana
11 |
12 | prometheus:
13 | image: prom/prometheus
14 | volumes:
15 | - prometheus_config:/etc/prometheus/
16 | command:
17 | - '--config.file=/etc/prometheus/prometheus.yml'
18 | - '--web.enable-lifecycle'
19 | ports:
20 | - '9090:9090'
21 | depends_on:
22 | - grafana
23 |
24 | grafana:
25 | image: grafana/grafana
26 | volumes:
27 | - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources/
28 | - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards
29 | - ./grafana/dashboards:/var/lib/grafana/dashboards
30 | environment:
31 | - GF_AUTH_ANONYMOUS_ENABLED=true
32 | - GF_AUTH_ANONYMOUS_ORG_NAME=Main Org.
33 | - GF_AUTH_ANONYMOUS_ORG_ROLE=Editor
34 | - GF_AUTH_ANONYMOUS_HIDE_VERSION=true
35 | - GF_SECURITY_ADMIN_PASSWORD=secret
36 | - GF_ALLOW_EMBEDDING=true
37 | - GF_SECURITY_ALLOW_EMBEDDING=true
38 | ports:
39 | - '3001:3000'
40 | volumes:
41 | prometheus_config:
42 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "brewkafka",
3 | "version": "1.0.0",
4 | "description": "",
5 | "main": "index.js",
6 | "scripts": {
7 | "start": "concurrently \"webpack-dev-server --mode development --open --hot\" \"nodemon ./server/server.js\" ",
8 | "build": "webpack --mode production",
9 | "test": "echo \"Error: no test specified\" && exit 1",
10 | "start2": "concurrently \"webpack-dev-server --mode development --open --hot\""
11 | },
12 | "keywords": [],
13 | "author": "",
14 | "license": "ISC",
15 | "dependencies": {
16 | "concurrently": "^8.2.2",
17 | "cors": "^2.8.5",
18 | "express": "^4.18.2",
19 | "fs": "^0.0.1-security",
20 | "js-yaml": "^4.1.0",
21 | "kafkajs": "^2.2.4",
22 | "path": "^0.12.7",
23 | "react": "^18.2.0",
24 | "react-dom": "^18.2.0",
25 | "react-router": "^6.18.0",
26 | "react-router-dom": "^6.18.0"
27 | },
28 | "devDependencies": {
29 | "@babel/core": "^7.23.2",
30 | "@babel/preset-env": "^7.23.2",
31 | "@babel/preset-react": "^7.22.15",
32 | "babel-loader": "^9.1.3",
33 | "css-loader": "^6.8.1",
34 | "html-webpack-plugin": "^5.5.3",
35 | "nodemon": "^3.0.1",
36 | "style-loader": "^3.3.3",
37 | "webpack": "^5.89.0",
38 | "webpack-cli": "^5.1.4",
39 | "webpack-dev-server": "^4.15.1"
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/client/services/KafkaService.js:
--------------------------------------------------------------------------------
1 | const KafkaService = {
2 | addKafkaServer: async serverUrl => {
3 | // check to make sure url is passed
4 |
5 | try {
6 | const response = await fetch('/config', {
7 | method: 'POST',
8 | headers: {
9 | 'Content-Type': 'application/json',
10 | },
11 | body: JSON.stringify({ serverUrl }),
12 | });
13 | console.log(serverUrl);
14 | return response.json();
15 | // await prometheusConfigWriter.writeConfig(serverUrl);
16 | } catch (error) {
17 | throw new Error('Error updating Kafka service server config: ' + error);
18 | }
19 | // return response.json();
20 | },
21 | reloadPrometheusConfig: async () => {
22 | // Replace with the correct URL if Prometheus is running elsewhere
23 | const prometheusReloadUrl = 'http://localhost:9090/-/reload'; //http reload for prometheus -- only POST or PUT works
24 | const response = await fetch(prometheusReloadUrl, { method: 'POST' });
25 |
26 | if (response.ok) {
27 | console.log('Prometheus configuration reload triggered successfully.');
28 | return;
29 | } else {
30 | const text = await response.text();
31 | throw new Error(`Failed to reload Prometheus configuration: ${text}`);
32 | }
33 | },
34 | };
35 |
36 | // const isValidUrl = urlString => {
37 | // new URL(urlString).catch(err => 'Error: ', err);
38 | // };
39 |
40 | module.exports = KafkaService;
41 |
--------------------------------------------------------------------------------
/client/components/pages/Brokers.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import Dashboards from '../Dashboards';
3 |
4 | export function Brokers() {
5 | return (
6 | <>
7 |
8 |
9 | {/*
14 |
19 | */}
24 | Active Broker count:
25 |
30 | Broker bytes in per sec:
31 |
36 |
37 | );
38 | }
39 |
--------------------------------------------------------------------------------
/client/components/pages/Topics.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import Dashboards from '../Dashboards';
3 |
4 | export function Topics() {
5 | return (
6 | <>
7 |
8 |
9 | {/*
15 |
21 | */}
27 | Topic count:
28 |
33 | Topic bytes per sec:
34 |
39 | Messages in per sec:
40 |
45 |
46 | );
47 | }
48 |
--------------------------------------------------------------------------------
/client/components/pages/ProducersAndConsumers.jsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import Dashboards from '../Dashboards';
3 |
4 | export function ProducersAndConsumers() {
5 | return (
6 | <>
7 |
8 |
9 | {/*
15 |
21 | */}
27 | Producer count:
28 |
33 | Producer request per sec:
34 |
39 | Number of Offsets:
40 |
45 |
46 | );
47 | }
48 |
--------------------------------------------------------------------------------
/client/components/Dashboards.jsx:
--------------------------------------------------------------------------------
1 | import React, { useEffect, useState } from 'react';
2 | import { NavLink } from 'react-router-dom';
3 |
4 | const Dashboards = () => {
5 | return (
6 |
7 |
8 |
53 |
54 |
55 |
56 | );
57 | }
58 |
59 | export default Dashboards;
--------------------------------------------------------------------------------
/server/controllers/kafkacontroller.js:
--------------------------------------------------------------------------------
1 | const { exec } = require('child_process');
2 | const prometheusConfigWriter = require('../services/prometheusConfigWriter');
3 |
4 | const kafkaController = {};
5 |
6 | //middleware functions
7 | kafkaController.connectButton = (req, res, next) => {
8 | //send user to display page
9 | console.log(req.body)
10 | return next();
11 | };
12 |
13 | kafkaController.displayBrokers = (req, res, next) => {
14 | //displays brokers on display page
15 | return next();
16 | };
17 |
18 | kafkaController.displayTopicsAndPartitions = (req, res, next) => {
19 | return next();
20 | };
21 |
22 | kafkaController.displayProducersAndConsumers = (req, res, next) => {
23 | return next();
24 | };
25 |
26 | // config writer for kafka server
27 | // server/controllers/kafkaController.js
28 |
29 | // ... other imports and code
30 |
31 | kafkaController.addKafkaServerConfig = async (req, res) => {
32 | try {
33 | console.log('writing server config for kafka');
34 | const { serverUrl } = req.body;
35 | await prometheusConfigWriter.writeConfig(serverUrl);
36 | console.log('Config file written, now reloading Prometheus');
37 |
38 | // Replace the Docker exec commands with a call to the reload endpoint
39 | // await KafkaService.reloadPrometheusConfig();
40 | console.log('Prometheus reloaded');
41 |
42 | res.status(200).json({
43 | message:
44 | 'Kafka server config successfully updated and Prometheus reloaded.',
45 | });
46 | } catch (error) {
47 | console.error('Error updating Kafka server config: ', error);
48 | res.status(500).json({ error: error.message });
49 | }
50 | };
51 |
52 | // ... rest of the kafkaController code
53 |
54 | kafkaController.displayAlerts = (req, res, next) => {};
55 |
56 | // kafkaController.addKafkaServerConfig = (req, res, next) => {
57 | // try {
58 | // prometheusConfigWriter.writeConfig(req.body.serverUrl);
59 | // res
60 | // .status(200)
61 | // .json({ message: 'Kafka server config updated successfully.' });
62 | // } catch (error) {
63 | // res.status(500).json({ error: error.message });
64 | // }
65 | // };
66 |
67 | module.exports = kafkaController;
68 |
--------------------------------------------------------------------------------
/server/services/prometheusConfigWriter.js:
--------------------------------------------------------------------------------
1 | const fs = require('fs');
2 | const path = require('path');
3 | const yaml = require('js-yaml');
4 |
5 | // const prometheusConfigFilePath = path.join(__dirname, '../prometheus.yml');
6 | console.log('DIRNAME:' + __dirname);
7 | console.log('---------------------------');
8 | const prometheusConfigFilePath = path.join(__dirname, '../prometheus.yml');
9 |
10 | const prometheusConfigWriter = {
11 | writeConfig: async serverUrl => {
12 | try {
13 | console.log('server url: ', serverUrl);
14 | const currentConfig = yaml.load(
15 | fs.readFileSync(prometheusConfigFilePath, 'utf8'),
16 | );
17 | if (currentConfig && Array.isArray(currentConfig.scrape_configs)) {
18 | currentConfig.scrape_configs.forEach(config => {
19 | if (
20 | config.job_name === 'kafka' &&
21 | Array.isArray(config.static_configs) &&
22 | config.static_configs.length > 0
23 | ) {
24 | config.static_configs[0].targets = [serverUrl];
25 | }
26 | });
27 | fs.writeFileSync(
28 | prometheusConfigFilePath,
29 | yaml.dump(currentConfig),
30 | 'utf8',
31 | );
32 | } else {
33 | throw new Error('Invalid Prometheus config format.');
34 | }
35 | } catch (error) {
36 | throw new Error('Error writing config: ' + error);
37 | }
38 |
39 | // const targetFile = path.join(targetsDirectory, 'kafka_targets.yml');
40 | // // load existing config
41 | // let currentConfig = [];
42 | // if (fs.existsSync(targetFile)) {
43 | // // currentConfig = JSON.parse(fs.readFileSync(targetFile));
44 | // currentConfig = yaml.load(fs.readFileSync(targetFile, 'utf8')) || [];
45 | // }
46 |
47 | // // add new target
48 | // currentConfig.push(targetConfig);
49 |
50 | // // Save updated config back to the file
51 | // // convert the js obj to yaml str
52 | // const yamlStr = yaml.dump(currentConfig);
53 | // fs.writeFileSync(targetFile, yamlStr, 'utf8');
54 | // // fs.writeFileSync(targetFile, JSON.stringify(currentConfig, null, 2));
55 | },
56 | };
57 |
58 | module.exports = prometheusConfigWriter;
59 |
--------------------------------------------------------------------------------
/client/components/pages/Loginpage.jsx:
--------------------------------------------------------------------------------
1 | import React, { useState, useEffect } from 'react';
2 | import beanCarryGif from '/client/assets/Untitled_Artwork.gif';
3 | import { useNavigate } from 'react-router-dom';
4 | import '/client/style.css';
5 | import KafkaService from '/client/services/KafkaService';
6 |
7 |
8 | const Loginpage = () => {
9 |
10 | const [inputValue, setInputValue] = useState('');
11 | const [isSubmitted, setIsSubmitted] = useState(false);
12 |
13 | const navigate = useNavigate();
14 |
15 | const handleChange = e => {
16 | setInputValue(e.target.value);
17 | };
18 |
19 | // handle submit
20 | const handleSubmit = async (e) => {
21 | e.preventDefault();
22 | console.log('Submitted: ', inputValue);
23 | try {
24 | console.log('KafkaService js file:', KafkaService)
25 | await KafkaService.addKafkaServer(inputValue);
26 | console.log('did this work please let it work');
27 |
28 | // await KafkaService.reloadPrometheusConfig();
29 | console.log('Prometheus reloaded');
30 | navigate('/dashboards')
31 | } catch (error) {
32 | console.log('Error with Kafka config updating: ' + error);
33 | }
34 | // setIsSubmitted(true)
35 | };
36 |
37 | // useEffect(() => {
38 | // console.log('enter useEffect', isSubmitted)
39 | // if (isSubmitted) {
40 | // navigate('/dashboards');
41 | // }
42 | // }, [isSubmitted]);
43 | // const navigate = useNavigate();
44 |
45 | // const handleSubmit = async () => {
46 | // console.log('Submitted: ', inputValue);
47 | // try {
48 | // await KafkaService.addKafkaServer(inputValue);
49 | // console.log('Updated server successfully');
50 | // await KafkaService.reloadPrometheusConfig();
51 | // console.log('Prometheus reloaded');
52 | // await navigate('/dashboards');
53 | // } catch (error) {
54 | // console.log('Error with Kafka config updating: ' + error);
55 | // }
56 | // }
57 |
58 | return (
59 | <>
60 | {/* */}
61 |
62 |
63 |
70 |
4 |
5 | # Setup
6 |
7 | ### This app assumes the user has:
8 | - a standalone kafka broker
9 | - their own Kafka Server
10 | - “jmx_prometheus_javaagent-0.19.0.jar” in “libs” folder
11 | - a specific "kafka_broker.yml" file in "config" folder
12 |
13 | ### To add your Kafka server:
14 | 1. Wait about 10s for grafana to load, then set your url, otherwise it may crash and stop
15 | - In the event of a crash, just go to docker and turn on grafana again via GUI
16 |
17 | 2. Track your metrics via opts
18 | - Use this line in your server-start.sh file, before ```exec`````` cmd
19 | ``` js
20 | export KAFKA_OPTS="$KAFKA_OPTS -javaagent:../libs/jmx_prometheus_javaagent-0.19.0.jar=7071:../config/kafka_broker.yml"
21 | ```
22 | This sets your Kafka JMX port to 7071 for prometheus to scrape metrics
23 |
24 | ### Ports to which we are exposing docker containers (Ports to keep open):
25 | - 8080 for BrewKafka
26 | - 9090 for Prometheus
27 | - 3001 for Grafana
28 |
29 | *If user is hosting their Kafka server locally, use* ```host.docker.internal:[port]```.
30 | ```Localhost:[port]``` *does not work*.
31 |
32 |
33 | # Developed using
34 |
35 | 
36 | 
37 | 
38 | 
39 | 
40 | 
41 | 
42 | 
43 | 
44 | 
45 | 
46 | 
47 |
48 | # Made by
49 |
50 | - Nick Dabreau [Github](https://github.com/nick-dab) | [LinkedIn](https://www.linkedIn.com/in/nickdabreau)
51 | - Robert Forrester [Github](https://github.com/Canarsie) | [LinkedIn](https://www.linkedin.com/in/brobertforrester)
52 | - Andrew Lo [Github](https://github.com/lomeintheory) | [LinkedIn](https://www.linkedin.com/in/andrew--lo/)
53 | - Helen Ta [Github](https://github.com/Helen-Ta) | [LinkedIn](https://www.linkedin.com/in/helen-ta)
54 | - Jordan Woods [Github](https://github.com/ejwoods) | [LinkedIn](https://www.linkedin.com/in/jordanwoods277/)
55 |
--------------------------------------------------------------------------------
/grafana/dashboards/messages in per sec topics-1701184634460.json:
--------------------------------------------------------------------------------
1 | {
2 | "annotations": {
3 | "list": [
4 | {
5 | "builtIn": 1,
6 | "datasource": {
7 | "type": "grafana",
8 | "uid": "-- Grafana --"
9 | },
10 | "enable": true,
11 | "hide": true,
12 | "iconColor": "rgba(0, 211, 255, 1)",
13 | "name": "Annotations & Alerts",
14 | "type": "dashboard"
15 | }
16 | ]
17 | },
18 | "editable": true,
19 | "fiscalYearStartMonth": 0,
20 | "graphTooltip": 0,
21 | "id": 10,
22 | "links": [],
23 | "liveNow": false,
24 | "panels": [
25 | {
26 | "datasource": {
27 | "type": "prometheus",
28 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
29 | },
30 | "fieldConfig": {
31 | "defaults": {
32 | "color": {
33 | "mode": "thresholds"
34 | },
35 | "mappings": [],
36 | "thresholds": {
37 | "mode": "absolute",
38 | "steps": [
39 | {
40 | "color": "green",
41 | "value": null
42 | },
43 | {
44 | "color": "red",
45 | "value": 80
46 | }
47 | ]
48 | }
49 | },
50 | "overrides": []
51 | },
52 | "gridPos": {
53 | "h": 8,
54 | "w": 12,
55 | "x": 0,
56 | "y": 0
57 | },
58 | "id": 1,
59 | "options": {
60 | "displayMode": "gradient",
61 | "minVizHeight": 10,
62 | "minVizWidth": 0,
63 | "orientation": "auto",
64 | "reduceOptions": {
65 | "calcs": [
66 | "lastNotNull"
67 | ],
68 | "fields": "",
69 | "values": false
70 | },
71 | "showUnfilled": true,
72 | "valueMode": "color"
73 | },
74 | "pluginVersion": "10.1.1",
75 | "targets": [
76 | {
77 | "datasource": {
78 | "type": "prometheus",
79 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
80 | },
81 | "disableTextWrap": false,
82 | "editorMode": "builder",
83 | "expr": "kafka_server_brokertopicmetrics_messagesinpersec",
84 | "fullMetaSearch": false,
85 | "includeNullMetadata": true,
86 | "instant": false,
87 | "legendFormat": "__auto",
88 | "range": true,
89 | "refId": "A",
90 | "useBackend": false
91 | }
92 | ],
93 | "title": "Panel Title",
94 | "type": "bargauge"
95 | }
96 | ],
97 | "refresh": "",
98 | "schemaVersion": 38,
99 | "style": "dark",
100 | "tags": [],
101 | "templating": {
102 | "list": []
103 | },
104 | "time": {
105 | "from": "now-6h",
106 | "to": "now"
107 | },
108 | "timepicker": {},
109 | "timezone": "",
110 | "title": "messages in per sec topics",
111 | "uid": "cf624c95-2c2e-465c-ada3-e3886f36ccbd",
112 | "version": 1,
113 | "weekStart": ""
114 | }
--------------------------------------------------------------------------------
/client/style.css:
--------------------------------------------------------------------------------
1 | body {
2 | background-color: rgb(153, 124, 95);
3 | }
4 |
5 | #dashboardHeader {
6 | display: flex;
7 | flex-direction: row;
8 | justify-content: left;
9 | }
10 |
11 | #wrapper {
12 | /* display: flex; */
13 | position: relative;
14 | }
15 |
16 | .container {
17 | /* border: 3px solid black; */
18 | background-color: pink;
19 | }
20 |
21 | #sidebar {
22 | border: 2px solid black;
23 | background-color: rgb(188, 153, 126);
24 | width: 15%;
25 | height: 90%;
26 | display: flex;
27 | flex-direction: column;
28 | position: fixed;
29 | left: 0;
30 | top: 0;
31 | bottom: 0;
32 | box-shadow: 0 6px 8px rgba(0, 0, 0, 0.4);
33 | overflow: auto;
34 | min-height: 300px;
35 | /* box-sizing: border-box; */
36 | /* min-width: 15%; */
37 | }
38 |
39 | /* #contentDisplay {
40 | float: center;
41 | width: 80%;
42 | background-color: rgb(188, 153, 126);
43 | box-shadow: 0 6px 8px rgba(0, 0, 0, 0.5);
44 | font-family: 'Hina Mincho', serif;
45 | } */
46 |
47 | #contentDisplay {
48 | float: right;
49 | width: 80%;
50 | background-color: rgb(153, 124, 95);
51 | font-family: 'Hina Mincho', serif;
52 | }
53 |
54 | .sbButton {
55 | width: auto;
56 | height: 15%;
57 | display: flex;
58 | flex-direction: column;
59 | align-items: center;
60 | font-family: 'Cabin', sans-serif;
61 | border-top: 2px solid black;
62 | border-bottom: 2px solid black;
63 | border-radius: 0px;
64 | text-align: center;
65 | margin-bottom: 1px;
66 | color: black;
67 | /* padding: 0px; */
68 | }
69 |
70 | #beancarry {
71 | /* padding-left: 500px; */
72 | display: flex;
73 | flex-direction: row;
74 | width: 500px;
75 | position: fixed;
76 | bottom: 0px;
77 | left: 40%;
78 | right: 40%;
79 | justify-content: center;
80 | /* margin-left: auto; */
81 | /* margin-right: auto; */
82 | }
83 |
84 | .beanCarryGif {
85 | display: flex;
86 | flex-direction: column;
87 | align-items: center;
88 | position: relative;
89 | }
90 |
91 | #bcgif {
92 | width: 80%;
93 | display: flex;
94 | position: absolute;
95 | }
96 |
97 | @media (max-width: 999px) {
98 | .sbButton,
99 | #sidebar,
100 | .container {
101 | width: auto;
102 | min-width: 100px; /* prevents the button from getting too small */
103 | padding: 10px; /* adjust as necessary */
104 | font-size: 0.8em; /* smaller font size for smaller screens */
105 | white-space: normal; /* for text wrapping */
106 | overflow: visible; /* makes sure text is not cut off */
107 | text-align: center; /* centers text in the button */
108 | }
109 | }
110 |
111 | /*Trying to have sidebar not overlap, but*/
112 | @media (max-width: 800px) {
113 | #sidebar,
114 | .sbButton,
115 | .container {
116 | width: auto;
117 | max-width: 99px; /* prevent overlapping on grafana panel */
118 | min-width: 10px; /* prevent overlapping on grafana panel */
119 | white-space: normal;
120 | overflow: visible;
121 | text-align: center;
122 | }
123 | }
124 | .formBox {
125 | display: flex;
126 | flex-direction: column;
127 | position: relative;
128 | margin-top: 10%;
129 | background-color: rgb(188, 153, 126);
130 | border: 1px solid rgb(136, 73, 73);
131 | border-radius: 10px;
132 | padding: 70px 20px 50px 20px;
133 | box-shadow: 0 6px 8px rgba(75, 41, 41, 0.5);
134 | font-family: 'Hina Mincho', serif;
135 | font-size: larger;
136 | }
137 |
138 | #inputBox {
139 | background-color: white;
140 | border: 1px solid rgb(136, 73, 73);
141 | border-radius: 5px;
142 | width: 200px;
143 | height: 20px;
144 | }
145 |
146 | .connectButton {
147 | background-color: white;
148 | border: 1px solid rgb(136, 73, 73);
149 | border-radius: 5px;
150 | height: 25px;
151 | }
152 |
153 |
154 | .inputbar2 {
155 | display: flex;
156 | justify-content: center;
157 | align-items: center;
158 | }
159 |
--------------------------------------------------------------------------------
/server/server.js:
--------------------------------------------------------------------------------
1 | const express = require('express');
2 | const bodyParser = require('body-parser');
3 | const cors = require('cors');
4 | const { Kafka } = require('kafkajs');
5 | const path = require('path');
6 | const fs = require('fs');
7 | const app = express();
8 | const PORT = 1234;
9 | const kafkaController = require('./controllers/kafkacontroller.js');
10 | // const kafkaRoutes = require('./routes/kafkaRoutes.js');
11 |
12 | app.use(cors());
13 | app.use(bodyParser.urlencoded({extended: true}));
14 | app.use(express.json());
15 |
16 | //post request on connect button
17 | app.post('/connect', kafkaController.connectButton, (req, res) => {
18 | console.log('in connect post request handler')
19 | return res.status(200);
20 | });
21 |
22 | // write server config for url of Kafka server
23 | // controller handles the response
24 | app.post('/config', kafkaController.addKafkaServerConfig);
25 | // app.post('/config', (req, res) => {
26 | // console.log(req.body);
27 | // return kafkaController.addKafkaServerConfig;
28 | // });
29 |
30 | app.use('/brokers', (req, res) => {
31 | return res.redirect('/brokers');
32 | })
33 |
34 | app.get('/topics-partitions', kafkaController.displayTopicsAndPartitions, (req, res) => {
35 | return res.status(200);
36 | })
37 |
38 | app.get('/producers-consumers', kafkaController.displayProducersAndConsumers, (req, res) => {
39 | return res.status(200);
40 | })
41 |
42 | app.get('/alerts', kafkaController.displayAlerts, (req, res) => {
43 | return res.status(200);
44 | })
45 |
46 | //new kafka instance
47 | // const kafka = new Kafka({
48 | // brokers: ['localhost:9092']
49 | // });
50 |
51 |
52 | // app.get('/getmessages', async (req, res) => {
53 | // //creating a producer
54 | // const producer = kafka.producer();
55 |
56 |
57 | // await producer.connect();
58 | // await producer.send({
59 | // topic: 'new-topic',
60 | // messages: [
61 | // { value: 'MESSSAGGEEEEEEEEE', partition: 0 },
62 | // ],
63 | // })
64 |
65 | // // creating a consumer
66 | // // const consumer = kafka.consumer({ groupId: 'new-consumer-group' });
67 | // const consumer = kafka.consumer({ groupId: 'test-consumer-group' });
68 |
69 | // await consumer.connect();
70 | // await consumer.subscribe({ topic: 'new-topic', fromBeginning: true });
71 |
72 | // const messages = [];
73 | // await consumer.run({
74 | // eachMessage: async ({ message }) => {
75 | // messages.push(message.value.toString());
76 | // console.log({
77 | // value: message.value.toString(),
78 | // })
79 | // console.log(messages);
80 | // }
81 | // })
82 | // res.status(200).json(messages);
83 | // // res.json(messages);
84 | // })
85 |
86 | // try {
87 | // const consumer = kafka.consumer({ groupId: 'test-consumer-group' });
88 |
89 | // await consumer.connect();
90 | // await consumer.subscribe({ topic: 'testing-topic', fromBeginning: true });
91 |
92 | // const messages = [];
93 | // await consumer.run({
94 | // eachMessage: async ({ message }) => {
95 | // messages.push(message.value.toString());
96 | // console.log({
97 | // value: message.value.toString(),
98 | // })
99 | // console.log(messages);
100 | // }
101 | // })
102 | // res.locals.messages = messages;
103 | // res.status(200).json(res.locals.messages);
104 | // }
105 | // catch (err) {
106 | // console.error(err, 'Error in kafka consumer function');
107 | // }
108 |
109 | //unknown route error
110 | app.use((req, res) => {
111 | return res.status(404).send('Page not found');
112 | });
113 |
114 | //global error handler
115 | app.use((err, req, res, next) => {
116 | const defaultErr = {
117 | log: 'Global error',
118 | status: 500,
119 | message: 'Error caught',
120 | };
121 | const errorObj = Object.assign(defaultErr, err);
122 | return res.status(errorObj.status).json(errorObj.message);
123 | });
124 |
125 | app.listen(PORT, () => console.log(`Listening on Port ${PORT}`));
126 |
127 | module.exports = app;
128 |
--------------------------------------------------------------------------------
/grafana/dashboards/KMinion Cluster Dashboard - Prometheus Exporter for Apache Kafka-1701184605403.json:
--------------------------------------------------------------------------------
1 | {
2 | "annotations": {
3 | "list": [
4 | {
5 | "builtIn": 1,
6 | "datasource": {
7 | "type": "datasource",
8 | "uid": "grafana"
9 | },
10 | "enable": true,
11 | "hide": true,
12 | "iconColor": "rgba(0, 211, 255, 1)",
13 | "name": "Annotations & Alerts",
14 | "type": "dashboard"
15 | }
16 | ]
17 | },
18 | "description": "Cluster wide metrics exported by KMinion",
19 | "editable": true,
20 | "fiscalYearStartMonth": 0,
21 | "gnetId": 14012,
22 | "graphTooltip": 0,
23 | "id": 8,
24 | "links": [],
25 | "liveNow": false,
26 | "panels": [
27 | {
28 | "aliasColors": {},
29 | "bars": false,
30 | "dashLength": 10,
31 | "dashes": false,
32 | "datasource": {
33 | "type": "prometheus",
34 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
35 | },
36 | "description": "Summed delta between High and Low watermark of all partitions in the given topic. Therefore not accurate for compacted topics that contain compacted records.",
37 | "fill": 1,
38 | "fillGradient": 0,
39 | "gridPos": {
40 | "h": 8,
41 | "w": 8,
42 | "x": 0,
43 | "y": 0
44 | },
45 | "hiddenSeries": false,
46 | "id": 16,
47 | "legend": {
48 | "avg": false,
49 | "current": false,
50 | "max": false,
51 | "min": false,
52 | "show": false,
53 | "total": false,
54 | "values": false
55 | },
56 | "lines": true,
57 | "linewidth": 2,
58 | "nullPointMode": "null",
59 | "options": {
60 | "alertThreshold": true
61 | },
62 | "percentage": false,
63 | "pluginVersion": "10.1.1",
64 | "pointradius": 2,
65 | "points": false,
66 | "renderer": "flot",
67 | "seriesOverrides": [],
68 | "spaceLength": 10,
69 | "stack": false,
70 | "steppedLine": false,
71 | "targets": [
72 | {
73 | "datasource": {
74 | "type": "prometheus",
75 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
76 | },
77 | "expr": "sum by (kubernetes_pod_name) (rate(kminion_kafka_topic_high_water_mark_sum[$__rate_interval]))",
78 | "interval": "",
79 | "legendFormat": "",
80 | "refId": "A"
81 | }
82 | ],
83 | "thresholds": [],
84 | "timeRegions": [],
85 | "title": "Messages in/sec",
86 | "tooltip": {
87 | "shared": true,
88 | "sort": 0,
89 | "value_type": "individual"
90 | },
91 | "type": "graph",
92 | "xaxis": {
93 | "mode": "time",
94 | "show": true,
95 | "values": []
96 | },
97 | "yaxes": [
98 | {
99 | "$$hashKey": "object:545",
100 | "decimals": 0,
101 | "format": "short",
102 | "logBase": 1,
103 | "min": "0",
104 | "show": true
105 | },
106 | {
107 | "$$hashKey": "object:546",
108 | "format": "short",
109 | "logBase": 1,
110 | "show": true
111 | }
112 | ],
113 | "yaxis": {
114 | "align": false
115 | }
116 | },
117 | {
118 | "datasource": {
119 | "type": "prometheus",
120 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
121 | },
122 | "fieldConfig": {
123 | "defaults": {
124 | "color": {
125 | "mode": "thresholds"
126 | },
127 | "mappings": [
128 | {
129 | "options": {
130 | "match": "null",
131 | "result": {
132 | "text": "N/A"
133 | }
134 | },
135 | "type": "special"
136 | }
137 | ],
138 | "thresholds": {
139 | "mode": "absolute",
140 | "steps": [
141 | {
142 | "color": "green",
143 | "value": null
144 | },
145 | {
146 | "color": "red",
147 | "value": 80
148 | }
149 | ]
150 | },
151 | "unit": "none"
152 | },
153 | "overrides": []
154 | },
155 | "gridPos": {
156 | "h": 4,
157 | "w": 4,
158 | "x": 8,
159 | "y": 0
160 | },
161 | "id": 11,
162 | "links": [],
163 | "maxDataPoints": 100,
164 | "options": {
165 | "colorMode": "none",
166 | "graphMode": "none",
167 | "justifyMode": "auto",
168 | "orientation": "horizontal",
169 | "reduceOptions": {
170 | "calcs": [
171 | "lastNotNull"
172 | ],
173 | "fields": "/^cluster_version$/",
174 | "values": false
175 | },
176 | "textMode": "auto"
177 | },
178 | "pluginVersion": "10.1.1",
179 | "targets": [
180 | {
181 | "datasource": {
182 | "type": "prometheus",
183 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
184 | },
185 | "expr": "kminion_kafka_cluster_info",
186 | "format": "table",
187 | "instant": true,
188 | "interval": "",
189 | "intervalFactor": 1,
190 | "legendFormat": "",
191 | "refId": "A"
192 | }
193 | ],
194 | "title": "Cluster Version",
195 | "type": "stat"
196 | },
197 | {
198 | "datasource": {
199 | "type": "prometheus",
200 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
201 | },
202 | "fieldConfig": {
203 | "defaults": {
204 | "custom": {
205 | "align": "auto",
206 | "cellOptions": {
207 | "type": "auto"
208 | },
209 | "filterable": false,
210 | "inspect": false
211 | },
212 | "mappings": [],
213 | "thresholds": {
214 | "mode": "absolute",
215 | "steps": [
216 | {
217 | "color": "green",
218 | "value": null
219 | },
220 | {
221 | "color": "red",
222 | "value": 80
223 | }
224 | ]
225 | }
226 | },
227 | "overrides": []
228 | },
229 | "gridPos": {
230 | "h": 8,
231 | "w": 12,
232 | "x": 12,
233 | "y": 0
234 | },
235 | "id": 10,
236 | "options": {
237 | "cellHeight": "sm",
238 | "footer": {
239 | "countRows": false,
240 | "fields": "",
241 | "reducer": [
242 | "sum"
243 | ],
244 | "show": false
245 | },
246 | "showHeader": true,
247 | "sortBy": [
248 | {
249 | "desc": false,
250 | "displayName": "Address"
251 | }
252 | ]
253 | },
254 | "pluginVersion": "10.1.1",
255 | "targets": [
256 | {
257 | "datasource": {
258 | "type": "prometheus",
259 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
260 | },
261 | "expr": "kminion_kafka_broker_info",
262 | "format": "table",
263 | "instant": true,
264 | "interval": "",
265 | "legendFormat": "",
266 | "refId": "A"
267 | }
268 | ],
269 | "title": "Brokers",
270 | "transformations": [
271 | {
272 | "id": "organize",
273 | "options": {
274 | "excludeByName": {
275 | "Time": true,
276 | "Value": true,
277 | "__cluster__": true,
278 | "__name__": true,
279 | "instance": true,
280 | "job": true,
281 | "kubernetes_namespace": true,
282 | "kubernetes_pod_name": true
283 | },
284 | "indexByName": {
285 | "Time": 0,
286 | "Value": 12,
287 | "__cluster__": 1,
288 | "__name__": 2,
289 | "address": 4,
290 | "broker_id": 3,
291 | "instance": 6,
292 | "is_controller": 8,
293 | "job": 9,
294 | "kubernetes_namespace": 10,
295 | "kubernetes_pod_name": 11,
296 | "port": 5,
297 | "rack_id": 7
298 | },
299 | "renameByName": {
300 | "address": "Address",
301 | "broker_id": "BrokerID",
302 | "instance": "",
303 | "is_controller": "Is Controller",
304 | "port": "Port",
305 | "rack_id": "RackID"
306 | }
307 | }
308 | }
309 | ],
310 | "type": "table"
311 | },
312 | {
313 | "datasource": {
314 | "type": "prometheus",
315 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
316 | },
317 | "fieldConfig": {
318 | "defaults": {
319 | "color": {
320 | "mode": "thresholds"
321 | },
322 | "mappings": [
323 | {
324 | "options": {
325 | "match": "null",
326 | "result": {
327 | "text": "N/A"
328 | }
329 | },
330 | "type": "special"
331 | }
332 | ],
333 | "thresholds": {
334 | "mode": "absolute",
335 | "steps": [
336 | {
337 | "color": "green",
338 | "value": null
339 | },
340 | {
341 | "color": "red",
342 | "value": 80
343 | }
344 | ]
345 | },
346 | "unit": "none"
347 | },
348 | "overrides": []
349 | },
350 | "gridPos": {
351 | "h": 4,
352 | "w": 4,
353 | "x": 8,
354 | "y": 4
355 | },
356 | "id": 7,
357 | "links": [],
358 | "maxDataPoints": 100,
359 | "options": {
360 | "colorMode": "none",
361 | "graphMode": "none",
362 | "justifyMode": "auto",
363 | "orientation": "horizontal",
364 | "reduceOptions": {
365 | "calcs": [
366 | "lastNotNull"
367 | ],
368 | "fields": "",
369 | "values": false
370 | },
371 | "textMode": "auto"
372 | },
373 | "pluginVersion": "10.1.1",
374 | "targets": [
375 | {
376 | "datasource": {
377 | "type": "prometheus",
378 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
379 | },
380 | "editorMode": "code",
381 | "expr": "kafka_controller_kafkacontroller_activebrokercount",
382 | "format": "table",
383 | "instant": true,
384 | "interval": "",
385 | "intervalFactor": 1,
386 | "legendFormat": "__auto",
387 | "refId": "A"
388 | }
389 | ],
390 | "title": "Brokers Online",
391 | "type": "stat"
392 | },
393 | {
394 | "aliasColors": {},
395 | "bars": false,
396 | "dashLength": 10,
397 | "dashes": false,
398 | "datasource": {
399 | "type": "prometheus",
400 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
401 | },
402 | "description": "Log dir size across all log dirs, topics and partitions for a given broker.",
403 | "fill": 1,
404 | "fillGradient": 0,
405 | "gridPos": {
406 | "h": 8,
407 | "w": 8,
408 | "x": 0,
409 | "y": 8
410 | },
411 | "hiddenSeries": false,
412 | "id": 14,
413 | "legend": {
414 | "avg": false,
415 | "current": false,
416 | "max": false,
417 | "min": false,
418 | "show": false,
419 | "total": false,
420 | "values": false
421 | },
422 | "lines": true,
423 | "linewidth": 2,
424 | "nullPointMode": "null",
425 | "options": {
426 | "alertThreshold": true
427 | },
428 | "percentage": false,
429 | "pluginVersion": "10.1.1",
430 | "pointradius": 2,
431 | "points": false,
432 | "renderer": "flot",
433 | "seriesOverrides": [],
434 | "spaceLength": 10,
435 | "stack": false,
436 | "steppedLine": false,
437 | "targets": [
438 | {
439 | "datasource": {
440 | "type": "prometheus",
441 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
442 | },
443 | "expr": "avg by (broker_id) (kminion_kafka_broker_log_dir_size_total_bytes)",
444 | "interval": "",
445 | "legendFormat": "Broker {{broker_id}}",
446 | "refId": "A"
447 | }
448 | ],
449 | "thresholds": [],
450 | "timeRegions": [],
451 | "title": "Log Dir Size by Broker",
452 | "tooltip": {
453 | "shared": true,
454 | "sort": 0,
455 | "value_type": "individual"
456 | },
457 | "type": "graph",
458 | "xaxis": {
459 | "mode": "time",
460 | "show": true,
461 | "values": []
462 | },
463 | "yaxes": [
464 | {
465 | "$$hashKey": "object:545",
466 | "decimals": 0,
467 | "format": "bytes",
468 | "logBase": 1,
469 | "min": "0",
470 | "show": true
471 | },
472 | {
473 | "$$hashKey": "object:546",
474 | "format": "short",
475 | "logBase": 1,
476 | "show": true
477 | }
478 | ],
479 | "yaxis": {
480 | "align": false
481 | }
482 | },
483 | {
484 | "aliasColors": {},
485 | "bars": false,
486 | "dashLength": 10,
487 | "dashes": false,
488 | "datasource": {
489 | "type": "prometheus",
490 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
491 | },
492 | "description": "",
493 | "fill": 1,
494 | "fillGradient": 0,
495 | "gridPos": {
496 | "h": 8,
497 | "w": 8,
498 | "x": 8,
499 | "y": 8
500 | },
501 | "hiddenSeries": false,
502 | "id": 2,
503 | "legend": {
504 | "alignAsTable": true,
505 | "avg": true,
506 | "current": false,
507 | "max": true,
508 | "min": false,
509 | "show": true,
510 | "sort": "avg",
511 | "sortDesc": true,
512 | "total": false,
513 | "values": true
514 | },
515 | "lines": true,
516 | "linewidth": 2,
517 | "nullPointMode": "null",
518 | "options": {
519 | "alertThreshold": true
520 | },
521 | "percentage": false,
522 | "pluginVersion": "10.1.1",
523 | "pointradius": 2,
524 | "points": false,
525 | "renderer": "flot",
526 | "seriesOverrides": [],
527 | "spaceLength": 10,
528 | "stack": false,
529 | "steppedLine": false,
530 | "targets": [
531 | {
532 | "datasource": {
533 | "type": "prometheus",
534 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
535 | },
536 | "expr": "avg by (topic_name) (rate(kminion_kafka_topic_high_water_mark_sum{topic_name=~\"$highThroughputTopics\"}[$__rate_interval]))",
537 | "interval": "",
538 | "legendFormat": "{{topic_name}}",
539 | "refId": "A"
540 | }
541 | ],
542 | "thresholds": [],
543 | "timeRegions": [],
544 | "title": "Messages in/sec by topic [Top 15]",
545 | "tooltip": {
546 | "shared": true,
547 | "sort": 2,
548 | "value_type": "individual"
549 | },
550 | "type": "graph",
551 | "xaxis": {
552 | "mode": "time",
553 | "show": true,
554 | "values": []
555 | },
556 | "yaxes": [
557 | {
558 | "$$hashKey": "object:39",
559 | "format": "short",
560 | "logBase": 1,
561 | "min": "0",
562 | "show": true
563 | },
564 | {
565 | "$$hashKey": "object:40",
566 | "format": "short",
567 | "logBase": 1,
568 | "show": true
569 | }
570 | ],
571 | "yaxis": {
572 | "align": false
573 | }
574 | },
575 | {
576 | "aliasColors": {},
577 | "bars": false,
578 | "dashLength": 10,
579 | "dashes": false,
580 | "datasource": {
581 | "type": "prometheus",
582 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
583 | },
584 | "description": "",
585 | "fill": 1,
586 | "fillGradient": 0,
587 | "gridPos": {
588 | "h": 8,
589 | "w": 8,
590 | "x": 16,
591 | "y": 8
592 | },
593 | "hiddenSeries": false,
594 | "id": 17,
595 | "legend": {
596 | "alignAsTable": true,
597 | "avg": true,
598 | "current": false,
599 | "max": true,
600 | "min": false,
601 | "show": true,
602 | "sort": "avg",
603 | "sortDesc": true,
604 | "total": false,
605 | "values": true
606 | },
607 | "lines": true,
608 | "linewidth": 2,
609 | "nullPointMode": "null",
610 | "options": {
611 | "alertThreshold": true
612 | },
613 | "percentage": false,
614 | "pluginVersion": "10.1.1",
615 | "pointradius": 2,
616 | "points": false,
617 | "renderer": "flot",
618 | "seriesOverrides": [],
619 | "spaceLength": 10,
620 | "stack": false,
621 | "steppedLine": false,
622 | "targets": [
623 | {
624 | "datasource": {
625 | "type": "prometheus",
626 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
627 | },
628 | "expr": "avg by (group_id) (rate(kminion_kafka_consumer_group_offset_commits_total{group_id=~\"$topOffsetCommitters\"}[$__rate_interval]))",
629 | "interval": "",
630 | "legendFormat": "{{group_id}}",
631 | "refId": "A"
632 | }
633 | ],
634 | "thresholds": [],
635 | "timeRegions": [],
636 | "title": "Top Offset Committers [Top 5]",
637 | "tooltip": {
638 | "shared": true,
639 | "sort": 2,
640 | "value_type": "individual"
641 | },
642 | "type": "graph",
643 | "xaxis": {
644 | "mode": "time",
645 | "show": true,
646 | "values": []
647 | },
648 | "yaxes": [
649 | {
650 | "$$hashKey": "object:39",
651 | "format": "short",
652 | "logBase": 1,
653 | "min": "0",
654 | "show": true
655 | },
656 | {
657 | "$$hashKey": "object:40",
658 | "format": "short",
659 | "logBase": 1,
660 | "show": true
661 | }
662 | ],
663 | "yaxis": {
664 | "align": false
665 | }
666 | },
667 | {
668 | "aliasColors": {},
669 | "bars": false,
670 | "dashLength": 10,
671 | "dashes": false,
672 | "datasource": {
673 | "type": "prometheus",
674 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
675 | },
676 | "fieldConfig": {
677 | "defaults": {
678 | "links": []
679 | },
680 | "overrides": []
681 | },
682 | "fill": 1,
683 | "fillGradient": 0,
684 | "gridPos": {
685 | "h": 7,
686 | "w": 24,
687 | "x": 0,
688 | "y": 16
689 | },
690 | "hiddenSeries": false,
691 | "id": 4,
692 | "legend": {
693 | "avg": false,
694 | "current": false,
695 | "max": false,
696 | "min": false,
697 | "show": true,
698 | "total": false,
699 | "values": false
700 | },
701 | "lines": true,
702 | "linewidth": 2,
703 | "links": [],
704 | "nullPointMode": "null",
705 | "options": {
706 | "alertThreshold": true
707 | },
708 | "percentage": false,
709 | "pluginVersion": "10.1.1",
710 | "pointradius": 2,
711 | "points": false,
712 | "renderer": "flot",
713 | "seriesOverrides": [],
714 | "spaceLength": 10,
715 | "stack": false,
716 | "steppedLine": false,
717 | "targets": [
718 | {
719 | "datasource": {
720 | "type": "prometheus",
721 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
722 | },
723 | "expr": "avg by (kubernetes_pod_name) (rate(kminion_exporter_offset_consumer_records_consumed_total[$__rate_interval]))",
724 | "format": "time_series",
725 | "interval": "",
726 | "intervalFactor": 1,
727 | "legendFormat": "{{kubernetes_pod_name}}",
728 | "refId": "A"
729 | }
730 | ],
731 | "thresholds": [],
732 | "timeRegions": [],
733 | "title": "Offset commits processed",
734 | "tooltip": {
735 | "shared": true,
736 | "sort": 0,
737 | "value_type": "individual"
738 | },
739 | "type": "graph",
740 | "xaxis": {
741 | "mode": "time",
742 | "show": true,
743 | "values": []
744 | },
745 | "yaxes": [
746 | {
747 | "$$hashKey": "object:163",
748 | "format": "short",
749 | "label": "Messages/s",
750 | "logBase": 1,
751 | "show": true
752 | },
753 | {
754 | "$$hashKey": "object:164",
755 | "format": "short",
756 | "logBase": 1,
757 | "show": true
758 | }
759 | ],
760 | "yaxis": {
761 | "align": false
762 | }
763 | }
764 | ],
765 | "refresh": "",
766 | "schemaVersion": 38,
767 | "style": "dark",
768 | "tags": [],
769 | "templating": {
770 | "list": [
771 | {
772 | "current": {},
773 | "datasource": {
774 | "type": "prometheus",
775 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
776 | },
777 | "definition": "query_result(topk(15, avg by (topic_name) (avg_over_time(kminion_kafka_topic_high_water_mark_sum[${__range_s}s]))))",
778 | "hide": 2,
779 | "includeAll": true,
780 | "multi": true,
781 | "name": "highThroughputTopics",
782 | "options": [],
783 | "query": "query_result(topk(15, avg by (topic_name) (avg_over_time(kminion_kafka_topic_high_water_mark_sum[${__range_s}s]))))",
784 | "refresh": 1,
785 | "regex": ".*topic_name=\"(.*?)\".*",
786 | "skipUrlSync": false,
787 | "sort": 0,
788 | "tagValuesQuery": "",
789 | "tagsQuery": "",
790 | "type": "query",
791 | "useTags": false
792 | },
793 | {
794 | "current": {},
795 | "datasource": {
796 | "type": "prometheus",
797 | "uid": "d67c7e38-dc9d-4ec6-a765-8973bac25d4a"
798 | },
799 | "definition": "query_result(topk(5, avg by (group_id) (avg_over_time(kminion_kafka_consumer_group_offset_commits_total[${__range_s}s]))))",
800 | "hide": 2,
801 | "includeAll": true,
802 | "multi": true,
803 | "name": "topOffsetCommitters",
804 | "options": [],
805 | "query": "query_result(topk(5, avg by (group_id) (avg_over_time(kminion_kafka_consumer_group_offset_commits_total[${__range_s}s]))))",
806 | "refresh": 1,
807 | "regex": ".*group_id=\"(.*?)\".*",
808 | "skipUrlSync": false,
809 | "sort": 0,
810 | "tagValuesQuery": "",
811 | "tagsQuery": "",
812 | "type": "query",
813 | "useTags": false
814 | }
815 | ]
816 | },
817 | "time": {
818 | "from": "now-6h",
819 | "to": "now"
820 | },
821 | "timepicker": {},
822 | "timezone": "",
823 | "title": "KMinion Cluster Dashboard - Prometheus Exporter for Apache Kafka",
824 | "uid": "Ax-9LJsGz",
825 | "version": 4,
826 | "weekStart": ""
827 | }
--------------------------------------------------------------------------------
/grafana/dashboards/nickdockerdashboards.json:
--------------------------------------------------------------------------------
1 | {
2 | "annotations": {
3 | "list": [
4 | {
5 | "builtIn": 1,
6 | "datasource": {
7 | "type": "grafana",
8 | "uid": "-- Grafana --"
9 | },
10 | "enable": true,
11 | "hide": true,
12 | "iconColor": "rgba(0, 211, 255, 1)",
13 | "name": "Annotations & Alerts",
14 | "type": "dashboard"
15 | }
16 | ]
17 | },
18 | "editable": true,
19 | "fiscalYearStartMonth": 0,
20 | "graphTooltip": 0,
21 | "links": [],
22 | "liveNow": false,
23 | "panels": [
24 | {
25 | "datasource": {
26 | "type": "prometheus",
27 | "uid": "PBFA97CFB590B2093"
28 | },
29 | "fieldConfig": {
30 | "defaults": {
31 | "color": {
32 | "mode": "palette-classic"
33 | },
34 | "custom": {
35 | "axisBorderShow": false,
36 | "axisCenteredZero": false,
37 | "axisColorMode": "text",
38 | "axisLabel": "",
39 | "axisPlacement": "auto",
40 | "barAlignment": 0,
41 | "drawStyle": "line",
42 | "fillOpacity": 0,
43 | "gradientMode": "none",
44 | "hideFrom": {
45 | "legend": false,
46 | "tooltip": false,
47 | "viz": false
48 | },
49 | "insertNulls": false,
50 | "lineInterpolation": "linear",
51 | "lineWidth": 1,
52 | "pointSize": 5,
53 | "scaleDistribution": {
54 | "type": "linear"
55 | },
56 | "showPoints": "auto",
57 | "spanNulls": false,
58 | "stacking": {
59 | "group": "A",
60 | "mode": "none"
61 | },
62 | "thresholdsStyle": {
63 | "mode": "off"
64 | }
65 | },
66 | "mappings": [],
67 | "thresholds": {
68 | "mode": "absolute",
69 | "steps": [
70 | {
71 | "color": "green",
72 | "value": null
73 | },
74 | {
75 | "color": "red",
76 | "value": 80
77 | }
78 | ]
79 | }
80 | },
81 | "overrides": []
82 | },
83 | "gridPos": {
84 | "h": 8,
85 | "w": 12,
86 | "x": 0,
87 | "y": 0
88 | },
89 | "id": 9,
90 | "options": {
91 | "legend": {
92 | "calcs": [],
93 | "displayMode": "list",
94 | "placement": "bottom",
95 | "showLegend": true
96 | },
97 | "tooltip": {
98 | "mode": "single",
99 | "sort": "none"
100 | }
101 | },
102 | "targets": [
103 | {
104 | "datasource": {
105 | "type": "prometheus",
106 | "uid": "PBFA97CFB590B2093"
107 | },
108 | "disableTextWrap": false,
109 | "editorMode": "builder",
110 | "expr": "kafka_server_brokertopicmetrics_bytesinpersec",
111 | "fullMetaSearch": false,
112 | "includeNullMetadata": true,
113 | "instant": false,
114 | "legendFormat": "__auto",
115 | "range": true,
116 | "refId": "A",
117 | "useBackend": false
118 | }
119 | ],
120 | "title": "Broker Bytes in per sec",
121 | "type": "timeseries"
122 | },
123 | {
124 | "datasource": {
125 | "type": "prometheus",
126 | "uid": "PBFA97CFB590B2093"
127 | },
128 | "fieldConfig": {
129 | "defaults": {
130 | "color": {
131 | "mode": "palette-classic"
132 | },
133 | "custom": {
134 | "axisBorderShow": false,
135 | "axisCenteredZero": false,
136 | "axisColorMode": "text",
137 | "axisLabel": "",
138 | "axisPlacement": "auto",
139 | "barAlignment": 0,
140 | "drawStyle": "line",
141 | "fillOpacity": 0,
142 | "gradientMode": "none",
143 | "hideFrom": {
144 | "legend": false,
145 | "tooltip": false,
146 | "viz": false
147 | },
148 | "insertNulls": false,
149 | "lineInterpolation": "linear",
150 | "lineWidth": 1,
151 | "pointSize": 5,
152 | "scaleDistribution": {
153 | "type": "linear"
154 | },
155 | "showPoints": "auto",
156 | "spanNulls": false,
157 | "stacking": {
158 | "group": "A",
159 | "mode": "none"
160 | },
161 | "thresholdsStyle": {
162 | "mode": "off"
163 | }
164 | },
165 | "mappings": [],
166 | "thresholds": {
167 | "mode": "absolute",
168 | "steps": [
169 | {
170 | "color": "green",
171 | "value": null
172 | },
173 | {
174 | "color": "red",
175 | "value": 80
176 | }
177 | ]
178 | }
179 | },
180 | "overrides": []
181 | },
182 | "gridPos": {
183 | "h": 8,
184 | "w": 12,
185 | "x": 0,
186 | "y": 8
187 | },
188 | "id": 8,
189 | "options": {
190 | "legend": {
191 | "calcs": [],
192 | "displayMode": "list",
193 | "placement": "bottom",
194 | "showLegend": true
195 | },
196 | "tooltip": {
197 | "mode": "single",
198 | "sort": "none"
199 | }
200 | },
201 | "targets": [
202 | {
203 | "datasource": {
204 | "type": "prometheus",
205 | "uid": "PBFA97CFB590B2093"
206 | },
207 | "disableTextWrap": false,
208 | "editorMode": "builder",
209 | "expr": "kafka_server_brokertopicmetrics_messagesinpersec",
210 | "fullMetaSearch": false,
211 | "includeNullMetadata": true,
212 | "instant": false,
213 | "legendFormat": "__auto",
214 | "range": true,
215 | "refId": "A",
216 | "useBackend": false
217 | }
218 | ],
219 | "title": "Topics - Messages in per sec",
220 | "type": "timeseries"
221 | },
222 | {
223 | "datasource": {
224 | "type": "prometheus",
225 | "uid": "PBFA97CFB590B2093"
226 | },
227 | "fieldConfig": {
228 | "defaults": {
229 | "color": {
230 | "mode": "palette-classic"
231 | },
232 | "custom": {
233 | "axisBorderShow": false,
234 | "axisCenteredZero": false,
235 | "axisColorMode": "text",
236 | "axisLabel": "",
237 | "axisPlacement": "auto",
238 | "barAlignment": 0,
239 | "drawStyle": "line",
240 | "fillOpacity": 0,
241 | "gradientMode": "none",
242 | "hideFrom": {
243 | "legend": false,
244 | "tooltip": false,
245 | "viz": false
246 | },
247 | "insertNulls": false,
248 | "lineInterpolation": "linear",
249 | "lineWidth": 1,
250 | "pointSize": 5,
251 | "scaleDistribution": {
252 | "type": "linear"
253 | },
254 | "showPoints": "auto",
255 | "spanNulls": false,
256 | "stacking": {
257 | "group": "A",
258 | "mode": "none"
259 | },
260 | "thresholdsStyle": {
261 | "mode": "off"
262 | }
263 | },
264 | "mappings": [],
265 | "thresholds": {
266 | "mode": "absolute",
267 | "steps": [
268 | {
269 | "color": "green",
270 | "value": null
271 | },
272 | {
273 | "color": "red",
274 | "value": 80
275 | }
276 | ]
277 | }
278 | },
279 | "overrides": []
280 | },
281 | "gridPos": {
282 | "h": 8,
283 | "w": 12,
284 | "x": 0,
285 | "y": 16
286 | },
287 | "id": 7,
288 | "options": {
289 | "legend": {
290 | "calcs": [],
291 | "displayMode": "list",
292 | "placement": "bottom",
293 | "showLegend": true
294 | },
295 | "tooltip": {
296 | "mode": "single",
297 | "sort": "none"
298 | }
299 | },
300 | "targets": [
301 | {
302 | "datasource": {
303 | "type": "prometheus",
304 | "uid": "PBFA97CFB590B2093"
305 | },
306 | "disableTextWrap": false,
307 | "editorMode": "builder",
308 | "expr": "kafka_server_brokertopicmetrics_bytesinpersec",
309 | "fullMetaSearch": false,
310 | "includeNullMetadata": true,
311 | "instant": false,
312 | "legendFormat": "__auto",
313 | "range": true,
314 | "refId": "A",
315 | "useBackend": false
316 | }
317 | ],
318 | "title": "Topic bytes in per sec",
319 | "type": "timeseries"
320 | },
321 | {
322 | "datasource": {
323 | "type": "prometheus",
324 | "uid": "PBFA97CFB590B2093"
325 | },
326 | "fieldConfig": {
327 | "defaults": {
328 | "color": {
329 | "mode": "thresholds"
330 | },
331 | "mappings": [],
332 | "thresholds": {
333 | "mode": "absolute",
334 | "steps": [
335 | {
336 | "color": "green",
337 | "value": null
338 | },
339 | {
340 | "color": "red",
341 | "value": 80
342 | }
343 | ]
344 | }
345 | },
346 | "overrides": []
347 | },
348 | "gridPos": {
349 | "h": 8,
350 | "w": 12,
351 | "x": 0,
352 | "y": 24
353 | },
354 | "id": 6,
355 | "options": {
356 | "colorMode": "value",
357 | "graphMode": "area",
358 | "justifyMode": "auto",
359 | "orientation": "auto",
360 | "reduceOptions": {
361 | "calcs": ["lastNotNull"],
362 | "fields": "",
363 | "values": false
364 | },
365 | "textMode": "auto",
366 | "wideLayout": true
367 | },
368 | "pluginVersion": "",
369 | "targets": [
370 | {
371 | "datasource": {
372 | "type": "prometheus",
373 | "uid": "PBFA97CFB590B2093"
374 | },
375 | "disableTextWrap": false,
376 | "editorMode": "builder",
377 | "expr": "kafka_controller_kafkacontroller_globaltopiccount",
378 | "fullMetaSearch": false,
379 | "includeNullMetadata": true,
380 | "instant": false,
381 | "legendFormat": "__auto",
382 | "range": true,
383 | "refId": "A",
384 | "useBackend": false
385 | }
386 | ],
387 | "title": "Topic count",
388 | "type": "stat"
389 | },
390 | {
391 | "datasource": {
392 | "type": "prometheus",
393 | "uid": "PBFA97CFB590B2093"
394 | },
395 | "fieldConfig": {
396 | "defaults": {
397 | "color": {
398 | "mode": "palette-classic"
399 | },
400 | "custom": {
401 | "axisBorderShow": false,
402 | "axisCenteredZero": false,
403 | "axisColorMode": "text",
404 | "axisLabel": "",
405 | "axisPlacement": "auto",
406 | "barAlignment": 0,
407 | "drawStyle": "line",
408 | "fillOpacity": 0,
409 | "gradientMode": "none",
410 | "hideFrom": {
411 | "legend": false,
412 | "tooltip": false,
413 | "viz": false
414 | },
415 | "insertNulls": false,
416 | "lineInterpolation": "linear",
417 | "lineWidth": 1,
418 | "pointSize": 5,
419 | "scaleDistribution": {
420 | "type": "linear"
421 | },
422 | "showPoints": "auto",
423 | "spanNulls": false,
424 | "stacking": {
425 | "group": "A",
426 | "mode": "none"
427 | },
428 | "thresholdsStyle": {
429 | "mode": "off"
430 | }
431 | },
432 | "mappings": [],
433 | "thresholds": {
434 | "mode": "absolute",
435 | "steps": [
436 | {
437 | "color": "green"
438 | },
439 | {
440 | "color": "red",
441 | "value": 80
442 | }
443 | ]
444 | }
445 | },
446 | "overrides": []
447 | },
448 | "gridPos": {
449 | "h": 8,
450 | "w": 12,
451 | "x": 0,
452 | "y": 32
453 | },
454 | "id": 5,
455 | "options": {
456 | "legend": {
457 | "calcs": [],
458 | "displayMode": "list",
459 | "placement": "bottom",
460 | "showLegend": true
461 | },
462 | "tooltip": {
463 | "mode": "single",
464 | "sort": "none"
465 | }
466 | },
467 | "targets": [
468 | {
469 | "datasource": {
470 | "type": "prometheus",
471 | "uid": "PBFA97CFB590B2093"
472 | },
473 | "disableTextWrap": false,
474 | "editorMode": "builder",
475 | "expr": "kafka_coordinator_group_groupmetadatamanager_numoffsets",
476 | "fullMetaSearch": false,
477 | "includeNullMetadata": true,
478 | "instant": false,
479 | "legendFormat": "__auto",
480 | "range": true,
481 | "refId": "A",
482 | "useBackend": false
483 | }
484 | ],
485 | "title": "Number of offsets",
486 | "type": "timeseries"
487 | },
488 | {
489 | "datasource": {
490 | "type": "prometheus",
491 | "uid": "PBFA97CFB590B2093"
492 | },
493 | "fieldConfig": {
494 | "defaults": {
495 | "color": {
496 | "mode": "palette-classic"
497 | },
498 | "custom": {
499 | "axisBorderShow": false,
500 | "axisCenteredZero": false,
501 | "axisColorMode": "text",
502 | "axisLabel": "",
503 | "axisPlacement": "auto",
504 | "barAlignment": 0,
505 | "drawStyle": "line",
506 | "fillOpacity": 0,
507 | "gradientMode": "none",
508 | "hideFrom": {
509 | "legend": false,
510 | "tooltip": false,
511 | "viz": false
512 | },
513 | "insertNulls": false,
514 | "lineInterpolation": "linear",
515 | "lineWidth": 1,
516 | "pointSize": 5,
517 | "scaleDistribution": {
518 | "type": "linear"
519 | },
520 | "showPoints": "auto",
521 | "spanNulls": false,
522 | "stacking": {
523 | "group": "A",
524 | "mode": "none"
525 | },
526 | "thresholdsStyle": {
527 | "mode": "off"
528 | }
529 | },
530 | "mappings": [],
531 | "thresholds": {
532 | "mode": "absolute",
533 | "steps": [
534 | {
535 | "color": "green"
536 | },
537 | {
538 | "color": "red",
539 | "value": 80
540 | }
541 | ]
542 | }
543 | },
544 | "overrides": []
545 | },
546 | "gridPos": {
547 | "h": 8,
548 | "w": 12,
549 | "x": 0,
550 | "y": 40
551 | },
552 | "id": 4,
553 | "options": {
554 | "legend": {
555 | "calcs": [],
556 | "displayMode": "list",
557 | "placement": "bottom",
558 | "showLegend": true
559 | },
560 | "tooltip": {
561 | "mode": "single",
562 | "sort": "none"
563 | }
564 | },
565 | "targets": [
566 | {
567 | "datasource": {
568 | "type": "prometheus",
569 | "uid": "PBFA97CFB590B2093"
570 | },
571 | "disableTextWrap": false,
572 | "editorMode": "builder",
573 | "expr": "kafka_server_brokertopicmetrics_failedproducerequestspersec",
574 | "fullMetaSearch": false,
575 | "includeNullMetadata": true,
576 | "instant": false,
577 | "legendFormat": "__auto",
578 | "range": true,
579 | "refId": "A",
580 | "useBackend": false
581 | }
582 | ],
583 | "title": "Producer Requests per sec",
584 | "type": "timeseries"
585 | },
586 | {
587 | "datasource": {
588 | "type": "prometheus",
589 | "uid": "PBFA97CFB590B2093"
590 | },
591 | "fieldConfig": {
592 | "defaults": {
593 | "color": {
594 | "mode": "thresholds"
595 | },
596 | "mappings": [],
597 | "thresholds": {
598 | "mode": "absolute",
599 | "steps": [
600 | {
601 | "color": "green"
602 | },
603 | {
604 | "color": "red",
605 | "value": 80
606 | }
607 | ]
608 | }
609 | },
610 | "overrides": []
611 | },
612 | "gridPos": {
613 | "h": 8,
614 | "w": 12,
615 | "x": 0,
616 | "y": 48
617 | },
618 | "id": 3,
619 | "options": {
620 | "colorMode": "value",
621 | "graphMode": "area",
622 | "justifyMode": "auto",
623 | "orientation": "auto",
624 | "reduceOptions": {
625 | "calcs": ["lastNotNull"],
626 | "fields": "",
627 | "values": false
628 | },
629 | "textMode": "auto",
630 | "wideLayout": true
631 | },
632 | "pluginVersion": "",
633 | "targets": [
634 | {
635 | "datasource": {
636 | "type": "prometheus",
637 | "uid": "PBFA97CFB590B2093"
638 | },
639 | "disableTextWrap": false,
640 | "editorMode": "builder",
641 | "expr": "kafka_server_replicamanager_produceridcount",
642 | "fullMetaSearch": false,
643 | "includeNullMetadata": true,
644 | "instant": false,
645 | "legendFormat": "__auto",
646 | "range": true,
647 | "refId": "A",
648 | "useBackend": false
649 | }
650 | ],
651 | "title": "Producer Count",
652 | "type": "stat"
653 | },
654 | {
655 | "datasource": {
656 | "type": "prometheus",
657 | "uid": "PBFA97CFB590B2093"
658 | },
659 | "fieldConfig": {
660 | "defaults": {
661 | "color": {
662 | "mode": "palette-classic"
663 | },
664 | "custom": {
665 | "axisBorderShow": false,
666 | "axisCenteredZero": false,
667 | "axisColorMode": "text",
668 | "axisLabel": "",
669 | "axisPlacement": "auto",
670 | "barAlignment": 0,
671 | "drawStyle": "line",
672 | "fillOpacity": 0,
673 | "gradientMode": "none",
674 | "hideFrom": {
675 | "legend": false,
676 | "tooltip": false,
677 | "viz": false
678 | },
679 | "insertNulls": false,
680 | "lineInterpolation": "linear",
681 | "lineWidth": 1,
682 | "pointSize": 5,
683 | "scaleDistribution": {
684 | "type": "linear"
685 | },
686 | "showPoints": "auto",
687 | "spanNulls": false,
688 | "stacking": {
689 | "group": "A",
690 | "mode": "none"
691 | },
692 | "thresholdsStyle": {
693 | "mode": "off"
694 | }
695 | },
696 | "mappings": [],
697 | "thresholds": {
698 | "mode": "absolute",
699 | "steps": [
700 | {
701 | "color": "green"
702 | },
703 | {
704 | "color": "red",
705 | "value": 80
706 | }
707 | ]
708 | }
709 | },
710 | "overrides": []
711 | },
712 | "gridPos": {
713 | "h": 8,
714 | "w": 12,
715 | "x": 0,
716 | "y": 56
717 | },
718 | "id": 2,
719 | "options": {
720 | "legend": {
721 | "calcs": [],
722 | "displayMode": "list",
723 | "placement": "bottom",
724 | "showLegend": true
725 | },
726 | "tooltip": {
727 | "mode": "single",
728 | "sort": "none"
729 | }
730 | },
731 | "targets": [
732 | {
733 | "datasource": {
734 | "type": "prometheus",
735 | "uid": "PBFA97CFB590B2093"
736 | },
737 | "disableTextWrap": false,
738 | "editorMode": "builder",
739 | "expr": "kafka_server_brokertopicmetrics_bytesinpersec",
740 | "fullMetaSearch": false,
741 | "includeNullMetadata": true,
742 | "instant": false,
743 | "legendFormat": "__auto",
744 | "range": true,
745 | "refId": "A",
746 | "useBackend": false
747 | }
748 | ],
749 | "title": "broker bytes in per second",
750 | "type": "timeseries"
751 | },
752 | {
753 | "datasource": {
754 | "type": "prometheus",
755 | "uid": "PBFA97CFB590B2093"
756 | },
757 | "fieldConfig": {
758 | "defaults": {
759 | "color": {
760 | "mode": "thresholds"
761 | },
762 | "mappings": [],
763 | "thresholds": {
764 | "mode": "absolute",
765 | "steps": [
766 | {
767 | "color": "green"
768 | },
769 | {
770 | "color": "red",
771 | "value": 80
772 | }
773 | ]
774 | }
775 | },
776 | "overrides": []
777 | },
778 | "gridPos": {
779 | "h": 8,
780 | "w": 12,
781 | "x": 0,
782 | "y": 64
783 | },
784 | "id": 1,
785 | "options": {
786 | "colorMode": "value",
787 | "graphMode": "area",
788 | "justifyMode": "auto",
789 | "orientation": "auto",
790 | "reduceOptions": {
791 | "calcs": ["lastNotNull"],
792 | "fields": "",
793 | "values": false
794 | },
795 | "textMode": "auto",
796 | "wideLayout": true
797 | },
798 | "pluginVersion": "",
799 | "targets": [
800 | {
801 | "datasource": {
802 | "type": "prometheus",
803 | "uid": "PBFA97CFB590B2093"
804 | },
805 | "disableTextWrap": false,
806 | "editorMode": "builder",
807 | "expr": "kafka_controller_kafkacontroller_activebrokercount",
808 | "fullMetaSearch": false,
809 | "includeNullMetadata": true,
810 | "instant": false,
811 | "legendFormat": "__auto",
812 | "range": true,
813 | "refId": "A",
814 | "useBackend": false
815 | }
816 | ],
817 | "title": "active broker count - docker ver",
818 | "type": "stat"
819 | }
820 | ],
821 | "refresh": "",
822 | "schemaVersion": 38,
823 | "tags": [],
824 | "templating": {
825 | "list": []
826 | },
827 | "time": {
828 | "from": "now-6h",
829 | "to": "now"
830 | },
831 | "timepicker": {},
832 | "timezone": "",
833 | "title": "docker dashboard",
834 | "uid": "c815566a-1a0f-4bea-a96b-df57bf18c720",
835 | "version": 1,
836 | "weekStart": ""
837 | }
--------------------------------------------------------------------------------
/grafana.ini:
--------------------------------------------------------------------------------
1 | ##################### Grafana Configuration Example #####################
2 | #
3 | # Everything has defaults so you only need to uncomment things you want to
4 | # change
5 |
6 | # possible values : production, development
7 | ;app_mode = production
8 |
9 | # instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty
10 | ;instance_name = ${HOSTNAME}
11 |
12 | # force migration will run migrations that might cause dataloss
13 | ;force_migration = false
14 |
15 | #################################### Paths ####################################
16 | [paths]
17 | # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)
18 | ;data = /var/lib/grafana
19 |
20 | # Temporary files in `data` directory older than given duration will be removed
21 | ;temp_data_lifetime = 24h
22 |
23 | # Directory where grafana can store logs
24 | ;logs = /var/log/grafana
25 |
26 | # Directory where grafana will automatically scan and look for plugins
27 | ;plugins = /var/lib/grafana/plugins
28 |
29 | # folder that contains provisioning config files that grafana will apply on startup and while running.
30 | ;provisioning = conf/provisioning
31 |
32 | #################################### Server ####################################
33 | [server]
34 | # Protocol (http, https, h2, socket)
35 | ;protocol = http
36 |
37 | # This is the minimum TLS version allowed. By default, this value is empty. Accepted values are: TLS1.2, TLS1.3. If nothing is set TLS1.2 would be taken
38 | ;min_tls_version = ""
39 |
40 | # The ip address to bind to, empty will bind to all interfaces
41 | ;http_addr =
42 |
43 | # The http port to use
44 | ;http_port = 3000
45 |
46 | # The public facing domain name used to access grafana from a browser
47 | ;domain = localhost
48 |
49 | # Redirect to correct domain if host header does not match domain
50 | # Prevents DNS rebinding attacks
51 | ;enforce_domain = false
52 |
53 | # The full public facing url you use in browser, used for redirects and emails
54 | # If you use reverse proxy and sub path specify full url (with sub path)
55 | ;root_url = %(protocol)s://%(domain)s:%(http_port)s/
56 |
57 | # Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons.
58 | ;serve_from_sub_path = false
59 |
60 | # Log web requests
61 | ;router_logging = false
62 |
63 | # the path relative working path
64 | ;static_root_path = public
65 |
66 | # enable gzip
67 | ;enable_gzip = false
68 |
69 | # https certs & key file
70 | ;cert_file =
71 | ;cert_key =
72 |
73 | # Unix socket gid
74 | # Changing the gid of a file without privileges requires that the target group is in the group of the process and that the process is the file owner
75 | # It is recommended to set the gid as http server user gid
76 | # Not set when the value is -1
77 | ;socket_gid =
78 |
79 | # Unix socket mode
80 | ;socket_mode =
81 |
82 | # Unix socket path
83 | ;socket =
84 |
85 | # CDN Url
86 | ;cdn_url =
87 |
88 | # Sets the maximum time using a duration format (5s/5m/5ms) before timing out read of an incoming request and closing idle connections.
89 | # `0` means there is no timeout for reading the request.
90 | ;read_timeout = 0
91 |
92 | # This setting enables you to specify additional headers that the server adds to HTTP(S) responses.
93 | [server.custom_response_headers]
94 | #exampleHeader1 = exampleValue1
95 | #exampleHeader2 = exampleValue2
96 |
97 | #################################### GRPC Server #########################
98 | ;[grpc_server]
99 | ;network = "tcp"
100 | ;address = "127.0.0.1:10000"
101 | ;use_tls = false
102 | ;cert_file =
103 | ;key_file =
104 |
105 | #################################### Database ####################################
106 | [database]
107 | # You can configure the database connection by specifying type, host, name, user and password
108 | # as separate properties or as on string using the url properties.
109 |
110 | # Either "mysql", "postgres" or "sqlite3", it's your choice
111 | ;type = sqlite3
112 | ;host = 127.0.0.1:3306
113 | ;name = grafana
114 | ;user = root
115 | # If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;"""
116 | ;password =
117 |
118 | # Use either URL or the previous fields to configure the database
119 | # Example: mysql://user:secret@host:port/database
120 | ;url =
121 |
122 | # For "postgres", use either "disable", "require" or "verify-full"
123 | # For "mysql", use either "true", "false", or "skip-verify".
124 | ;ssl_mode = disable
125 |
126 | # Database drivers may support different transaction isolation levels.
127 | # Currently, only "mysql" driver supports isolation levels.
128 | # If the value is empty - driver's default isolation level is applied.
129 | # For "mysql" use "READ-UNCOMMITTED", "READ-COMMITTED", "REPEATABLE-READ" or "SERIALIZABLE".
130 | ;isolation_level =
131 |
132 | ;ca_cert_path =
133 | ;client_key_path =
134 | ;client_cert_path =
135 | ;server_cert_name =
136 |
137 | # For "sqlite3" only, path relative to data_path setting
138 | ;path = grafana.db
139 |
140 | # Max idle conn setting default is 2
141 | ;max_idle_conn = 2
142 |
143 | # Max conn setting default is 0 (mean not set)
144 | ;max_open_conn =
145 |
146 | # Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours)
147 | ;conn_max_lifetime = 14400
148 |
149 | # Set to true to log the sql calls and execution times.
150 | ;log_queries =
151 |
152 | # For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared)
153 | ;cache_mode = private
154 |
155 | # For "sqlite3" only. Enable/disable Write-Ahead Logging, https://sqlite.org/wal.html. Default is false.
156 | ;wal = false
157 |
158 | # For "mysql" only if migrationLocking feature toggle is set. How many seconds to wait before failing to lock the database for the migrations, default is 0.
159 | ;locking_attempt_timeout_sec = 0
160 |
161 | # For "sqlite" only. How many times to retry query in case of database is locked failures. Default is 0 (disabled).
162 | ;query_retries = 0
163 |
164 | # For "sqlite" only. How many times to retry transaction in case of database is locked failures. Default is 5.
165 | ;transaction_retries = 5
166 |
167 | # Set to true to add metrics and tracing for database queries.
168 | ;instrument_queries = false
169 |
170 | ################################### Data sources #########################
171 | [datasources]
172 | # Upper limit of data sources that Grafana will return. This limit is a temporary configuration and it will be deprecated when pagination will be introduced on the list data sources API.
173 | ;datasource_limit = 5000
174 |
175 | #################################### Cache server #############################
176 | [remote_cache]
177 | # Either "redis", "memcached" or "database" default is "database"
178 | ;type = database
179 |
180 | # cache connectionstring options
181 | # database: will use Grafana primary database.
182 | # redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0,ssl=false`. Only addr is required. ssl may be 'true', 'false', or 'insecure'.
183 | # memcache: 127.0.0.1:11211
184 | ;connstr =
185 |
186 | # prefix prepended to all the keys in the remote cache
187 | ; prefix =
188 |
189 | # This enables encryption of values stored in the remote cache
190 | ;encryption =
191 |
192 | #################################### Data proxy ###########################
193 | [dataproxy]
194 |
195 | # This enables data proxy logging, default is false
196 | ;logging = false
197 |
198 | # How long the data proxy waits to read the headers of the response before timing out, default is 30 seconds.
199 | # This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set.
200 | ;timeout = 30
201 |
202 | # How long the data proxy waits to establish a TCP connection before timing out, default is 10 seconds.
203 | ;dialTimeout = 10
204 |
205 | # How many seconds the data proxy waits before sending a keepalive probe request.
206 | ;keep_alive_seconds = 30
207 |
208 | # How many seconds the data proxy waits for a successful TLS Handshake before timing out.
209 | ;tls_handshake_timeout_seconds = 10
210 |
211 | # How many seconds the data proxy will wait for a server's first response headers after
212 | # fully writing the request headers if the request has an "Expect: 100-continue"
213 | # header. A value of 0 will result in the body being sent immediately, without
214 | # waiting for the server to approve.
215 | ;expect_continue_timeout_seconds = 1
216 |
217 | # Optionally limits the total number of connections per host, including connections in the dialing,
218 | # active, and idle states. On limit violation, dials will block.
219 | # A value of zero (0) means no limit.
220 | ;max_conns_per_host = 0
221 |
222 | # The maximum number of idle connections that Grafana will keep alive.
223 | ;max_idle_connections = 100
224 |
225 | # How many seconds the data proxy keeps an idle connection open before timing out.
226 | ;idle_conn_timeout_seconds = 90
227 |
228 | # If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false.
229 | ;send_user_header = false
230 |
231 | # Limit the amount of bytes that will be read/accepted from responses of outgoing HTTP requests.
232 | ;response_limit = 0
233 |
234 | # Limits the number of rows that Grafana will process from SQL data sources.
235 | ;row_limit = 1000000
236 |
237 | # Sets a custom value for the `User-Agent` header for outgoing data proxy requests. If empty, the default value is `Grafana/` (for example `Grafana/9.0.0`).
238 | ;user_agent =
239 |
240 | #################################### Analytics ####################################
241 | [analytics]
242 | # Server reporting, sends usage counters to stats.grafana.org every 24 hours.
243 | # No ip addresses are being tracked, only simple counters to track
244 | # running instances, dashboard and error counts. It is very helpful to us.
245 | # Change this option to false to disable reporting.
246 | ;reporting_enabled = true
247 |
248 | # The name of the distributor of the Grafana instance. Ex hosted-grafana, grafana-labs
249 | ;reporting_distributor = grafana-labs
250 |
251 | # Set to false to disable all checks to https://grafana.com
252 | # for new versions of grafana. The check is used
253 | # in some UI views to notify that a grafana update exists.
254 | # This option does not cause any auto updates, nor send any information
255 | # only a GET request to https://raw.githubusercontent.com/grafana/grafana/main/latest.json to get the latest version.
256 | ;check_for_updates = true
257 |
258 | # Set to false to disable all checks to https://grafana.com
259 | # for new versions of plugins. The check is used
260 | # in some UI views to notify that a plugin update exists.
261 | # This option does not cause any auto updates, nor send any information
262 | # only a GET request to https://grafana.com to get the latest versions.
263 | ;check_for_plugin_updates = true
264 |
265 | # Google Analytics universal tracking code, only enabled if you specify an id here
266 | ;google_analytics_ua_id =
267 |
268 | # Google Analytics 4 tracking code, only enabled if you specify an id here
269 | ;google_analytics_4_id =
270 |
271 | # When Google Analytics 4 Enhanced event measurement is enabled, we will try to avoid sending duplicate events and let Google Analytics 4 detect navigation changes, etc.
272 | ;google_analytics_4_send_manual_page_views = false
273 |
274 | # Google Tag Manager ID, only enabled if you specify an id here
275 | ;google_tag_manager_id =
276 |
277 | # Rudderstack write key, enabled only if rudderstack_data_plane_url is also set
278 | ;rudderstack_write_key =
279 |
280 | # Rudderstack data plane url, enabled only if rudderstack_write_key is also set
281 | ;rudderstack_data_plane_url =
282 |
283 | # Rudderstack SDK url, optional, only valid if rudderstack_write_key and rudderstack_data_plane_url is also set
284 | ;rudderstack_sdk_url =
285 |
286 | # Rudderstack Config url, optional, used by Rudderstack SDK to fetch source config
287 | ;rudderstack_config_url =
288 |
289 | # Intercom secret, optional, used to hash user_id before passing to Intercom via Rudderstack
290 | ;intercom_secret =
291 |
292 | # Controls if the UI contains any links to user feedback forms
293 | ;feedback_links_enabled = true
294 |
295 | #################################### Security ####################################
296 | [security]
297 | # disable creation of admin user on first start of grafana
298 | ;disable_initial_admin_creation = false
299 |
300 | # default admin user, created on startup
301 | ;admin_user = admin
302 |
303 | # default admin password, can be changed before first start of grafana, or in profile settings
304 | ;admin_password = admin
305 |
306 | # default admin email, created on startup
307 | ;admin_email = admin@localhost
308 |
309 | # used for signing
310 | ;secret_key = SW2YcwTIb9zpOOhoPsMm
311 |
312 | # current key provider used for envelope encryption, default to static value specified by secret_key
313 | ;encryption_provider = secretKey.v1
314 |
315 | # list of configured key providers, space separated (Enterprise only): e.g., awskms.v1 azurekv.v1
316 | ;available_encryption_providers =
317 |
318 | # disable gravatar profile images
319 | ;disable_gravatar = false
320 |
321 | # data source proxy whitelist (ip_or_domain:port separated by spaces)
322 | ;data_source_proxy_whitelist =
323 |
324 | # disable protection against brute force login attempts
325 | ;disable_brute_force_login_protection = false
326 |
327 | # set to true if you host Grafana behind HTTPS. default is false.
328 | ;cookie_secure = false
329 |
330 | # set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict", "none" and "disabled"
331 | ;cookie_samesite = lax
332 |
333 | # set to true if you want to allow browsers to render Grafana in a ,