├── apps └── orders-consumer │ ├── .gitignore │ ├── project │ ├── build.properties │ └── plugins.sbt │ ├── build.sbt │ ├── .travis.yml │ ├── README.md │ └── src │ └── main │ └── scala │ └── com │ └── xebia │ └── orders │ └── OrderConsumer.scala ├── job └── kafka-load-generator.yaml ├── README.md ├── LICENSE ├── setup ├── grafana.yaml ├── zookeeper.yaml └── kafka.yaml ├── reporting-backends ├── influxdb.yaml └── prometheus.yaml ├── consumer ├── alpakka-consumer.yaml └── console-consumer.yaml └── dashboards ├── influxdb-consumer-metrics.json ├── prometheus-consumer-metrics.json └── influxdb-kafka-broker-metrics.json /apps/orders-consumer/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | .idea 3 | -------------------------------------------------------------------------------- /apps/orders-consumer/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.2.8 2 | -------------------------------------------------------------------------------- /apps/orders-consumer/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.20") 2 | -------------------------------------------------------------------------------- /apps/orders-consumer/build.sbt: -------------------------------------------------------------------------------- 1 | name := "order-consumer" 2 | organization := "com.xebia" 3 | scalaVersion := "2.12.1" 4 | 5 | libraryDependencies += "com.typesafe.akka" %% "akka-stream-kafka" % "1.0.3" 6 | 7 | enablePlugins(JavaAppPackaging) 8 | enablePlugins(DockerPlugin) 9 | dockerBaseImage := "openjdk:11-jre-slim" 10 | dockerRepository := Some("jeroenrosenberg") 11 | -------------------------------------------------------------------------------- /apps/orders-consumer/.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | language: scala 3 | cache: 4 | directories: 5 | - $HOME/.ivy2/cache 6 | - $HOME/.sbt/boot/ 7 | jdk: 8 | - oraclejdk8 9 | 10 | services: 11 | - docker 12 | 13 | script: 14 | - sbt test 15 | 16 | after_success: 17 | - if [ "$TRAVIS_BRANCH" == "master" ]; then 18 | docker login -u="$DOCKERHUB_USERNAME" -p="$DOCKERHUB_PASSWORD"; 19 | sbt "set dockerUpdateLatest := true" docker:publish; 20 | fi 21 | -------------------------------------------------------------------------------- /job/kafka-load-generator.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1beta1 2 | kind: CronJob 3 | metadata: 4 | name: kafka-load-generator 5 | spec: 6 | schedule: "*/1 * * * *" 7 | jobTemplate: 8 | spec: 9 | template: 10 | spec: 11 | containers: 12 | - name: kafka-load-generator 13 | image: solsson/kafkacat-curl 14 | args: [ 15 | '-c', 16 | 'COUNT=$((RANDOM * 200 / 32768)); curl -k -s "https://api.mockaroo.com/api/d5a195e0?key=4bf22500&count=$COUNT" | kafkacat -P -b kafka:9092 -t "$TOPIC" -p 0 -c "$COUNT"' 17 | ] 18 | command: ["/bin/bash"] 19 | env: 20 | - { name: TOPIC, value: "orders" } 21 | restartPolicy: OnFailure -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kafka-k8s-monitoring 2 | 3 | ## Initial setup 4 | `kubectl create ns playground` 5 | 6 | `kubectl -n playground apply -f config` 7 | `kubectl -n playground apply -f setup` 8 | 9 | ## Grafana 10 | 11 | `kubectl -n playground port-forward $(kubectl -n playground get po -lname=grafana -o jsonpath="{.items[0].metadata.name}") 3000:3000` 12 | 13 | Add datasource named "influxdb-kafka" 14 | 15 | ## Consuming sample data 16 | 17 | `kubectl -n playground apply -f job` 18 | 19 | ### Checking we are getting data 20 | `kubectl -n playground run --rm -it kafkacatjob --image=solsson/kafkacat --restart=OnFailure --command -- kafkacat -C -b kafka -t orders -o -10 -e` 21 | 22 | `kubectl -n playground apply -f consumer` 23 | 24 | ## Prometheus 25 | 26 | `kubectl -n playground port-forward $(kubectl -n playground get po -lname=prometheus -o jsonpath="{.items[0].metadata.name}") 9090:9090` -------------------------------------------------------------------------------- /apps/orders-consumer/README.md: -------------------------------------------------------------------------------- 1 | # orders-consumer 2 | 3 | # Building 4 | 5 | * Docker image, locally: `sbt docker:publishLocal` 6 | * Docker image: `sbt docker:publish` 7 | * Distribution package (bare): `sbt 'show stage'` 8 | * Distribution package (zip): `sbt 'show stage'` or `sbt 'show dist'` 9 | 10 | ## Configuration options 11 | 12 | * `input-topic` 13 | * `output-topic` 14 | * `akka.kafka.consumer.kafka-clients.group.id` 15 | * `akka.kafka.consumer.kafka-clients.bootstrap.servers` 16 | * `akka.kafka.producer.kafka-clients.bootstrap.servers` 17 | 18 | ## Running locally 19 | 20 | ``` 21 | $ ./target/universal/stage/bin/order-consumer \ 22 | -Dinput-topic=blah -Doutput-topic=bleh ... 23 | ``` 24 | 25 | ## Running in Docker 26 | 27 | ``` 28 | $ export JAVA_OPTS=" \ 29 | -Dinput-topic=blah \ 30 | -Doutput-topic=blah \ 31 | ... 32 | " 33 | $ docker run -e JAVA_OPTS -i order-consumer 34 | ``` 35 | 36 | ## Further detail 37 | 38 | Built using [akka-stream-kafka-template.g8](https://github.com/ScalaWilliam/akka-stream-kafka-template.g8) 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Jeroen Rosenberg 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /apps/orders-consumer/src/main/scala/com/xebia/orders/OrderConsumer.scala: -------------------------------------------------------------------------------- 1 | package com.xebia.orders 2 | 3 | import akka.Done 4 | import akka.actor.ActorSystem 5 | import akka.kafka.scaladsl._ 6 | import akka.kafka._ 7 | import akka.stream.{ActorMaterializer, Materializer} 8 | import akka.stream.scaladsl.Sink 9 | import org.apache.kafka.clients.consumer.ConsumerConfig 10 | import org.apache.kafka.common.serialization._ 11 | 12 | object OrderConsumer extends App { 13 | 14 | private implicit val actorSystem = ActorSystem("orders") 15 | implicit val mat: Materializer = ActorMaterializer() 16 | implicit val ec = actorSystem.dispatcher 17 | 18 | val settings = ConsumerSettings(actorSystem, new StringDeserializer, new ByteArrayDeserializer) 19 | .withBootstrapServers("kafka:9092") 20 | .withGroupId("my-group") 21 | .withProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") 22 | 23 | Consumer.plainSource(settings, Subscriptions.topics("orders")) 24 | .map(_.value()) 25 | .map(new String(_)) 26 | .map(println) 27 | .runWith(Sink.ignore).onComplete { 28 | _ => 29 | println("Stream is dead!") 30 | sys.exit(1) 31 | } 32 | 33 | 34 | } 35 | -------------------------------------------------------------------------------- /setup/grafana.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: grafana-secret 5 | type: Opaque 6 | data: 7 | admin-password: YWRtaW5hZG1pbg== 8 | --- 9 | kind: Deployment 10 | apiVersion: apps/v1 11 | metadata: 12 | name: grafana 13 | spec: 14 | template: 15 | metadata: 16 | labels: 17 | name: grafana 18 | spec: 19 | dnsPolicy: ClusterFirst 20 | restartPolicy: Always 21 | terminationGracePeriodSeconds: 5 22 | containers: 23 | - name: grafana 24 | image: grafana/grafana:6.1.3 25 | env: 26 | - { name: "GF_SECURITY_ADMIN_PASSWORD", valueFrom: { secretKeyRef: { name: "grafana-secret", key: "admin-password" }}} 27 | ports: 28 | - { containerPort: 3000, protocol: TCP } 29 | resources: 30 | requests: 31 | memory: "200Mi" 32 | limits: 33 | memory: "500Mi" 34 | livenessProbe: 35 | tcpSocket: 36 | port: 3000 37 | initialDelaySeconds: 30 38 | readinessProbe: 39 | tcpSocket: 40 | port: 3000 41 | initialDelaySeconds: 30 42 | volumeMounts: 43 | - { name: tmp, mountPath: "/tmp" } 44 | - { name: vartmp, mountPath: "/var/tmp" } 45 | - { name: grafana-data, mountPath: "/var/lib/grafana" } 46 | volumes: 47 | - { name: "tmp", emptyDir: {} } 48 | - { name: "vartmp", emptyDir: {} } 49 | - { name: "grafana-data", emptyDir: {} } 50 | replicas: 1 51 | selector: 52 | matchLabels: 53 | name: grafana 54 | -------------------------------------------------------------------------------- /setup/zookeeper.yaml: -------------------------------------------------------------------------------- 1 | 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | labels: 6 | app: zookeeper 7 | name: zookeeper 8 | spec: 9 | clusterIP: None 10 | ports: 11 | - { name: "zookeeper-client", port: 2181 } 12 | selector: 13 | app: zookeeper 14 | --- 15 | apiVersion: policy/v1beta1 16 | kind: PodDisruptionBudget 17 | metadata: 18 | name: zookeeper 19 | spec: 20 | minAvailable: 2 21 | selector: 22 | matchLabels: 23 | app: zookeeper 24 | apiVersion: apps/v1 25 | kind: StatefulSet 26 | metadata: 27 | name: zookeeper 28 | labels: 29 | app: zookeeper 30 | spec: 31 | serviceName: zookeeper 32 | replicas: 3 33 | podManagementPolicy: Parallel 34 | updateStrategy: 35 | type: RollingUpdate 36 | selector: 37 | matchLabels: 38 | app: zookeeper 39 | template: 40 | metadata: 41 | labels: 42 | app: zookeeper 43 | spec: 44 | terminationGracePeriodSeconds: 60 45 | containers: 46 | - name: zookeeper 47 | image: zookeeper:3.5 48 | env: 49 | - { name: ZOOKEEPER_COUNT, value: "3" } 50 | - { name: HOST_DNS_PATTERN, value: "zookeeper" } 51 | - { name: SERVER_JVMFLAGS, value: "-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1" } 52 | - { name: ZK_ROOT_LOGGER_LEVEL, value: "INFO, CONSOLE" } 53 | - { name: ZK_CONSOLE_THRESHOLD, value: "INFO" } 54 | resources: 55 | requests: { memory: "1Gi" } 56 | limits: { memory: "1Gi" } 57 | ports: 58 | - { containerPort: 2181, protocol: TCP } 59 | - { containerPort: 2888, protocol: TCP } 60 | - { containerPort: 3888, protocol: TCP } 61 | readinessProbe: 62 | tcpSocket: 63 | port: 2181 64 | volumeMounts: 65 | - {name: data, mountPath: /var/lib/zookeeper } 66 | - {name: tmp, mountPath: /tmp } 67 | - {name: vartmp, mountPath: /var/tmp } 68 | volumes: 69 | - { name: tmp, emptyDir: {} } 70 | - { name: vartmp, emptyDir: {} } 71 | - { name: data, emptyDir: {} } 72 | -------------------------------------------------------------------------------- /reporting-backends/influxdb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: influxdb 5 | labels: 6 | name: influxdb 7 | spec: 8 | ports: 9 | - { name: "influxdb", port: 80, targetPort: 8086 } 10 | selector: 11 | name: influxdb 12 | sessionAffinity: None 13 | type: ClusterIP 14 | --- 15 | apiVersion: v1 16 | kind: Secret 17 | metadata: 18 | name: influxdb-secret 19 | type: Opaque 20 | data: 21 | user-password: MWYyZDFlMmU2N2Rm 22 | admin-password: Vabeguto0D7NJO23 23 | --- 24 | apiVersion: apps/v1 25 | kind: Deployment 26 | metadata: 27 | name: influxdb 28 | spec: 29 | template: 30 | metadata: 31 | labels: 32 | name: influxdb 33 | spec: 34 | dnsPolicy: ClusterFirst 35 | restartPolicy: Always 36 | terminationGracePeriodSeconds: 5 37 | containers: 38 | - name: influxdb 39 | image: influxdb:1.7-alpine 40 | env: 41 | - { name: "INFLUXDB_USER", value: "jeroen" } 42 | - { name: "INFLUXDB_USER_PASSWORD", valueFrom: { secretKeyRef: { name: "influxdb-secret", key: "user-password" } } } 43 | - { name: "INFLUXDB_DB" , value: "kafka" } 44 | - { name: "INFLUXDB_ADMIN_USER", value: "admin" } 45 | - { name: "INFLUXDB_ADMIN_PASSWORD", valueFrom: { secretKeyRef: { name: "influxdb-secret", key: "admin-password" } } } 46 | - { name: "INFLUXDB_META_DIR", value: "/influxdbdata/meta" } 47 | - { name: "INFLUXDB_DATA_DIR", value: "/influxdbdata/data" } 48 | - { name: "INFLUXDB_DATA_WAL_DIR", value: "/influxdbdata/wal" } 49 | - { name: "INFLUXDB_REPORTING_DISABLED", value: "true" } 50 | ports: 51 | - { containerPort: 8086, protocol: TCP } 52 | resources: 53 | requests: { memory: "1Gi" } 54 | limits: { memory: "1Gi" } 55 | readinessProbe: 56 | tcpSocket: 57 | port: 8086 58 | volumeMounts: 59 | - { name: tmp, mountPath: "/tmp" } 60 | - { name: vartmp, mountPath: "/var/tmp" } 61 | - { name: influxdb-data, mountPath: "/influxdbdata" } 62 | volumes: 63 | - { name: "tmp", emptyDir: {} } 64 | - { name: "vartmp", emptyDir: {} } 65 | - { name: "influxdb-data", emptyDir: {} } 66 | replicas: 1 67 | selector: 68 | matchLabels: 69 | name: influxdb 70 | -------------------------------------------------------------------------------- /consumer/alpakka-consumer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: orders-consumer-jmxexporter-config 5 | data: 6 | config.yml: |+ 7 | --- 8 | hostPort: localhost:9999 9 | --- 10 | apiVersion: v1 11 | kind: ConfigMap 12 | metadata: 13 | name: orders-consumer-jmxtrans-config 14 | data: 15 | metrics.json: |+ 16 | {"obj": "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*", "attr": ["records-lag-max","fetch-size-avg","fetch-size-max", "bytes-consumed-rate", "records-per-request-avg", "records-consumed-rate", "fetch-rate", "fetch-latency-avg", "fetch-latency-max", "fetch-throttle-time-avg", "fetch-throttle-time-max"], "resultAlias": "mmConsumerFetchMetrics"} 17 | {"obj": "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*,topic=*", "attr": ["fetch-size-avg","fetch-size-max", "bytes-consumed-rate", "records-per-request-avg", "records-consumed-rate"], "resultAlias": "mmConsumerPerTopicFetchMetrics"} 18 | {"obj": "kafka.consumer:type=consumer-coordinator-metrics,client-id=*", "attr": ["assigned-partitions","commit-latency-avg", "commit-latency-max", "commit-rate", "join-rate", "join-time-avg", "join-time-max","sync-rate","sync-time-avg","sync-time-max","heartbeat-rate","heartbeat-response-time-max","last-heartbeat-seconds-ago"], "resultAlias": "mmConsumerGroupMetrics"} 19 | {"obj": "kafka.consumer:type=consumer-metrics,client-id=*", "attr": ["request-rate", "response-rate", "incoming-byte-rate", "outgoing-byte-rate"], "resultAlias": "mmConsumerRequestMetrics"} 20 | {"obj": "kafka.consumer:type=consumer-metrics,client-id=*", "attr": ["connection-count","connection-creation-rate","connection-close-rate", "io-ratio", "io-time-ns-avg","io-wait-ratio","select-rate","io-wait-time-ns-avg"], "resultAlias": "mmConsumerConnectionMetrics"} 21 | {"obj": "kafka.consumer:type=consumer-node-metrics,client-id=*,node-id=*", "attr": ["request-size-max","request-size-avg","request-rate", "response-rate", "incoming-byte-rate", "outgoing-byte-rate"], "resultAlias": "mmConsumerPerBrokerMetrics"} 22 | boot.sh: |+ 23 | #!/bin/sh 24 | 25 | set -e 26 | set -x 27 | 28 | export JMXTRANS_OPTS="$JMXTRANS_OPTS -Dlogback.configurationFile=file:///${JMXTRANS_HOME}/conf/logback.xml -DkafkaJmxPort=$KAFKA_JMX_PORT -Dalias=$POD_NAME -DinfluxUrl=$INFLUXDB_HOST -DinfluxDatabase=$INFLUXDB_DATABASE -DinfluxUser=$INFLUXDB_USER -DinfluxPass=$INFLUXDB_PASSWORD" 29 | exec java $JAVA_OPTS $JMXTRANS_OPTS -jar $JAR_FILE -e -j $JSON_DIR -s $SECONDS_BETWEEN_RUNS -c $CONTINUE_ON_ERROR 30 | --- 31 | apiVersion: apps/v1 32 | kind: Deployment 33 | metadata: 34 | name: orders-consumer 35 | spec: 36 | template: 37 | metadata: 38 | labels: 39 | name: orders-consumer 40 | annotations: 41 | prometheus.io/scrape: 'true' 42 | spec: 43 | dnsPolicy: ClusterFirst 44 | restartPolicy: Always 45 | terminationGracePeriodSeconds: 5 46 | initContainers: 47 | - name: jmxtrans-init 48 | image: realguess/jq:1.4 49 | volumeMounts: 50 | - name: orders-consumer-jmxtrans-config 51 | mountPath: /metrics-input/metrics.json 52 | subPath: metrics.json 53 | - name: jmxtrans-input 54 | mountPath: /jmxtrans-input 55 | command: 56 | - sh 57 | - -c 58 | - >- 59 | jq -s -c '{"servers": [{"port": "${kafkaJmxPort}","host":"localhost","alias":"${alias}", "queries":map( {"outputWriters": [{"@class":"com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory","url":"${influxUrl}","database":"${influxDatabase}","createDatabase":false,"username":"${influxUser}","password":"${influxPass}"}], "obj": .obj, "attr": .attr, "resultAlias": .resultAlias})}]}' 60 | /metrics-input/metrics.json > /jmxtrans-input/kafka.json 61 | containers: 62 | - name: orders-consumer 63 | image: jeroenrosenberg/order-consumer:0.1.0-SNAPSHOT 64 | imagePullPolicy: Always 65 | resources: 66 | requests: { memory: "300Mi" } 67 | limits: { memory: "300Mi" } 68 | env: 69 | - { name: "JAVA_OPTS", value: "-Dcom.sun.management.jmxremote.port=9999 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" } 70 | - name: "jmx-exporter" 71 | image: sscaling/jmx-prometheus-exporter 72 | ports: 73 | - { containerPort: 5556, name: metrics, protocol: TCP } 74 | env: 75 | - { name: JVM_OPTS, value: "-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1" } 76 | volumeMounts: 77 | - name: orders-consumer-jmxexporter-config 78 | mountPath: /opt/jmx_exporter/config.yml 79 | subPath: config.yml 80 | - name: "jmxtrans" 81 | image: "jmxtrans/jmxtrans" 82 | command: ["/boot.sh"] 83 | env: 84 | - { name: "POD_NAME", valueFrom: { fieldRef: { fieldPath: "metadata.name" }}} 85 | - { name: "HOST_NAME", valueFrom: { fieldRef: { fieldPath: "spec.nodeName" }}} 86 | - { name: "KAFKA_JMX_PORT", value: "9999" } 87 | - { name: "INFLUXDB_DATABASE", value: "kafka" } 88 | - { name: "INFLUXDB_HOST", value: "http://influxdb" } 89 | - { name: "INFLUXDB_USER", value: "jeroen" } 90 | - { name: "INFLUXDB_PASSWORD", valueFrom: { secretKeyRef: { name: influxdb-secret, key: user-password }}} 91 | volumeMounts: 92 | - name: jmxtrans-input 93 | mountPath: /var/lib/jmxtrans 94 | readOnly: true 95 | - name: orders-consumer-jmxtrans-config 96 | mountPath: /boot.sh 97 | subPath: boot.sh 98 | readOnly: true 99 | volumes: 100 | - { name: orders-consumer-jmxexporter-config, configMap: { name: orders-consumer-jmxexporter-config, defaultMode: 0744 }} 101 | - { name: jmxtrans-input, emptyDir: {} } 102 | - { name: orders-consumer-jmxtrans-config, configMap: { name: orders-consumer-jmxtrans-config, defaultMode: 0744 } } 103 | replicas: 1 104 | selector: 105 | matchLabels: 106 | name: orders-consumer -------------------------------------------------------------------------------- /reporting-backends/prometheus.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | labels: 5 | app: prometheus 6 | name: prometheus 7 | data: 8 | prometheus.yml: | 9 | global: 10 | evaluation_interval: 1m 11 | scrape_interval: 1m 12 | scrape_timeout: 10s 13 | rule_files: 14 | - /etc/config/rules 15 | - /etc/config/alerts 16 | scrape_configs: 17 | - job_name: prometheus 18 | static_configs: 19 | - targets: 20 | - localhost:9090 21 | - job_name: kubernetes-services 22 | kubernetes_sd_configs: 23 | - role: service 24 | bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token 25 | api_server: https://kubernetes.default.svc 26 | tls_config: 27 | ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt 28 | namespaces: 29 | names: 30 | - playground 31 | metrics_path: /probe 32 | params: 33 | module: 34 | - http_2xx 35 | relabel_configs: 36 | - action: keep 37 | regex: true 38 | source_labels: 39 | - __meta_kubernetes_service_annotation_prometheus_io_probe 40 | - source_labels: 41 | - __address__ 42 | target_label: __param_target 43 | - replacement: blackbox 44 | target_label: __address__ 45 | - source_labels: 46 | - __param_target 47 | target_label: instance 48 | - action: labelmap 49 | regex: __meta_kubernetes_service_label_(.+) 50 | - source_labels: 51 | - __meta_kubernetes_namespace 52 | target_label: kubernetes_namespace 53 | - source_labels: 54 | - __meta_kubernetes_service_name 55 | target_label: kubernetes_name 56 | - job_name: kubernetes-pods 57 | kubernetes_sd_configs: 58 | - role: pod 59 | bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token 60 | api_server: https://kubernetes.default.svc 61 | tls_config: 62 | ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt 63 | namespaces: 64 | names: 65 | - playground 66 | relabel_configs: 67 | - action: keep 68 | regex: true 69 | source_labels: 70 | - __meta_kubernetes_pod_annotation_prometheus_io_scrape 71 | - source_labels: 72 | - __meta_kubernetes_pod_container_port_name 73 | action: keep 74 | regex: metrics 75 | - action: replace 76 | regex: (.+) 77 | source_labels: 78 | - __meta_kubernetes_pod_annotation_prometheus_io_path 79 | target_label: __metrics_path__ 80 | - action: replace 81 | regex: ([^:]+)(?::\d+)?;(\d+) 82 | replacement: $1:$2 83 | source_labels: 84 | - __address__ 85 | - __meta_kubernetes_pod_annotation_prometheus_io_port 86 | target_label: __address__ 87 | - action: labelmap 88 | regex: __meta_kubernetes_pod_label_(.+) 89 | - action: replace 90 | source_labels: 91 | - __meta_kubernetes_namespace 92 | target_label: kubernetes_namespace 93 | - action: replace 94 | source_labels: 95 | - __meta_kubernetes_pod_name 96 | target_label: kubernetes_pod_name 97 | --- 98 | kind: ServiceAccount 99 | apiVersion: v1 100 | metadata: 101 | name: prometheus 102 | namespace: playground 103 | labels: 104 | app: prometheus 105 | kubernetes.io/cluster-service: "true" 106 | --- 107 | apiVersion: rbac.authorization.k8s.io/v1beta1 108 | kind: ClusterRole 109 | metadata: 110 | name: prometheus 111 | labels: 112 | kubernetes.io/cluster-service: "true" 113 | rules: 114 | - apiGroups: 115 | - "" 116 | resources: 117 | - nodes 118 | - nodes/metrics 119 | - services 120 | - endpoints 121 | - pods 122 | verbs: 123 | - get 124 | - list 125 | - watch 126 | - apiGroups: 127 | - "" 128 | resources: 129 | - configmaps 130 | verbs: 131 | - get 132 | - nonResourceURLs: 133 | - "/metrics" 134 | verbs: 135 | - get 136 | --- 137 | apiVersion: rbac.authorization.k8s.io/v1beta1 138 | kind: ClusterRoleBinding 139 | metadata: 140 | name: prometheus 141 | labels: 142 | kubernetes.io/cluster-service: "true" 143 | roleRef: 144 | apiGroup: rbac.authorization.k8s.io 145 | kind: ClusterRole 146 | name: prometheus 147 | subjects: 148 | - kind: ServiceAccount 149 | name: prometheus 150 | namespace: playground 151 | --- 152 | kind: Deployment 153 | apiVersion: apps/v1 154 | metadata: 155 | name: prometheus 156 | spec: 157 | template: 158 | metadata: 159 | labels: 160 | name: prometheus 161 | annotations: 162 | prometheus.io/scrape: 'true' 163 | prometheus.io/port: '9913' 164 | spec: 165 | dnsPolicy: ClusterFirst 166 | restartPolicy: Always 167 | terminationGracePeriodSeconds: 5 168 | serviceAccountName: prometheus 169 | containers: 170 | - name: prometheus 171 | image: prom/prometheus:v2.9.0 172 | imagePullPolicy: "IfNotPresent" 173 | args: 174 | - --config.file=/etc/config/prometheus.yml 175 | - --storage.tsdb.path=/prometheus-data 176 | - --web.console.libraries=/etc/prometheus/console_libraries 177 | - --web.console.templates=/etc/prometheus/consoles 178 | - --web.enable-lifecycle 179 | ports: 180 | - containerPort: 9090 181 | protocol: TCP 182 | resources: 183 | requests: 184 | memory: "256Mi" 185 | limits: 186 | memory: "1Gi" 187 | readinessProbe: 188 | httpGet: 189 | path: /-/ready 190 | port: 9090 191 | initialDelaySeconds: 30 192 | timeoutSeconds: 30 193 | livenessProbe: 194 | httpGet: 195 | path: /-/healthy 196 | port: 9090 197 | initialDelaySeconds: 30 198 | timeoutSeconds: 30 199 | volumeMounts: 200 | - name: config-volume 201 | mountPath: /etc/config 202 | - name: prometheus-data 203 | mountPath: /prometheus-data 204 | volumes: 205 | - name: config-volume 206 | configMap: 207 | name: prometheus 208 | - name: prometheus-data 209 | emptyDir: {} 210 | replicas: 1 211 | selector: 212 | matchLabels: 213 | name: prometheus 214 | --- 215 | apiVersion: v1 216 | kind: Service 217 | metadata: 218 | name: prometheus 219 | labels: 220 | name: prometheus 221 | spec: 222 | ports: 223 | - name: "prometheus" 224 | protocol: TCP 225 | port: 9090 226 | targetPort: 9090 227 | selector: 228 | name: prometheus 229 | sessionAffinity: None 230 | type: ClusterIP 231 | -------------------------------------------------------------------------------- /consumer/console-consumer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: console-consumer-jmxexporter-config 5 | data: 6 | config.yml: |+ 7 | --- 8 | hostPort: localhost:9010 9 | --- 10 | apiVersion: v1 11 | kind: ConfigMap 12 | metadata: 13 | name: consumer-jmxtrans-config 14 | data: 15 | metrics.json: |+ 16 | {"obj": "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*", "attr": ["records-lag-max","fetch-size-avg","fetch-size-max", "bytes-consumed-rate", "records-per-request-avg", "records-consumed-rate", "fetch-rate", "fetch-latency-avg", "fetch-latency-max", "fetch-throttle-time-avg", "fetch-throttle-time-max"], "resultAlias": "mmConsumerFetchMetrics"} 17 | {"obj": "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*,topic=*", "attr": ["fetch-size-avg","fetch-size-max", "bytes-consumed-rate", "records-per-request-avg", "records-consumed-rate"], "resultAlias": "mmConsumerPerTopicFetchMetrics"} 18 | {"obj": "kafka.consumer:type=consumer-coordinator-metrics,client-id=*", "attr": ["assigned-partitions","commit-latency-avg", "commit-latency-max", "commit-rate", "join-rate", "join-time-avg", "join-time-max","sync-rate","sync-time-avg","sync-time-max","heartbeat-rate","heartbeat-response-time-max","last-heartbeat-seconds-ago"], "resultAlias": "mmConsumerGroupMetrics"} 19 | {"obj": "kafka.consumer:type=consumer-metrics,client-id=*", "attr": ["request-rate", "response-rate", "incoming-byte-rate", "outgoing-byte-rate"], "resultAlias": "mmConsumerRequestMetrics"} 20 | {"obj": "kafka.consumer:type=consumer-metrics,client-id=*", "attr": ["connection-count","connection-creation-rate","connection-close-rate", "io-ratio", "io-time-ns-avg","io-wait-ratio","select-rate","io-wait-time-ns-avg"], "resultAlias": "mmConsumerConnectionMetrics"} 21 | {"obj": "kafka.consumer:type=consumer-node-metrics,client-id=*,node-id=*", "attr": ["request-size-max","request-size-avg","request-rate", "response-rate", "incoming-byte-rate", "outgoing-byte-rate"], "resultAlias": "mmConsumerPerBrokerMetrics"} 22 | boot.sh: |+ 23 | #!/bin/sh 24 | 25 | set -e 26 | set -x 27 | 28 | export JMXTRANS_OPTS="$JMXTRANS_OPTS -Dlogback.configurationFile=file:///${JMXTRANS_HOME}/conf/logback.xml -DkafkaJmxPort=$KAFKA_JMX_PORT -Dalias=$POD_NAME -DinfluxUrl=$INFLUXDB_HOST -DinfluxDatabase=$INFLUXDB_DATABASE -DinfluxUser=$INFLUXDB_USER -DinfluxPass=$INFLUXDB_PASSWORD" 29 | exec java $JAVA_OPTS $JMXTRANS_OPTS -jar $JAR_FILE -e -j $JSON_DIR -s $SECONDS_BETWEEN_RUNS -c $CONTINUE_ON_ERROR 30 | --- 31 | apiVersion: apps/v1 32 | kind: Deployment 33 | metadata: 34 | name: kafka-consumer 35 | labels: 36 | app: kafka-consumer 37 | name: kafka-consumer 38 | spec: 39 | replicas: 1 40 | selector: 41 | matchLabels: 42 | app: kafka-consumer 43 | template: 44 | metadata: 45 | labels: 46 | app: kafka-consumer 47 | name: kafka-consumer 48 | annotations: 49 | prometheus.io/scrape: 'true' 50 | spec: 51 | initContainers: 52 | - name: jmxtrans-init 53 | image: realguess/jq:1.4 54 | volumeMounts: 55 | - name: consumer-jmxtrans-config 56 | mountPath: /metrics-input/metrics.json 57 | subPath: metrics.json 58 | - name: jmxtrans-input 59 | mountPath: /jmxtrans-input 60 | command: 61 | - sh 62 | - -c 63 | - >- 64 | jq -s -c '{"servers": [{"port": "${kafkaJmxPort}","host":"localhost","alias":"${alias}", "queries":map( {"outputWriters": [{"@class":"com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory","url":"${influxUrl}","database":"${influxDatabase}","createDatabase":false,"username":"${influxUser}","password":"${influxPass}"}], "obj": .obj, "attr": .attr, "resultAlias": .resultAlias})}]}' 65 | /metrics-input/metrics.json > /jmxtrans-input/kafka.json 66 | containers: 67 | - name: kafka-consumer 68 | image: wurstmeister/kafka:2.12-2.2.0 69 | command: ["/opt/kafka/bin/kafka-console-consumer.sh", "--topic", "orders", "--bootstrap-server", "http://kafka:9092"] 70 | env: 71 | - { name: KAFKA_JMX_OPTS, value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=127.0.0.1 -Dcom.sun.management.jmxremote.rmi.port=9010" } 72 | - { name: JMX_PORT, value: "9010" } 73 | resources: 74 | requests: { memory: "100Mi" } 75 | limits: { memory: "200Mi" } 76 | ports: 77 | - { name: jmx-port, containerPort: 9010, protocol: TCP } 78 | volumeMounts: 79 | - {name: tmp, mountPath: /tmp } 80 | - {name: vartmp, mountPath: /var/tmp } 81 | - name: "jmxtrans" 82 | image: "jmxtrans/jmxtrans" 83 | command: ["/boot.sh"] 84 | env: 85 | - { name: "POD_NAME", valueFrom: { fieldRef: { fieldPath: "metadata.name" }}} 86 | - { name: "HOST_NAME", valueFrom: { fieldRef: { fieldPath: "spec.nodeName" }}} 87 | - { name: "KAFKA_JMX_PORT", value: "9010" } 88 | - { name: "INFLUXDB_DATABASE", value: "kafka" } 89 | - { name: "INFLUXDB_HOST", value: "http://influxdb" } 90 | - { name: "INFLUXDB_USER", value: "jeroen" } 91 | - { name: "INFLUXDB_PASSWORD", valueFrom: { secretKeyRef: { name: influxdb-secret, key: user-password }}} 92 | volumeMounts: 93 | - name: jmxtrans-input 94 | mountPath: /var/lib/jmxtrans 95 | readOnly: true 96 | - name: consumer-jmxtrans-config 97 | mountPath: /boot.sh 98 | subPath: boot.sh 99 | readOnly: true 100 | - name: "jmx-exporter" 101 | image: sscaling/jmx-prometheus-exporter 102 | ports: 103 | - { containerPort: 5556, name: metrics, protocol: TCP } 104 | env: 105 | - { name: JVM_OPTS, value: "-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1" } 106 | volumeMounts: 107 | - name: console-consumer-jmxexporter-config 108 | mountPath: /opt/jmx_exporter/config.yml 109 | subPath: config.yml 110 | volumes: 111 | - { name: tmp, emptyDir: {} } 112 | - { name: vartmp, emptyDir: {} } 113 | - { name: jmxtrans-input, emptyDir: {} } 114 | - { name: console-consumer-jmxexporter-config, configMap: { name: console-consumer-jmxexporter-config, defaultMode: 0744 } } 115 | - { name: consumer-jmxtrans-config, configMap: { name: consumer-jmxtrans-config, defaultMode: 0744 } } -------------------------------------------------------------------------------- /setup/kafka.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: kafka-jmxtrans-config 5 | data: 6 | metrics.json: |+ 7 | {"obj": "java.lang:type=Memory", "attr": ["HeapMemoryUsage","NonHeapMemoryUsage"], "resultAlias": "jvmMemory"} 8 | {"obj": "kafka.server:type=ReplicaManager,name=UnderReplicatedPartitions", "attr": ["Value"], "resultAlias": "UnderReplicatedPartitions"} 9 | {"obj": "kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec", "attr": ["Count","MeanRate","OneMinuteRate","FiveMinuteRate","FifteenMinuteRate"], "resultAlias": "MessagesInPerSec"} 10 | {"obj": "kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec,topic=*", "attr": ["Count","MeanRate","OneMinuteRate","FiveMinuteRate","FifteenMinuteRate"], "resultAlias": "MessagesInPerSecPerTopic"} 11 | {"obj": "kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec", "attr": ["Count","MeanRate","OneMinuteRate","FiveMinuteRate","FifteenMinuteRate"], "resultAlias": "BytesInPerSec"} 12 | {"obj": "kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec,topic=*", "attr": ["Count","MeanRate","OneMinuteRate","FiveMinuteRate","FifteenMinuteRate"], "resultAlias": "BytesInPerSecPerTopic"} 13 | {"obj": "kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec", "attr": ["Count","MeanRate","OneMinuteRate","FiveMinuteRate","FifteenMinuteRate"], "resultAlias": "BytesOutPerSec"} 14 | {"obj": "kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec,topic=*", "attr": ["Count","MeanRate","OneMinuteRate","FiveMinuteRate","FifteenMinuteRate"], "resultAlias": "BytesOutPerSecPerTopic"} 15 | {"obj": "kafka.server:type=ReplicaManager,name=PartitionCount", "attr": ["Value"], "resultAlias": "PartitionCount"} 16 | {"obj": "kafka.server:type=ReplicaManager,name=LeaderCount", "attr": ["Value"], "resultAlias": "LeaderCount"} 17 | {"obj": "kafka.server:type=ReplicaManager,name=IsrShrinksPerSec", "attr": ["Count","MeanRate","OneMinuteRate","FiveMinuteRate","FifteenMinuteRate"], "resultAlias": "IsrShrinksPerSec"} 18 | {"obj": "kafka.server:type=ReplicaManager,name=IsrExpandsPerSec", "attr": ["Count","MeanRate","OneMinuteRate","FiveMinuteRate","FifteenMinuteRate"], "resultAlias": "IsrExpandsPerSec"} 19 | {"obj": "kafka.server:type=KafkaRequestHandlerPool,name=RequestHandlerAvgIdlePercent", "attr": ["MeanRate"], "resultAlias": "RequestHandlerAvgIdlePercent"} 20 | {"obj": "kafka.controller:type=KafkaController,name=OfflinePartitionsCount", "attr": ["Value"], "resultAlias": "OfflinePartitionsCount"} 21 | {"obj": "kafka.controller:type=KafkaController,name=ActiveControllerCount", "attr": ["Value"], "resultAlias": "ActiveControllerCount"} 22 | {"obj": "kafka.controller:type=ControllerStats,name=LeaderElectionRateAndTimeMs", "attr": ["Count","MeanRate","OneMinuteRate","FiveMinuteRate","FifteenMinuteRate"], "resultAlias": "LeaderElectionRateAndTimeMs"} 23 | {"obj": "kafka.controller:type=ControllerStats,name=UncleanLeaderElectionsPerSec", "attr": ["Count","MeanRate","OneMinuteRate","FiveMinuteRate","FifteenMinuteRate"], "resultAlias": "UncleanLeaderElectionsPerSec"} 24 | {"obj": "kafka.network:type=RequestMetrics,name=RequestsPerSec,request=Produce", "attr": ["Count","MeanRate","OneMinuteRate","FiveMinuteRate","FifteenMinuteRate"], "resultAlias": "ProduceRequestsPerSec"} 25 | {"obj": "kafka.network:type=RequestMetrics,name=RequestsPerSec,request=FetchConsumer", "attr": ["Count","MeanRate","OneMinuteRate","FiveMinuteRate","FifteenMinuteRate"], "resultAlias": "FetchConsumerRequestsPerSec"} 26 | {"obj": "kafka.network:type=RequestMetrics,name=RequestsPerSec,request=FetchFollower", "attr": ["Count","MeanRate","OneMinuteRate","FiveMinuteRate","FifteenMinuteRate"], "resultAlias": "FetchFollower"} 27 | {"obj": "kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Produce", "attr": ["Count","Min","Max","Mean","StdDev", "50thPercentile","75thPercentile","95thPercentile","98thPercentile","99thPercentile","999thPercentile"], "resultAlias": "ProduceTotalTimeMs"} 28 | {"obj": "kafka.network:type=RequestMetrics,name=TotalTimeMs,request=FetchConsumer", "attr": ["Count","Min","Max","Mean","StdDev", "50thPercentile","75thPercentile","95thPercentile","98thPercentile","99thPercentile","999thPercentile"], "resultAlias": "FetchConsumerTotalTimeMs"} 29 | {"obj": "kafka.network:type=RequestMetrics,name=TotalTimeMs,request=FetchFollower", "attr": ["Count","Min","Max","Mean","StdDev", "50thPercentile","75thPercentile","95thPercentile","98thPercentile","99thPercentile","999thPercentile"], "resultAlias": "FetchFollowerTotalTimeMs"} 30 | {"obj": "kafka.network:type=RequestMetrics,name=RequestQueueTimeMs,request=FetchFollower", "attr": ["Count","Min","Max","Mean","StdDev", "50thPercentile","75thPercentile","95thPercentile","98thPercentile","99thPercentile","999thPercentile"], "resultAlias": "FetchFollowerRequestQueueTimeMs"} 31 | {"obj": "kafka.network:type=RequestMetrics,name=RequestQueueTimeMs,request=FetchConsumer", "attr": ["Count","Min","Max","Mean","StdDev", "50thPercentile","75thPercentile","95thPercentile","98thPercentile","99thPercentile","999thPercentile"], "resultAlias": "FetchConsumerRequestQueueTimeMs"} 32 | {"obj": "kafka.network:type=RequestMetrics,name=RequestQueueTimeMs,request=Produce", "attr": ["Count","Min","Max","Mean","StdDev", "50thPercentile","75thPercentile","95thPercentile","98thPercentile","99thPercentile","999thPercentile"], "resultAlias": "ProduceRequestQueueTimeMs"} 33 | {"obj": "kafka.log:type=LogFlushStats,name=LogFlushRateAndTimeMs", "attr": ["Count","MeanRate","OneMinuteRate","FiveMinuteRate","FifteenMinuteRate","Min","Max","Mean","StdDev", "50thPercentile","75thPercentile","95thPercentile","98thPercentile","99thPercentile","999thPercentile"], "resultAlias": "LogFlush"} 34 | {"obj": "kafka.network:type=SocketServer,name=NetworkProcessorAvgIdlePercent", "attr": ["Value"], "resultAlias": "NetworkProcessorAvgIdlePercent"} 35 | {"obj": "kafka.server:type=FetcherLagMetrics,name=ConsumerLag,clientId=*,topic=*,partition=*", "attr": ["Value"], "resultAlias": "ConsumerLag"} 36 | {"obj": "kafka.server:type=ProducerRequestPurgatory,name=PurgatorySize", "attr": ["Value"], "resultAlias": "ProducerRequestsPurgatorySize"} 37 | {"obj": "kafka.server:type=FetchRequestPurgatory,name=PurgatorySize", "attr": ["Value"], "resultAlias": "FetchRequestPurgatorySize"} 38 | boot.sh: |+ 39 | #!/bin/sh 40 | 41 | set -e 42 | set -x 43 | 44 | export JMXTRANS_OPTS="$JMXTRANS_OPTS -Dlogback.configurationFile=file:///${JMXTRANS_HOME}/conf/logback.xml -DkafkaJmxPort=$KAFKA_JMX_PORT -Dalias=$POD_NAME -DinfluxUrl=$INFLUXDB_HOST -DinfluxDatabase=$INFLUXDB_DATABASE -DinfluxUser=$INFLUXDB_USER -DinfluxPass=$INFLUXDB_PASSWORD" 45 | exec java $JAVA_OPTS $JMXTRANS_OPTS -jar $JAR_FILE -e -j $JSON_DIR -s $SECONDS_BETWEEN_RUNS -c $CONTINUE_ON_ERROR 46 | # A headless service to create DNS records 47 | apiVersion: v1 48 | kind: Service 49 | metadata: 50 | labels: 51 | app: kafka 52 | name: kafka 53 | spec: 54 | clusterIP: None 55 | ports: 56 | - { name: "kafka", port: 9092 } 57 | selector: 58 | app: kafka 59 | --- 60 | apiVersion: policy/v1beta1 61 | kind: PodDisruptionBudget 62 | metadata: 63 | name: kafka 64 | spec: 65 | minAvailable: 2 66 | selector: 67 | matchLabels: 68 | app: kafka 69 | --- 70 | apiVersion: apps/v1 71 | kind: StatefulSet 72 | metadata: 73 | name: kafka 74 | labels: 75 | app: kafka 76 | spec: 77 | serviceName: kafka 78 | replicas: 3 79 | podManagementPolicy: Parallel 80 | updateStrategy: 81 | type: RollingUpdate 82 | selector: 83 | matchLabels: 84 | app: kafka 85 | template: 86 | metadata: 87 | labels: 88 | app: kafka 89 | spec: 90 | initContainers: 91 | - name: jmxtrans-init 92 | image: realguess/jq:1.4 93 | volumeMounts: 94 | - name: kafka-jmxtrans-config 95 | mountPath: /metrics-input/metrics.json 96 | subPath: metrics.json 97 | - name: jmxtrans-input 98 | mountPath: /jmxtrans-input 99 | command: 100 | - sh 101 | - -c 102 | - >- 103 | jq -s -c '{"servers": [{"port": "${kafkaJmxPort}","host":"localhost","alias":"${alias}", "queries":map( {"outputWriters": [{"@class":"com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory","url":"${influxUrl}","database":"${influxDatabase}","createDatabase":false,"username":"${influxUser}","password":"${influxPass}"}], "obj": .obj, "attr": .attr, "resultAlias": .resultAlias})}]}' 104 | /metrics-input/metrics.json > /jmxtrans-input/kafka.json 105 | containers: 106 | - name: kafka 107 | image: wurstmeister/kafka:2.12-2.2.0 108 | env: 109 | - { name: KAFKA_LISTENERS, value: "PLAINTEXT://:9092" } 110 | - { name: HOSTNAME_COMMAND, value: "echo $HOSTNAME" } 111 | - { name: KAFKA_ADVERTISED_LISTENERS, value: "PLAINTEXT://_{HOSTNAME_COMMAND}.kafka:9092" } 112 | - { name: BROKER_ID_COMMAND, value: "echo ${HOSTNAME##*-}" } 113 | - { name: KAFKA_HEAP_OPTS, value: "-Xms1g -Xmx1g" } 114 | - { name: KAFKA_ZOOKEEPER_CONNECT, value: "zookeeper" } 115 | - { name: KAFKA_JMX_OPTS, value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=127.0.0.1 -Dcom.sun.management.jmxremote.rmi.port=9010" } 116 | - { name: JMX_PORT, value: "9010" } 117 | resources: 118 | requests: { memory: "2Gi" } 119 | limits: { memory: "2Gi" } 120 | ports: 121 | - { name: jmx-port, containerPort: 9010, protocol: TCP } 122 | - { name: broker-port, containerPort: 9092, protocol: TCP } 123 | readinessProbe: 124 | tcpSocket: 125 | port: 9092 126 | volumeMounts: 127 | - {name: data, mountPath: /var/lib/kafka } 128 | - {name: tmp, mountPath: /tmp } 129 | - {name: vartmp, mountPath: /var/tmp } 130 | - name: "jmxtrans" 131 | image: "jmxtrans/jmxtrans" 132 | command: ["/boot.sh"] 133 | env: 134 | - { name: "POD_NAME", valueFrom: { fieldRef: { fieldPath: "metadata.name" }}} 135 | - { name: "HOST_NAME", valueFrom: { fieldRef: { fieldPath: "spec.nodeName" }}} 136 | - { name: "KAFKA_JMX_PORT", value: "9010" } 137 | - { name: "INFLUXDB_DATABASE", value: "kafka" } 138 | - { name: "INFLUXDB_HOST", value: "http://influxdb" } 139 | - { name: "INFLUXDB_USER", value: "jeroen" } 140 | - { name: "INFLUXDB_PASSWORD", valueFrom: { secretKeyRef: { name: influxdb-secret, key: user-password }}} 141 | volumeMounts: 142 | - name: jmxtrans-input 143 | mountPath: /var/lib/jmxtrans 144 | readOnly: true 145 | - name: kafka-jmxtrans-config 146 | mountPath: /boot.sh 147 | subPath: boot.sh 148 | readOnly: true 149 | volumes: 150 | - { name: tmp, emptyDir: {} } 151 | - { name: vartmp, emptyDir: {} } 152 | - { name: data, emptyDir: {} } 153 | - { name: jmxtrans-input, emptyDir: {} } 154 | - { name: kafka-jmxtrans-config, configMap: { name: kafka-jmxtrans-config, defaultMode: 0744 } } -------------------------------------------------------------------------------- /dashboards/influxdb-consumer-metrics.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "editable": true, 16 | "gnetId": null, 17 | "graphTooltip": 0, 18 | "id": 2, 19 | "iteration": 1559676468215, 20 | "links": [], 21 | "panels": [ 22 | { 23 | "aliasColors": {}, 24 | "bars": false, 25 | "dashLength": 10, 26 | "dashes": false, 27 | "datasource": "influxdb-kafka", 28 | "description": "The maximum lag in terms of number of records for any partition in this window. An increasing value over time is your best indication that the consumer group is not keeping up with the producers.", 29 | "editable": true, 30 | "error": false, 31 | "fill": 1, 32 | "grid": {}, 33 | "gridPos": { 34 | "h": 7, 35 | "w": 24, 36 | "x": 0, 37 | "y": 0 38 | }, 39 | "id": 5, 40 | "isNew": true, 41 | "legend": { 42 | "avg": false, 43 | "current": false, 44 | "hideEmpty": false, 45 | "max": false, 46 | "min": false, 47 | "show": true, 48 | "total": false, 49 | "values": false 50 | }, 51 | "lines": true, 52 | "linewidth": 2, 53 | "links": [], 54 | "nullPointMode": "connected", 55 | "paceLength": 10, 56 | "percentage": false, 57 | "pointradius": 5, 58 | "points": false, 59 | "renderer": "flot", 60 | "seriesOverrides": [], 61 | "spaceLength": 10, 62 | "stack": false, 63 | "steppedLine": false, 64 | "targets": [ 65 | { 66 | "alias": "[[tag_hostname]]", 67 | "dsType": "influxdb", 68 | "groupBy": [ 69 | { 70 | "params": [ 71 | "$interval" 72 | ], 73 | "type": "time" 74 | }, 75 | { 76 | "params": [ 77 | "hostname" 78 | ], 79 | "type": "tag" 80 | } 81 | ], 82 | "measurement": "mmConsumerFetchMetrics", 83 | "orderByTime": "ASC", 84 | "policy": "default", 85 | "refId": "A", 86 | "resultFormat": "time_series", 87 | "select": [ 88 | [ 89 | { 90 | "params": [ 91 | "records-lag-max" 92 | ], 93 | "type": "field" 94 | }, 95 | { 96 | "params": [], 97 | "type": "max" 98 | } 99 | ] 100 | ], 101 | "tags": [] 102 | } 103 | ], 104 | "thresholds": [], 105 | "timeFrom": null, 106 | "timeRegions": [], 107 | "timeShift": null, 108 | "title": "Max lag for any partition", 109 | "tooltip": { 110 | "msResolution": false, 111 | "shared": true, 112 | "sort": 0, 113 | "value_type": "cumulative" 114 | }, 115 | "type": "graph", 116 | "xaxis": { 117 | "buckets": null, 118 | "mode": "time", 119 | "name": null, 120 | "show": true, 121 | "values": [] 122 | }, 123 | "yaxes": [ 124 | { 125 | "format": "short", 126 | "label": "# records", 127 | "logBase": 1, 128 | "max": null, 129 | "min": null, 130 | "show": true 131 | }, 132 | { 133 | "format": "short", 134 | "label": null, 135 | "logBase": 1, 136 | "max": null, 137 | "min": null, 138 | "show": true 139 | } 140 | ], 141 | "yaxis": { 142 | "align": false, 143 | "alignLevel": null 144 | } 145 | }, 146 | { 147 | "aliasColors": {}, 148 | "bars": false, 149 | "dashLength": 10, 150 | "dashes": false, 151 | "datasource": "influxdb-kafka", 152 | "editable": true, 153 | "error": false, 154 | "fill": 1, 155 | "grid": {}, 156 | "gridPos": { 157 | "h": 7, 158 | "w": 8, 159 | "x": 0, 160 | "y": 7 161 | }, 162 | "id": 2, 163 | "isNew": true, 164 | "legend": { 165 | "avg": false, 166 | "current": false, 167 | "max": false, 168 | "min": false, 169 | "show": true, 170 | "total": false, 171 | "values": false 172 | }, 173 | "lines": true, 174 | "linewidth": 2, 175 | "links": [], 176 | "nullPointMode": "connected", 177 | "paceLength": 10, 178 | "percentage": false, 179 | "pointradius": 5, 180 | "points": false, 181 | "renderer": "flot", 182 | "seriesOverrides": [], 183 | "spaceLength": 10, 184 | "stack": false, 185 | "steppedLine": false, 186 | "targets": [ 187 | { 188 | "alias": "$tag_hostname", 189 | "dsType": "influxdb", 190 | "groupBy": [ 191 | { 192 | "params": [ 193 | "$interval" 194 | ], 195 | "type": "time" 196 | }, 197 | { 198 | "params": [ 199 | "hostname" 200 | ], 201 | "type": "tag" 202 | } 203 | ], 204 | "measurement": "mmConsumerFetchMetrics", 205 | "orderByTime": "ASC", 206 | "policy": "default", 207 | "refId": "A", 208 | "resultFormat": "time_series", 209 | "select": [ 210 | [ 211 | { 212 | "params": [ 213 | "bytes-consumed-rate" 214 | ], 215 | "type": "field" 216 | }, 217 | { 218 | "params": [], 219 | "type": "mean" 220 | } 221 | ] 222 | ], 223 | "tags": [] 224 | } 225 | ], 226 | "thresholds": [], 227 | "timeFrom": null, 228 | "timeRegions": [], 229 | "timeShift": null, 230 | "title": "Bytes Consumed per sec (avg)", 231 | "tooltip": { 232 | "msResolution": false, 233 | "shared": true, 234 | "sort": 0, 235 | "value_type": "cumulative" 236 | }, 237 | "type": "graph", 238 | "xaxis": { 239 | "buckets": null, 240 | "mode": "time", 241 | "name": null, 242 | "show": true, 243 | "values": [] 244 | }, 245 | "yaxes": [ 246 | { 247 | "format": "bytes", 248 | "label": null, 249 | "logBase": 1, 250 | "max": null, 251 | "min": null, 252 | "show": true 253 | }, 254 | { 255 | "format": "short", 256 | "label": null, 257 | "logBase": 1, 258 | "max": null, 259 | "min": null, 260 | "show": true 261 | } 262 | ], 263 | "yaxis": { 264 | "align": false, 265 | "alignLevel": null 266 | } 267 | }, 268 | { 269 | "aliasColors": {}, 270 | "bars": false, 271 | "dashLength": 10, 272 | "dashes": false, 273 | "datasource": "influxdb-kafka", 274 | "editable": true, 275 | "error": false, 276 | "fill": 1, 277 | "grid": {}, 278 | "gridPos": { 279 | "h": 7, 280 | "w": 8, 281 | "x": 8, 282 | "y": 7 283 | }, 284 | "id": 3, 285 | "isNew": true, 286 | "legend": { 287 | "avg": false, 288 | "current": false, 289 | "max": false, 290 | "min": false, 291 | "show": true, 292 | "total": false, 293 | "values": false 294 | }, 295 | "lines": true, 296 | "linewidth": 2, 297 | "links": [], 298 | "nullPointMode": "connected", 299 | "paceLength": 10, 300 | "percentage": false, 301 | "pointradius": 5, 302 | "points": false, 303 | "renderer": "flot", 304 | "seriesOverrides": [], 305 | "spaceLength": 10, 306 | "stack": false, 307 | "steppedLine": false, 308 | "targets": [ 309 | { 310 | "alias": "$tag_hostname", 311 | "dsType": "influxdb", 312 | "groupBy": [ 313 | { 314 | "params": [ 315 | "$interval" 316 | ], 317 | "type": "time" 318 | }, 319 | { 320 | "params": [ 321 | "hostname" 322 | ], 323 | "type": "tag" 324 | } 325 | ], 326 | "measurement": "mmConsumerFetchMetrics", 327 | "orderByTime": "ASC", 328 | "policy": "default", 329 | "refId": "A", 330 | "resultFormat": "time_series", 331 | "select": [ 332 | [ 333 | { 334 | "params": [ 335 | "fetch-latency-avg" 336 | ], 337 | "type": "field" 338 | }, 339 | { 340 | "params": [], 341 | "type": "last" 342 | } 343 | ] 344 | ], 345 | "tags": [] 346 | } 347 | ], 348 | "thresholds": [], 349 | "timeFrom": null, 350 | "timeRegions": [], 351 | "timeShift": null, 352 | "title": "Fetch Latency (avg)", 353 | "tooltip": { 354 | "msResolution": false, 355 | "shared": true, 356 | "sort": 0, 357 | "value_type": "cumulative" 358 | }, 359 | "type": "graph", 360 | "xaxis": { 361 | "buckets": null, 362 | "mode": "time", 363 | "name": null, 364 | "show": true, 365 | "values": [] 366 | }, 367 | "yaxes": [ 368 | { 369 | "format": "ms", 370 | "label": null, 371 | "logBase": 1, 372 | "max": null, 373 | "min": null, 374 | "show": true 375 | }, 376 | { 377 | "format": "short", 378 | "label": null, 379 | "logBase": 1, 380 | "max": null, 381 | "min": null, 382 | "show": true 383 | } 384 | ], 385 | "yaxis": { 386 | "align": false, 387 | "alignLevel": null 388 | } 389 | }, 390 | { 391 | "aliasColors": {}, 392 | "bars": false, 393 | "dashLength": 10, 394 | "dashes": false, 395 | "datasource": "influxdb-kafka", 396 | "editable": true, 397 | "error": false, 398 | "fill": 1, 399 | "grid": {}, 400 | "gridPos": { 401 | "h": 7, 402 | "w": 8, 403 | "x": 16, 404 | "y": 7 405 | }, 406 | "id": 4, 407 | "interval": "", 408 | "isNew": true, 409 | "legend": { 410 | "avg": false, 411 | "current": false, 412 | "max": false, 413 | "min": false, 414 | "show": true, 415 | "total": false, 416 | "values": false 417 | }, 418 | "lines": true, 419 | "linewidth": 2, 420 | "links": [], 421 | "nullPointMode": "connected", 422 | "paceLength": 10, 423 | "percentage": false, 424 | "pointradius": 5, 425 | "points": false, 426 | "renderer": "flot", 427 | "seriesOverrides": [], 428 | "spaceLength": 10, 429 | "stack": false, 430 | "steppedLine": false, 431 | "targets": [ 432 | { 433 | "alias": "$tag_hostname", 434 | "dsType": "influxdb", 435 | "groupBy": [ 436 | { 437 | "params": [ 438 | "$interval" 439 | ], 440 | "type": "time" 441 | }, 442 | { 443 | "params": [ 444 | "hostname" 445 | ], 446 | "type": "tag" 447 | } 448 | ], 449 | "measurement": "mmConsumerFetchMetrics", 450 | "orderByTime": "ASC", 451 | "policy": "default", 452 | "refId": "A", 453 | "resultFormat": "time_series", 454 | "select": [ 455 | [ 456 | { 457 | "params": [ 458 | "fetch-rate" 459 | ], 460 | "type": "field" 461 | }, 462 | { 463 | "params": [], 464 | "type": "last" 465 | } 466 | ] 467 | ], 468 | "tags": [] 469 | } 470 | ], 471 | "thresholds": [], 472 | "timeFrom": null, 473 | "timeRegions": [], 474 | "timeShift": null, 475 | "title": "Fetch Requests per second", 476 | "tooltip": { 477 | "msResolution": false, 478 | "shared": true, 479 | "sort": 0, 480 | "value_type": "cumulative" 481 | }, 482 | "type": "graph", 483 | "xaxis": { 484 | "buckets": null, 485 | "mode": "time", 486 | "name": null, 487 | "show": true, 488 | "values": [] 489 | }, 490 | "yaxes": [ 491 | { 492 | "format": "short", 493 | "label": "req/s", 494 | "logBase": 1, 495 | "max": null, 496 | "min": null, 497 | "show": true 498 | }, 499 | { 500 | "format": "short", 501 | "label": null, 502 | "logBase": 1, 503 | "max": null, 504 | "min": null, 505 | "show": true 506 | } 507 | ], 508 | "yaxis": { 509 | "align": false, 510 | "alignLevel": null 511 | } 512 | }, 513 | { 514 | "aliasColors": {}, 515 | "bars": false, 516 | "dashLength": 10, 517 | "dashes": false, 518 | "datasource": "influxdb-kafka", 519 | "editable": true, 520 | "error": false, 521 | "fill": 1, 522 | "grid": {}, 523 | "gridPos": { 524 | "h": 7, 525 | "w": 12, 526 | "x": 0, 527 | "y": 14 528 | }, 529 | "id": 6, 530 | "isNew": true, 531 | "legend": { 532 | "avg": false, 533 | "current": false, 534 | "max": false, 535 | "min": false, 536 | "show": true, 537 | "total": false, 538 | "values": false 539 | }, 540 | "lines": true, 541 | "linewidth": 2, 542 | "links": [], 543 | "nullPointMode": "connected", 544 | "paceLength": 10, 545 | "percentage": false, 546 | "pointradius": 5, 547 | "points": false, 548 | "renderer": "flot", 549 | "seriesOverrides": [], 550 | "spaceLength": 10, 551 | "stack": false, 552 | "steppedLine": false, 553 | "targets": [ 554 | { 555 | "alias": "$tag_hostname", 556 | "dsType": "influxdb", 557 | "groupBy": [ 558 | { 559 | "params": [ 560 | "$interval" 561 | ], 562 | "type": "time" 563 | }, 564 | { 565 | "params": [ 566 | "hostname" 567 | ], 568 | "type": "tag" 569 | } 570 | ], 571 | "measurement": "mmConsumerFetchMetrics", 572 | "orderByTime": "ASC", 573 | "policy": "default", 574 | "refId": "A", 575 | "resultFormat": "time_series", 576 | "select": [ 577 | [ 578 | { 579 | "params": [ 580 | "records-per-request-avg" 581 | ], 582 | "type": "field" 583 | }, 584 | { 585 | "params": [], 586 | "type": "last" 587 | } 588 | ] 589 | ], 590 | "tags": [] 591 | } 592 | ], 593 | "thresholds": [], 594 | "timeFrom": null, 595 | "timeRegions": [], 596 | "timeShift": null, 597 | "title": "Records per request", 598 | "tooltip": { 599 | "msResolution": false, 600 | "shared": true, 601 | "sort": 0, 602 | "value_type": "cumulative" 603 | }, 604 | "type": "graph", 605 | "xaxis": { 606 | "buckets": null, 607 | "mode": "time", 608 | "name": null, 609 | "show": true, 610 | "values": [] 611 | }, 612 | "yaxes": [ 613 | { 614 | "format": "short", 615 | "label": "rec/req", 616 | "logBase": 1, 617 | "max": null, 618 | "min": null, 619 | "show": true 620 | }, 621 | { 622 | "format": "short", 623 | "label": null, 624 | "logBase": 1, 625 | "max": null, 626 | "min": null, 627 | "show": true 628 | } 629 | ], 630 | "yaxis": { 631 | "align": false, 632 | "alignLevel": null 633 | } 634 | }, 635 | { 636 | "aliasColors": {}, 637 | "bars": false, 638 | "dashLength": 10, 639 | "dashes": false, 640 | "datasource": "influxdb-kafka", 641 | "description": "The average number of records per second", 642 | "editable": true, 643 | "error": false, 644 | "fill": 1, 645 | "grid": {}, 646 | "gridPos": { 647 | "h": 7, 648 | "w": 12, 649 | "x": 12, 650 | "y": 14 651 | }, 652 | "id": 7, 653 | "isNew": true, 654 | "legend": { 655 | "avg": false, 656 | "current": false, 657 | "max": false, 658 | "min": false, 659 | "show": true, 660 | "total": false, 661 | "values": false 662 | }, 663 | "lines": true, 664 | "linewidth": 2, 665 | "links": [], 666 | "nullPointMode": "connected", 667 | "paceLength": 10, 668 | "percentage": false, 669 | "pointradius": 5, 670 | "points": false, 671 | "renderer": "flot", 672 | "seriesOverrides": [], 673 | "spaceLength": 10, 674 | "stack": false, 675 | "steppedLine": false, 676 | "targets": [ 677 | { 678 | "alias": "$tag_hostname", 679 | "dsType": "influxdb", 680 | "groupBy": [ 681 | { 682 | "params": [ 683 | "$interval" 684 | ], 685 | "type": "time" 686 | }, 687 | { 688 | "params": [ 689 | "hostname" 690 | ], 691 | "type": "tag" 692 | } 693 | ], 694 | "measurement": "mmConsumerPerTopicFetchMetrics", 695 | "orderByTime": "ASC", 696 | "policy": "default", 697 | "refId": "A", 698 | "resultFormat": "time_series", 699 | "select": [ 700 | [ 701 | { 702 | "params": [ 703 | "records-consumed-rate" 704 | ], 705 | "type": "field" 706 | }, 707 | { 708 | "params": [], 709 | "type": "last" 710 | } 711 | ] 712 | ], 713 | "tags": [] 714 | } 715 | ], 716 | "thresholds": [], 717 | "timeFrom": null, 718 | "timeRegions": [], 719 | "timeShift": null, 720 | "title": "Records per second", 721 | "tooltip": { 722 | "msResolution": false, 723 | "shared": true, 724 | "sort": 0, 725 | "value_type": "cumulative" 726 | }, 727 | "type": "graph", 728 | "xaxis": { 729 | "buckets": null, 730 | "mode": "time", 731 | "name": null, 732 | "show": true, 733 | "values": [] 734 | }, 735 | "yaxes": [ 736 | { 737 | "format": "short", 738 | "label": "rec/s", 739 | "logBase": 1, 740 | "max": null, 741 | "min": null, 742 | "show": true 743 | }, 744 | { 745 | "format": "short", 746 | "label": null, 747 | "logBase": 1, 748 | "max": null, 749 | "min": null, 750 | "show": true 751 | } 752 | ], 753 | "yaxis": { 754 | "align": false, 755 | "alignLevel": null 756 | } 757 | }, 758 | { 759 | "aliasColors": {}, 760 | "bars": false, 761 | "dashLength": 10, 762 | "dashes": false, 763 | "datasource": "influxdb-kafka", 764 | "description": "The average time taken for a commit request.", 765 | "editable": true, 766 | "error": false, 767 | "fill": 1, 768 | "grid": {}, 769 | "gridPos": { 770 | "h": 7, 771 | "w": 12, 772 | "x": 0, 773 | "y": 21 774 | }, 775 | "id": 9, 776 | "isNew": true, 777 | "legend": { 778 | "avg": false, 779 | "current": false, 780 | "max": false, 781 | "min": false, 782 | "show": true, 783 | "total": false, 784 | "values": false 785 | }, 786 | "lines": true, 787 | "linewidth": 2, 788 | "links": [], 789 | "nullPointMode": "connected", 790 | "paceLength": 10, 791 | "percentage": false, 792 | "pointradius": 5, 793 | "points": false, 794 | "renderer": "flot", 795 | "seriesOverrides": [], 796 | "spaceLength": 10, 797 | "stack": false, 798 | "steppedLine": false, 799 | "targets": [ 800 | { 801 | "alias": "$tag_hostname", 802 | "dsType": "influxdb", 803 | "groupBy": [ 804 | { 805 | "params": [ 806 | "$interval" 807 | ], 808 | "type": "time" 809 | }, 810 | { 811 | "params": [ 812 | "hostname" 813 | ], 814 | "type": "tag" 815 | } 816 | ], 817 | "measurement": "mmConsumerGroupMetrics", 818 | "orderByTime": "ASC", 819 | "policy": "default", 820 | "refId": "A", 821 | "resultFormat": "time_series", 822 | "select": [ 823 | [ 824 | { 825 | "params": [ 826 | "commit-latency-avg" 827 | ], 828 | "type": "field" 829 | }, 830 | { 831 | "params": [], 832 | "type": "last" 833 | } 834 | ] 835 | ], 836 | "tags": [] 837 | } 838 | ], 839 | "thresholds": [], 840 | "timeFrom": null, 841 | "timeRegions": [], 842 | "timeShift": null, 843 | "title": "Commit Latency (avg)", 844 | "tooltip": { 845 | "msResolution": false, 846 | "shared": true, 847 | "sort": 0, 848 | "value_type": "cumulative" 849 | }, 850 | "type": "graph", 851 | "xaxis": { 852 | "buckets": null, 853 | "mode": "time", 854 | "name": null, 855 | "show": true, 856 | "values": [] 857 | }, 858 | "yaxes": [ 859 | { 860 | "format": "ms", 861 | "label": "", 862 | "logBase": 1, 863 | "max": null, 864 | "min": null, 865 | "show": true 866 | }, 867 | { 868 | "format": "short", 869 | "label": null, 870 | "logBase": 1, 871 | "max": null, 872 | "min": null, 873 | "show": true 874 | } 875 | ], 876 | "yaxis": { 877 | "align": false, 878 | "alignLevel": null 879 | } 880 | }, 881 | { 882 | "aliasColors": {}, 883 | "bars": false, 884 | "dashLength": 10, 885 | "dashes": false, 886 | "datasource": "influxdb-kafka", 887 | "description": "The number of commit calls per second.", 888 | "editable": true, 889 | "error": false, 890 | "fill": 1, 891 | "grid": {}, 892 | "gridPos": { 893 | "h": 7, 894 | "w": 12, 895 | "x": 12, 896 | "y": 21 897 | }, 898 | "id": 8, 899 | "isNew": true, 900 | "legend": { 901 | "avg": false, 902 | "current": false, 903 | "max": false, 904 | "min": false, 905 | "show": true, 906 | "total": false, 907 | "values": false 908 | }, 909 | "lines": true, 910 | "linewidth": 2, 911 | "links": [], 912 | "nullPointMode": "connected", 913 | "paceLength": 10, 914 | "percentage": false, 915 | "pointradius": 5, 916 | "points": false, 917 | "renderer": "flot", 918 | "seriesOverrides": [], 919 | "spaceLength": 10, 920 | "stack": false, 921 | "steppedLine": false, 922 | "targets": [ 923 | { 924 | "alias": "$tag_hostname", 925 | "dsType": "influxdb", 926 | "groupBy": [ 927 | { 928 | "params": [ 929 | "$interval" 930 | ], 931 | "type": "time" 932 | }, 933 | { 934 | "params": [ 935 | "hostname" 936 | ], 937 | "type": "tag" 938 | } 939 | ], 940 | "measurement": "mmConsumerGroupMetrics", 941 | "orderByTime": "ASC", 942 | "policy": "default", 943 | "refId": "A", 944 | "resultFormat": "time_series", 945 | "select": [ 946 | [ 947 | { 948 | "params": [ 949 | "commit-rate" 950 | ], 951 | "type": "field" 952 | }, 953 | { 954 | "params": [], 955 | "type": "last" 956 | } 957 | ] 958 | ], 959 | "tags": [] 960 | } 961 | ], 962 | "thresholds": [], 963 | "timeFrom": null, 964 | "timeRegions": [], 965 | "timeShift": null, 966 | "title": "Commits per second", 967 | "tooltip": { 968 | "msResolution": false, 969 | "shared": true, 970 | "sort": 0, 971 | "value_type": "cumulative" 972 | }, 973 | "type": "graph", 974 | "xaxis": { 975 | "buckets": null, 976 | "mode": "time", 977 | "name": null, 978 | "show": true, 979 | "values": [] 980 | }, 981 | "yaxes": [ 982 | { 983 | "format": "short", 984 | "label": "commit/s", 985 | "logBase": 1, 986 | "max": null, 987 | "min": null, 988 | "show": true 989 | }, 990 | { 991 | "format": "short", 992 | "label": null, 993 | "logBase": 1, 994 | "max": null, 995 | "min": null, 996 | "show": true 997 | } 998 | ], 999 | "yaxis": { 1000 | "align": false, 1001 | "alignLevel": null 1002 | } 1003 | }, 1004 | { 1005 | "aliasColors": {}, 1006 | "bars": false, 1007 | "dashLength": 10, 1008 | "dashes": false, 1009 | "datasource": "influxdb-kafka", 1010 | "description": "The number of group syncs per second. Group synchronization is the second and last phase of the rebalance protocol. Similar to join-rate, a large value indicates group instability.", 1011 | "editable": true, 1012 | "error": false, 1013 | "fill": 1, 1014 | "grid": {}, 1015 | "gridPos": { 1016 | "h": 7, 1017 | "w": 12, 1018 | "x": 0, 1019 | "y": 28 1020 | }, 1021 | "id": 10, 1022 | "isNew": true, 1023 | "legend": { 1024 | "avg": false, 1025 | "current": false, 1026 | "max": false, 1027 | "min": false, 1028 | "show": true, 1029 | "total": false, 1030 | "values": false 1031 | }, 1032 | "lines": true, 1033 | "linewidth": 2, 1034 | "links": [], 1035 | "nullPointMode": "connected", 1036 | "paceLength": 10, 1037 | "percentage": false, 1038 | "pointradius": 5, 1039 | "points": false, 1040 | "renderer": "flot", 1041 | "seriesOverrides": [], 1042 | "spaceLength": 10, 1043 | "stack": false, 1044 | "steppedLine": false, 1045 | "targets": [ 1046 | { 1047 | "alias": "$tag_hostname", 1048 | "dsType": "influxdb", 1049 | "groupBy": [ 1050 | { 1051 | "params": [ 1052 | "$interval" 1053 | ], 1054 | "type": "time" 1055 | }, 1056 | { 1057 | "params": [ 1058 | "hostname" 1059 | ], 1060 | "type": "tag" 1061 | } 1062 | ], 1063 | "measurement": "mmConsumerGroupMetrics", 1064 | "orderByTime": "ASC", 1065 | "policy": "default", 1066 | "refId": "A", 1067 | "resultFormat": "time_series", 1068 | "select": [ 1069 | [ 1070 | { 1071 | "params": [ 1072 | "sync-rate" 1073 | ], 1074 | "type": "field" 1075 | }, 1076 | { 1077 | "params": [], 1078 | "type": "last" 1079 | } 1080 | ] 1081 | ], 1082 | "tags": [] 1083 | } 1084 | ], 1085 | "thresholds": [], 1086 | "timeFrom": null, 1087 | "timeRegions": [], 1088 | "timeShift": null, 1089 | "title": "Sync Rate", 1090 | "tooltip": { 1091 | "msResolution": false, 1092 | "shared": true, 1093 | "sort": 0, 1094 | "value_type": "cumulative" 1095 | }, 1096 | "type": "graph", 1097 | "xaxis": { 1098 | "buckets": null, 1099 | "mode": "time", 1100 | "name": null, 1101 | "show": true, 1102 | "values": [] 1103 | }, 1104 | "yaxes": [ 1105 | { 1106 | "format": "short", 1107 | "label": "syncs/s", 1108 | "logBase": 1, 1109 | "max": null, 1110 | "min": null, 1111 | "show": true 1112 | }, 1113 | { 1114 | "format": "short", 1115 | "label": null, 1116 | "logBase": 1, 1117 | "max": null, 1118 | "min": null, 1119 | "show": true 1120 | } 1121 | ], 1122 | "yaxis": { 1123 | "align": false, 1124 | "alignLevel": null 1125 | } 1126 | }, 1127 | { 1128 | "aliasColors": {}, 1129 | "bars": false, 1130 | "dashLength": 10, 1131 | "dashes": false, 1132 | "datasource": "influxdb-kafka", 1133 | "description": "The number of group joins per second. Group joining is the first phase of the rebalance protocol. A large value indicates that the consumer group is unstable and will likely be coupled with increased lag.", 1134 | "editable": true, 1135 | "error": false, 1136 | "fill": 1, 1137 | "grid": {}, 1138 | "gridPos": { 1139 | "h": 7, 1140 | "w": 12, 1141 | "x": 12, 1142 | "y": 28 1143 | }, 1144 | "id": 11, 1145 | "isNew": true, 1146 | "legend": { 1147 | "avg": false, 1148 | "current": false, 1149 | "max": false, 1150 | "min": false, 1151 | "show": true, 1152 | "total": false, 1153 | "values": false 1154 | }, 1155 | "lines": true, 1156 | "linewidth": 2, 1157 | "links": [], 1158 | "nullPointMode": "connected", 1159 | "paceLength": 10, 1160 | "percentage": false, 1161 | "pointradius": 5, 1162 | "points": false, 1163 | "renderer": "flot", 1164 | "seriesOverrides": [], 1165 | "spaceLength": 10, 1166 | "stack": false, 1167 | "steppedLine": false, 1168 | "targets": [ 1169 | { 1170 | "alias": "$tag_hostname", 1171 | "dsType": "influxdb", 1172 | "groupBy": [ 1173 | { 1174 | "params": [ 1175 | "$interval" 1176 | ], 1177 | "type": "time" 1178 | }, 1179 | { 1180 | "params": [ 1181 | "hostname" 1182 | ], 1183 | "type": "tag" 1184 | } 1185 | ], 1186 | "measurement": "mmConsumerGroupMetrics", 1187 | "orderByTime": "ASC", 1188 | "policy": "default", 1189 | "refId": "A", 1190 | "resultFormat": "time_series", 1191 | "select": [ 1192 | [ 1193 | { 1194 | "params": [ 1195 | "join-rate" 1196 | ], 1197 | "type": "field" 1198 | }, 1199 | { 1200 | "params": [], 1201 | "type": "last" 1202 | } 1203 | ] 1204 | ], 1205 | "tags": [] 1206 | } 1207 | ], 1208 | "thresholds": [], 1209 | "timeFrom": null, 1210 | "timeRegions": [], 1211 | "timeShift": null, 1212 | "title": "Join Rate", 1213 | "tooltip": { 1214 | "msResolution": false, 1215 | "shared": true, 1216 | "sort": 0, 1217 | "value_type": "cumulative" 1218 | }, 1219 | "type": "graph", 1220 | "xaxis": { 1221 | "buckets": null, 1222 | "mode": "time", 1223 | "name": null, 1224 | "show": true, 1225 | "values": [] 1226 | }, 1227 | "yaxes": [ 1228 | { 1229 | "format": "short", 1230 | "label": "joins/s", 1231 | "logBase": 1, 1232 | "max": null, 1233 | "min": null, 1234 | "show": true 1235 | }, 1236 | { 1237 | "format": "short", 1238 | "label": null, 1239 | "logBase": 1, 1240 | "max": null, 1241 | "min": null, 1242 | "show": true 1243 | } 1244 | ], 1245 | "yaxis": { 1246 | "align": false, 1247 | "alignLevel": null 1248 | } 1249 | }, 1250 | { 1251 | "cacheTimeout": null, 1252 | "colorBackground": false, 1253 | "colorValue": false, 1254 | "colors": [ 1255 | "#299c46", 1256 | "rgba(237, 129, 40, 0.89)", 1257 | "#d44a3a" 1258 | ], 1259 | "datasource": "influxdb-kafka", 1260 | "description": "The number of partitions currently assigned to this consumer of host $Hostname", 1261 | "format": "none", 1262 | "gauge": { 1263 | "maxValue": 100, 1264 | "minValue": 0, 1265 | "show": false, 1266 | "thresholdLabels": false, 1267 | "thresholdMarkers": true 1268 | }, 1269 | "gridPos": { 1270 | "h": 2, 1271 | "w": 24, 1272 | "x": 0, 1273 | "y": 35 1274 | }, 1275 | "id": 14, 1276 | "interval": null, 1277 | "links": [], 1278 | "mappingType": 1, 1279 | "mappingTypes": [ 1280 | { 1281 | "name": "value to text", 1282 | "value": 1 1283 | }, 1284 | { 1285 | "name": "range to text", 1286 | "value": 2 1287 | } 1288 | ], 1289 | "maxDataPoints": 100, 1290 | "nullPointMode": "connected", 1291 | "nullText": null, 1292 | "postfix": "", 1293 | "postfixFontSize": "50%", 1294 | "prefix": "", 1295 | "prefixFontSize": "50%", 1296 | "rangeMaps": [ 1297 | { 1298 | "from": "null", 1299 | "text": "N/A", 1300 | "to": "null" 1301 | } 1302 | ], 1303 | "repeat": "Hostname", 1304 | "repeatDirection": "h", 1305 | "scopedVars": { 1306 | "Hostname": { 1307 | "selected": true, 1308 | "text": "orders-consumer-65d857559f-wtgl7", 1309 | "value": "orders-consumer-65d857559f-wtgl7" 1310 | } 1311 | }, 1312 | "sparkline": { 1313 | "fillColor": "rgba(31, 118, 189, 0.18)", 1314 | "full": false, 1315 | "lineColor": "rgb(31, 120, 193)", 1316 | "show": true 1317 | }, 1318 | "tableColumn": "", 1319 | "targets": [ 1320 | { 1321 | "groupBy": [ 1322 | { 1323 | "params": [ 1324 | "$__interval" 1325 | ], 1326 | "type": "time" 1327 | }, 1328 | { 1329 | "params": [ 1330 | "null" 1331 | ], 1332 | "type": "fill" 1333 | } 1334 | ], 1335 | "measurement": "mmConsumerGroupMetrics", 1336 | "orderByTime": "ASC", 1337 | "policy": "default", 1338 | "refId": "A", 1339 | "resultFormat": "time_series", 1340 | "select": [ 1341 | [ 1342 | { 1343 | "params": [ 1344 | "assigned-partitions" 1345 | ], 1346 | "type": "field" 1347 | }, 1348 | { 1349 | "params": [], 1350 | "type": "mean" 1351 | } 1352 | ] 1353 | ], 1354 | "tags": [] 1355 | } 1356 | ], 1357 | "thresholds": "0", 1358 | "timeFrom": null, 1359 | "timeShift": null, 1360 | "title": "Assigned Partitions for $Hostname", 1361 | "type": "singlestat", 1362 | "valueFontSize": "80%", 1363 | "valueMaps": [ 1364 | { 1365 | "op": "=", 1366 | "text": "N/A", 1367 | "value": "null" 1368 | } 1369 | ], 1370 | "valueName": "avg" 1371 | }, 1372 | { 1373 | "aliasColors": {}, 1374 | "bars": false, 1375 | "dashLength": 10, 1376 | "dashes": false, 1377 | "datasource": "influxdb-kafka", 1378 | "description": "The average number of heartbeats per second. After a rebalance, the consumer sends heartbeats to the coordinator to keep itself active in the group. You can control this using the heartbeat.interval.ms setting for the consumer. You may see a lower rate than configured if the processing loop is taking more time to handle message batches. Usually this is OK as long as you see no increase in the join rate.", 1379 | "editable": true, 1380 | "error": false, 1381 | "fill": 1, 1382 | "grid": {}, 1383 | "gridPos": { 1384 | "h": 7, 1385 | "w": 12, 1386 | "x": 12, 1387 | "y": 37 1388 | }, 1389 | "id": 12, 1390 | "isNew": true, 1391 | "legend": { 1392 | "avg": false, 1393 | "current": false, 1394 | "max": false, 1395 | "min": false, 1396 | "show": true, 1397 | "total": false, 1398 | "values": false 1399 | }, 1400 | "lines": true, 1401 | "linewidth": 2, 1402 | "links": [], 1403 | "nullPointMode": "connected", 1404 | "paceLength": 10, 1405 | "percentage": false, 1406 | "pointradius": 5, 1407 | "points": false, 1408 | "renderer": "flot", 1409 | "seriesOverrides": [], 1410 | "spaceLength": 10, 1411 | "stack": false, 1412 | "steppedLine": false, 1413 | "targets": [ 1414 | { 1415 | "alias": "$tag_hostname", 1416 | "dsType": "influxdb", 1417 | "groupBy": [ 1418 | { 1419 | "params": [ 1420 | "$interval" 1421 | ], 1422 | "type": "time" 1423 | }, 1424 | { 1425 | "params": [ 1426 | "hostname" 1427 | ], 1428 | "type": "tag" 1429 | } 1430 | ], 1431 | "measurement": "mmConsumerGroupMetrics", 1432 | "orderByTime": "ASC", 1433 | "policy": "default", 1434 | "refId": "A", 1435 | "resultFormat": "time_series", 1436 | "select": [ 1437 | [ 1438 | { 1439 | "params": [ 1440 | "heartbeat-rate" 1441 | ], 1442 | "type": "field" 1443 | }, 1444 | { 1445 | "params": [], 1446 | "type": "last" 1447 | } 1448 | ] 1449 | ], 1450 | "tags": [] 1451 | } 1452 | ], 1453 | "thresholds": [], 1454 | "timeFrom": null, 1455 | "timeRegions": [], 1456 | "timeShift": null, 1457 | "title": "Heartbeat Rate", 1458 | "tooltip": { 1459 | "msResolution": false, 1460 | "shared": true, 1461 | "sort": 0, 1462 | "value_type": "cumulative" 1463 | }, 1464 | "type": "graph", 1465 | "xaxis": { 1466 | "buckets": null, 1467 | "mode": "time", 1468 | "name": null, 1469 | "show": true, 1470 | "values": [] 1471 | }, 1472 | "yaxes": [ 1473 | { 1474 | "format": "short", 1475 | "label": "heartbeats/s", 1476 | "logBase": 1, 1477 | "max": null, 1478 | "min": null, 1479 | "show": true 1480 | }, 1481 | { 1482 | "format": "short", 1483 | "label": null, 1484 | "logBase": 1, 1485 | "max": null, 1486 | "min": null, 1487 | "show": true 1488 | } 1489 | ], 1490 | "yaxis": { 1491 | "align": false, 1492 | "alignLevel": null 1493 | } 1494 | } 1495 | ], 1496 | "schemaVersion": 18, 1497 | "style": "dark", 1498 | "tags": [], 1499 | "templating": { 1500 | "list": [] 1501 | }, 1502 | "time": { 1503 | "from": "now-1h", 1504 | "to": "now" 1505 | }, 1506 | "timepicker": { 1507 | "refresh_intervals": [ 1508 | "5s", 1509 | "10s", 1510 | "30s", 1511 | "1m", 1512 | "5m", 1513 | "15m", 1514 | "30m", 1515 | "1h", 1516 | "2h", 1517 | "1d" 1518 | ], 1519 | "time_options": [ 1520 | "5m", 1521 | "15m", 1522 | "1h", 1523 | "6h", 1524 | "12h", 1525 | "24h", 1526 | "2d", 1527 | "7d", 1528 | "30d" 1529 | ] 1530 | }, 1531 | "timezone": "browser", 1532 | "title": "Kafka Consumer Metrics (InfluxDB)", 1533 | "uid": "b49SoXjiz", 1534 | "version": 5 1535 | } -------------------------------------------------------------------------------- /dashboards/prometheus-consumer-metrics.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "editable": true, 16 | "gnetId": null, 17 | "graphTooltip": 0, 18 | "id": 1, 19 | "iteration": 1559676738537, 20 | "links": [], 21 | "panels": [ 22 | { 23 | "aliasColors": {}, 24 | "bars": false, 25 | "dashLength": 10, 26 | "dashes": false, 27 | "datasource": "prometheus", 28 | "description": "The maximum lag in terms of number of records for any partition in this window. An increasing value over time is your best indication that the consumer group is not keeping up with the producers.", 29 | "editable": true, 30 | "error": false, 31 | "fill": 1, 32 | "grid": {}, 33 | "gridPos": { 34 | "h": 7, 35 | "w": 12, 36 | "x": 0, 37 | "y": 0 38 | }, 39 | "id": 5, 40 | "isNew": true, 41 | "legend": { 42 | "avg": false, 43 | "current": false, 44 | "hideEmpty": false, 45 | "max": false, 46 | "min": false, 47 | "show": true, 48 | "total": false, 49 | "values": false 50 | }, 51 | "lines": true, 52 | "linewidth": 2, 53 | "links": [], 54 | "nullPointMode": "connected", 55 | "paceLength": 10, 56 | "percentage": false, 57 | "pointradius": 5, 58 | "points": false, 59 | "renderer": "flot", 60 | "seriesOverrides": [], 61 | "spaceLength": 10, 62 | "stack": false, 63 | "steppedLine": false, 64 | "targets": [ 65 | { 66 | "expr": "sum(kafka_consumer_consumer_fetch_manager_metrics_records_lag{name=~\"$App\"}) by (topic)", 67 | "format": "time_series", 68 | "intervalFactor": 1, 69 | "legendFormat": "{{topic}}", 70 | "refId": "A" 71 | } 72 | ], 73 | "thresholds": [], 74 | "timeFrom": null, 75 | "timeRegions": [], 76 | "timeShift": null, 77 | "title": "Max lag for any partition", 78 | "tooltip": { 79 | "msResolution": false, 80 | "shared": true, 81 | "sort": 0, 82 | "value_type": "cumulative" 83 | }, 84 | "type": "graph", 85 | "xaxis": { 86 | "buckets": null, 87 | "mode": "time", 88 | "name": null, 89 | "show": true, 90 | "values": [] 91 | }, 92 | "yaxes": [ 93 | { 94 | "format": "short", 95 | "label": "# records", 96 | "logBase": 1, 97 | "max": null, 98 | "min": null, 99 | "show": true 100 | }, 101 | { 102 | "format": "short", 103 | "label": null, 104 | "logBase": 1, 105 | "max": null, 106 | "min": null, 107 | "show": true 108 | } 109 | ], 110 | "yaxis": { 111 | "align": false, 112 | "alignLevel": null 113 | } 114 | }, 115 | { 116 | "aliasColors": {}, 117 | "bars": false, 118 | "cacheTimeout": null, 119 | "dashLength": 10, 120 | "dashes": false, 121 | "datasource": "prometheus", 122 | "description": "The number of partitions currently assigned to this consumer", 123 | "fill": 1, 124 | "gridPos": { 125 | "h": 7, 126 | "w": 12, 127 | "x": 12, 128 | "y": 0 129 | }, 130 | "id": 14, 131 | "legend": { 132 | "alignAsTable": false, 133 | "avg": false, 134 | "current": false, 135 | "max": false, 136 | "min": false, 137 | "show": true, 138 | "total": false, 139 | "values": false 140 | }, 141 | "lines": true, 142 | "linewidth": 1, 143 | "links": [], 144 | "nullPointMode": "null", 145 | "paceLength": 10, 146 | "percentage": false, 147 | "pointradius": 2, 148 | "points": false, 149 | "renderer": "flot", 150 | "repeat": null, 151 | "repeatDirection": "h", 152 | "seriesOverrides": [], 153 | "spaceLength": 10, 154 | "stack": false, 155 | "steppedLine": false, 156 | "targets": [ 157 | { 158 | "expr": "min(kafka_consumer_consumer_coordinator_metrics_assigned_partitions{name=~\"$App\"}) by (kubernetes_pod_name) ", 159 | "format": "time_series", 160 | "intervalFactor": 1, 161 | "legendFormat": "{{kubernetes_pod_name}}", 162 | "refId": "A" 163 | } 164 | ], 165 | "thresholds": [], 166 | "timeFrom": null, 167 | "timeRegions": [], 168 | "timeShift": null, 169 | "title": "Assigned Partitions", 170 | "tooltip": { 171 | "shared": true, 172 | "sort": 2, 173 | "value_type": "individual" 174 | }, 175 | "type": "graph", 176 | "xaxis": { 177 | "buckets": null, 178 | "mode": "time", 179 | "name": null, 180 | "show": true, 181 | "values": [] 182 | }, 183 | "yaxes": [ 184 | { 185 | "format": "short", 186 | "label": null, 187 | "logBase": 1, 188 | "max": null, 189 | "min": null, 190 | "show": true 191 | }, 192 | { 193 | "format": "short", 194 | "label": null, 195 | "logBase": 1, 196 | "max": null, 197 | "min": null, 198 | "show": true 199 | } 200 | ], 201 | "yaxis": { 202 | "align": false, 203 | "alignLevel": null 204 | } 205 | }, 206 | { 207 | "aliasColors": {}, 208 | "bars": false, 209 | "dashLength": 10, 210 | "dashes": false, 211 | "datasource": "prometheus", 212 | "editable": true, 213 | "error": false, 214 | "fill": 1, 215 | "grid": {}, 216 | "gridPos": { 217 | "h": 7, 218 | "w": 8, 219 | "x": 0, 220 | "y": 7 221 | }, 222 | "id": 2, 223 | "isNew": true, 224 | "legend": { 225 | "avg": false, 226 | "current": false, 227 | "max": false, 228 | "min": false, 229 | "show": true, 230 | "total": false, 231 | "values": false 232 | }, 233 | "lines": true, 234 | "linewidth": 2, 235 | "links": [], 236 | "nullPointMode": "connected", 237 | "paceLength": 10, 238 | "percentage": false, 239 | "pointradius": 5, 240 | "points": false, 241 | "renderer": "flot", 242 | "seriesOverrides": [], 243 | "spaceLength": 10, 244 | "stack": false, 245 | "steppedLine": false, 246 | "targets": [ 247 | { 248 | "expr": "sum(kafka_consumer_consumer_fetch_manager_metrics_bytes_consumed_rate{name=~\"$App\"}) by (kubernetes_pod_name)", 249 | "format": "time_series", 250 | "intervalFactor": 1, 251 | "legendFormat": "{{kubernetes_pod_name}}", 252 | "refId": "A" 253 | } 254 | ], 255 | "thresholds": [], 256 | "timeFrom": null, 257 | "timeRegions": [], 258 | "timeShift": null, 259 | "title": "Bytes Consumed per sec", 260 | "tooltip": { 261 | "msResolution": false, 262 | "shared": true, 263 | "sort": 0, 264 | "value_type": "cumulative" 265 | }, 266 | "type": "graph", 267 | "xaxis": { 268 | "buckets": null, 269 | "mode": "time", 270 | "name": null, 271 | "show": true, 272 | "values": [] 273 | }, 274 | "yaxes": [ 275 | { 276 | "format": "bytes", 277 | "label": null, 278 | "logBase": 1, 279 | "max": null, 280 | "min": null, 281 | "show": true 282 | }, 283 | { 284 | "format": "short", 285 | "label": null, 286 | "logBase": 1, 287 | "max": null, 288 | "min": null, 289 | "show": true 290 | } 291 | ], 292 | "yaxis": { 293 | "align": false, 294 | "alignLevel": null 295 | } 296 | }, 297 | { 298 | "aliasColors": {}, 299 | "bars": false, 300 | "dashLength": 10, 301 | "dashes": false, 302 | "datasource": "prometheus", 303 | "editable": true, 304 | "error": false, 305 | "fill": 1, 306 | "grid": {}, 307 | "gridPos": { 308 | "h": 7, 309 | "w": 8, 310 | "x": 8, 311 | "y": 7 312 | }, 313 | "id": 3, 314 | "isNew": true, 315 | "legend": { 316 | "avg": false, 317 | "current": false, 318 | "max": false, 319 | "min": false, 320 | "show": true, 321 | "total": false, 322 | "values": false 323 | }, 324 | "lines": true, 325 | "linewidth": 2, 326 | "links": [], 327 | "nullPointMode": "connected", 328 | "paceLength": 10, 329 | "percentage": false, 330 | "pointradius": 5, 331 | "points": false, 332 | "renderer": "flot", 333 | "seriesOverrides": [], 334 | "spaceLength": 10, 335 | "stack": false, 336 | "steppedLine": false, 337 | "targets": [ 338 | { 339 | "expr": "avg(kafka_consumer_consumer_fetch_manager_metrics_fetch_latency_avg{name=~\"$App\"}) by (kubernetes_pod_name)", 340 | "format": "time_series", 341 | "intervalFactor": 1, 342 | "legendFormat": "{{kubernetes_pod_name}}", 343 | "refId": "A" 344 | } 345 | ], 346 | "thresholds": [], 347 | "timeFrom": null, 348 | "timeRegions": [], 349 | "timeShift": null, 350 | "title": "Fetch Latency (avg)", 351 | "tooltip": { 352 | "msResolution": false, 353 | "shared": true, 354 | "sort": 0, 355 | "value_type": "cumulative" 356 | }, 357 | "type": "graph", 358 | "xaxis": { 359 | "buckets": null, 360 | "mode": "time", 361 | "name": null, 362 | "show": true, 363 | "values": [] 364 | }, 365 | "yaxes": [ 366 | { 367 | "format": "ms", 368 | "label": null, 369 | "logBase": 1, 370 | "max": null, 371 | "min": null, 372 | "show": true 373 | }, 374 | { 375 | "format": "short", 376 | "label": null, 377 | "logBase": 1, 378 | "max": null, 379 | "min": null, 380 | "show": true 381 | } 382 | ], 383 | "yaxis": { 384 | "align": false, 385 | "alignLevel": null 386 | } 387 | }, 388 | { 389 | "aliasColors": {}, 390 | "bars": false, 391 | "dashLength": 10, 392 | "dashes": false, 393 | "datasource": "prometheus", 394 | "editable": true, 395 | "error": false, 396 | "fill": 1, 397 | "grid": {}, 398 | "gridPos": { 399 | "h": 7, 400 | "w": 8, 401 | "x": 16, 402 | "y": 7 403 | }, 404 | "id": 4, 405 | "isNew": true, 406 | "legend": { 407 | "avg": false, 408 | "current": false, 409 | "max": false, 410 | "min": false, 411 | "show": true, 412 | "total": false, 413 | "values": false 414 | }, 415 | "lines": true, 416 | "linewidth": 2, 417 | "links": [], 418 | "nullPointMode": "connected", 419 | "paceLength": 10, 420 | "percentage": false, 421 | "pointradius": 5, 422 | "points": false, 423 | "renderer": "flot", 424 | "seriesOverrides": [], 425 | "spaceLength": 10, 426 | "stack": false, 427 | "steppedLine": false, 428 | "targets": [ 429 | { 430 | "expr": "sum(kafka_consumer_consumer_fetch_manager_metrics_fetch_rate{name=~\"$App\"}) by (kubernetes_pod_name)", 431 | "format": "time_series", 432 | "intervalFactor": 1, 433 | "legendFormat": "{{kubernetes_pod_name}}", 434 | "refId": "A" 435 | } 436 | ], 437 | "thresholds": [], 438 | "timeFrom": null, 439 | "timeRegions": [], 440 | "timeShift": null, 441 | "title": "Fetch Requests per second", 442 | "tooltip": { 443 | "msResolution": false, 444 | "shared": true, 445 | "sort": 0, 446 | "value_type": "cumulative" 447 | }, 448 | "type": "graph", 449 | "xaxis": { 450 | "buckets": null, 451 | "mode": "time", 452 | "name": null, 453 | "show": true, 454 | "values": [] 455 | }, 456 | "yaxes": [ 457 | { 458 | "format": "short", 459 | "label": "req/s", 460 | "logBase": 1, 461 | "max": null, 462 | "min": null, 463 | "show": true 464 | }, 465 | { 466 | "format": "short", 467 | "label": null, 468 | "logBase": 1, 469 | "max": null, 470 | "min": null, 471 | "show": true 472 | } 473 | ], 474 | "yaxis": { 475 | "align": false, 476 | "alignLevel": null 477 | } 478 | }, 479 | { 480 | "aliasColors": {}, 481 | "bars": false, 482 | "dashLength": 10, 483 | "dashes": false, 484 | "datasource": "prometheus", 485 | "editable": true, 486 | "error": false, 487 | "fill": 1, 488 | "grid": {}, 489 | "gridPos": { 490 | "h": 7, 491 | "w": 12, 492 | "x": 0, 493 | "y": 14 494 | }, 495 | "id": 6, 496 | "isNew": true, 497 | "legend": { 498 | "avg": false, 499 | "current": false, 500 | "max": false, 501 | "min": false, 502 | "show": true, 503 | "total": false, 504 | "values": false 505 | }, 506 | "lines": true, 507 | "linewidth": 2, 508 | "links": [], 509 | "nullPointMode": "connected", 510 | "paceLength": 10, 511 | "percentage": false, 512 | "pointradius": 5, 513 | "points": false, 514 | "renderer": "flot", 515 | "seriesOverrides": [], 516 | "spaceLength": 10, 517 | "stack": false, 518 | "steppedLine": false, 519 | "targets": [ 520 | { 521 | "expr": "avg(kafka_consumer_consumer_fetch_manager_metrics_records_per_request_avg{topic!=\"\", name=~\"$App\"}) by (topic)", 522 | "format": "time_series", 523 | "intervalFactor": 1, 524 | "legendFormat": "{{topic}}", 525 | "refId": "A" 526 | } 527 | ], 528 | "thresholds": [], 529 | "timeFrom": null, 530 | "timeRegions": [], 531 | "timeShift": null, 532 | "title": "Records per request", 533 | "tooltip": { 534 | "msResolution": false, 535 | "shared": true, 536 | "sort": 0, 537 | "value_type": "cumulative" 538 | }, 539 | "type": "graph", 540 | "xaxis": { 541 | "buckets": null, 542 | "mode": "time", 543 | "name": null, 544 | "show": true, 545 | "values": [] 546 | }, 547 | "yaxes": [ 548 | { 549 | "format": "short", 550 | "label": "rec/req", 551 | "logBase": 1, 552 | "max": null, 553 | "min": null, 554 | "show": true 555 | }, 556 | { 557 | "format": "short", 558 | "label": null, 559 | "logBase": 1, 560 | "max": null, 561 | "min": null, 562 | "show": true 563 | } 564 | ], 565 | "yaxis": { 566 | "align": false, 567 | "alignLevel": null 568 | } 569 | }, 570 | { 571 | "aliasColors": {}, 572 | "bars": false, 573 | "dashLength": 10, 574 | "dashes": false, 575 | "datasource": "prometheus", 576 | "description": "The average number of records per second", 577 | "editable": true, 578 | "error": false, 579 | "fill": 1, 580 | "grid": {}, 581 | "gridPos": { 582 | "h": 7, 583 | "w": 12, 584 | "x": 12, 585 | "y": 14 586 | }, 587 | "id": 7, 588 | "isNew": true, 589 | "legend": { 590 | "avg": false, 591 | "current": false, 592 | "max": false, 593 | "min": false, 594 | "show": true, 595 | "total": false, 596 | "values": false 597 | }, 598 | "lines": true, 599 | "linewidth": 2, 600 | "links": [], 601 | "nullPointMode": "connected", 602 | "paceLength": 10, 603 | "percentage": false, 604 | "pointradius": 5, 605 | "points": false, 606 | "renderer": "flot", 607 | "seriesOverrides": [], 608 | "spaceLength": 10, 609 | "stack": false, 610 | "steppedLine": false, 611 | "targets": [ 612 | { 613 | "expr": "sum(rate(kafka_consumer_consumer_fetch_manager_metrics_records_consumed_rate{topic!=\"\", name=~\"$App\"}[5m])) by (topic)", 614 | "format": "time_series", 615 | "intervalFactor": 1, 616 | "legendFormat": "{{topic}}", 617 | "refId": "A" 618 | } 619 | ], 620 | "thresholds": [], 621 | "timeFrom": null, 622 | "timeRegions": [], 623 | "timeShift": null, 624 | "title": "Records per second", 625 | "tooltip": { 626 | "msResolution": false, 627 | "shared": true, 628 | "sort": 0, 629 | "value_type": "cumulative" 630 | }, 631 | "type": "graph", 632 | "xaxis": { 633 | "buckets": null, 634 | "mode": "time", 635 | "name": null, 636 | "show": true, 637 | "values": [] 638 | }, 639 | "yaxes": [ 640 | { 641 | "format": "short", 642 | "label": "rec/s", 643 | "logBase": 1, 644 | "max": null, 645 | "min": null, 646 | "show": true 647 | }, 648 | { 649 | "format": "short", 650 | "label": null, 651 | "logBase": 1, 652 | "max": null, 653 | "min": null, 654 | "show": true 655 | } 656 | ], 657 | "yaxis": { 658 | "align": false, 659 | "alignLevel": null 660 | } 661 | }, 662 | { 663 | "aliasColors": {}, 664 | "bars": false, 665 | "dashLength": 10, 666 | "dashes": false, 667 | "datasource": "prometheus", 668 | "description": "The average time taken for a commit request.", 669 | "editable": true, 670 | "error": false, 671 | "fill": 1, 672 | "grid": {}, 673 | "gridPos": { 674 | "h": 7, 675 | "w": 12, 676 | "x": 0, 677 | "y": 21 678 | }, 679 | "id": 9, 680 | "isNew": true, 681 | "legend": { 682 | "avg": false, 683 | "current": false, 684 | "max": false, 685 | "min": false, 686 | "show": true, 687 | "total": false, 688 | "values": false 689 | }, 690 | "lines": true, 691 | "linewidth": 2, 692 | "links": [], 693 | "nullPointMode": "connected", 694 | "paceLength": 10, 695 | "percentage": false, 696 | "pointradius": 5, 697 | "points": false, 698 | "renderer": "flot", 699 | "seriesOverrides": [], 700 | "spaceLength": 10, 701 | "stack": false, 702 | "steppedLine": false, 703 | "targets": [ 704 | { 705 | "expr": "avg(kafka_consumer_consumer_coordinator_metrics_commit_latency_avg{name=~\"$App\"}) by (kubernetes_pod_name)", 706 | "format": "time_series", 707 | "intervalFactor": 1, 708 | "legendFormat": "{{kubernetes_pod_name}}", 709 | "refId": "A" 710 | } 711 | ], 712 | "thresholds": [], 713 | "timeFrom": null, 714 | "timeRegions": [], 715 | "timeShift": null, 716 | "title": "Commit Latency (avg)", 717 | "tooltip": { 718 | "msResolution": false, 719 | "shared": true, 720 | "sort": 0, 721 | "value_type": "cumulative" 722 | }, 723 | "type": "graph", 724 | "xaxis": { 725 | "buckets": null, 726 | "mode": "time", 727 | "name": null, 728 | "show": true, 729 | "values": [] 730 | }, 731 | "yaxes": [ 732 | { 733 | "format": "ms", 734 | "label": "", 735 | "logBase": 1, 736 | "max": null, 737 | "min": null, 738 | "show": true 739 | }, 740 | { 741 | "format": "short", 742 | "label": null, 743 | "logBase": 1, 744 | "max": null, 745 | "min": null, 746 | "show": true 747 | } 748 | ], 749 | "yaxis": { 750 | "align": false, 751 | "alignLevel": null 752 | } 753 | }, 754 | { 755 | "aliasColors": {}, 756 | "bars": false, 757 | "dashLength": 10, 758 | "dashes": false, 759 | "datasource": "prometheus", 760 | "description": "The number of commit calls per second.", 761 | "editable": true, 762 | "error": false, 763 | "fill": 1, 764 | "grid": {}, 765 | "gridPos": { 766 | "h": 7, 767 | "w": 12, 768 | "x": 12, 769 | "y": 21 770 | }, 771 | "id": 8, 772 | "isNew": true, 773 | "legend": { 774 | "avg": false, 775 | "current": false, 776 | "max": false, 777 | "min": false, 778 | "show": true, 779 | "total": false, 780 | "values": false 781 | }, 782 | "lines": true, 783 | "linewidth": 2, 784 | "links": [], 785 | "nullPointMode": "connected", 786 | "paceLength": 10, 787 | "percentage": false, 788 | "pointradius": 5, 789 | "points": false, 790 | "renderer": "flot", 791 | "seriesOverrides": [], 792 | "spaceLength": 10, 793 | "stack": false, 794 | "steppedLine": false, 795 | "targets": [ 796 | { 797 | "expr": "avg(kafka_consumer_consumer_coordinator_metrics_commit_rate{name=~\"$App\"}) by (kubernetes_pod_name)", 798 | "format": "time_series", 799 | "intervalFactor": 1, 800 | "legendFormat": "{{kubernetes_pod_name}}", 801 | "refId": "A" 802 | } 803 | ], 804 | "thresholds": [], 805 | "timeFrom": null, 806 | "timeRegions": [], 807 | "timeShift": null, 808 | "title": "Commits per second", 809 | "tooltip": { 810 | "msResolution": false, 811 | "shared": true, 812 | "sort": 0, 813 | "value_type": "cumulative" 814 | }, 815 | "type": "graph", 816 | "xaxis": { 817 | "buckets": null, 818 | "mode": "time", 819 | "name": null, 820 | "show": true, 821 | "values": [] 822 | }, 823 | "yaxes": [ 824 | { 825 | "format": "short", 826 | "label": "commit/s", 827 | "logBase": 1, 828 | "max": null, 829 | "min": null, 830 | "show": true 831 | }, 832 | { 833 | "format": "short", 834 | "label": null, 835 | "logBase": 1, 836 | "max": null, 837 | "min": null, 838 | "show": true 839 | } 840 | ], 841 | "yaxis": { 842 | "align": false, 843 | "alignLevel": null 844 | } 845 | }, 846 | { 847 | "aliasColors": {}, 848 | "bars": false, 849 | "dashLength": 10, 850 | "dashes": false, 851 | "datasource": "prometheus", 852 | "description": "The average length of time for I/O per select call in nanoseconds.", 853 | "fill": 1, 854 | "gridPos": { 855 | "h": 8, 856 | "w": 12, 857 | "x": 0, 858 | "y": 28 859 | }, 860 | "id": 16, 861 | "legend": { 862 | "avg": false, 863 | "current": false, 864 | "max": false, 865 | "min": false, 866 | "show": true, 867 | "total": false, 868 | "values": false 869 | }, 870 | "lines": true, 871 | "linewidth": 1, 872 | "links": [], 873 | "nullPointMode": "null", 874 | "paceLength": 10, 875 | "percentage": false, 876 | "pointradius": 2, 877 | "points": false, 878 | "renderer": "flot", 879 | "seriesOverrides": [], 880 | "spaceLength": 10, 881 | "stack": false, 882 | "steppedLine": false, 883 | "targets": [ 884 | { 885 | "expr": "avg(kafka_consumer_consumer_metrics_io_time_ns_avg{name=~\"$App\"}) by (kubernetes_pod_name)", 886 | "format": "time_series", 887 | "intervalFactor": 1, 888 | "legendFormat": "{{kubernetes_pod_name}}", 889 | "refId": "A" 890 | } 891 | ], 892 | "thresholds": [], 893 | "timeFrom": null, 894 | "timeRegions": [], 895 | "timeShift": null, 896 | "title": "IO Time (avg)", 897 | "tooltip": { 898 | "shared": true, 899 | "sort": 0, 900 | "value_type": "individual" 901 | }, 902 | "type": "graph", 903 | "xaxis": { 904 | "buckets": null, 905 | "mode": "time", 906 | "name": null, 907 | "show": true, 908 | "values": [] 909 | }, 910 | "yaxes": [ 911 | { 912 | "format": "ns", 913 | "label": null, 914 | "logBase": 1, 915 | "max": null, 916 | "min": null, 917 | "show": true 918 | }, 919 | { 920 | "format": "short", 921 | "label": null, 922 | "logBase": 1, 923 | "max": null, 924 | "min": null, 925 | "show": true 926 | } 927 | ], 928 | "yaxis": { 929 | "align": false, 930 | "alignLevel": null 931 | } 932 | }, 933 | { 934 | "aliasColors": {}, 935 | "bars": false, 936 | "dashLength": 10, 937 | "dashes": false, 938 | "datasource": "prometheus", 939 | "description": "The average length of time for I/O per select call in nanoseconds.", 940 | "fill": 1, 941 | "gridPos": { 942 | "h": 8, 943 | "w": 12, 944 | "x": 12, 945 | "y": 28 946 | }, 947 | "id": 18, 948 | "legend": { 949 | "avg": false, 950 | "current": false, 951 | "max": false, 952 | "min": false, 953 | "show": true, 954 | "total": false, 955 | "values": false 956 | }, 957 | "lines": true, 958 | "linewidth": 1, 959 | "links": [], 960 | "nullPointMode": "null", 961 | "paceLength": 10, 962 | "percentage": false, 963 | "pointradius": 2, 964 | "points": false, 965 | "renderer": "flot", 966 | "seriesOverrides": [], 967 | "spaceLength": 10, 968 | "stack": false, 969 | "steppedLine": false, 970 | "targets": [ 971 | { 972 | "expr": "avg(kafka_consumer_consumer_metrics_io_time_ns_avg{name=~\"$App\"}) by (kubernetes_pod_name)", 973 | "format": "time_series", 974 | "intervalFactor": 1, 975 | "legendFormat": "{{kubernetes_pod_name}}", 976 | "refId": "A" 977 | } 978 | ], 979 | "thresholds": [], 980 | "timeFrom": null, 981 | "timeRegions": [], 982 | "timeShift": null, 983 | "title": "IO Time (avg)", 984 | "tooltip": { 985 | "shared": true, 986 | "sort": 0, 987 | "value_type": "individual" 988 | }, 989 | "type": "graph", 990 | "xaxis": { 991 | "buckets": null, 992 | "mode": "time", 993 | "name": null, 994 | "show": true, 995 | "values": [] 996 | }, 997 | "yaxes": [ 998 | { 999 | "format": "ns", 1000 | "label": null, 1001 | "logBase": 1, 1002 | "max": null, 1003 | "min": null, 1004 | "show": true 1005 | }, 1006 | { 1007 | "format": "short", 1008 | "label": null, 1009 | "logBase": 1, 1010 | "max": null, 1011 | "min": null, 1012 | "show": true 1013 | } 1014 | ], 1015 | "yaxis": { 1016 | "align": false, 1017 | "alignLevel": null 1018 | } 1019 | }, 1020 | { 1021 | "aliasColors": {}, 1022 | "bars": false, 1023 | "dashLength": 10, 1024 | "dashes": false, 1025 | "datasource": "prometheus", 1026 | "description": "The fraction of time the I/O thread spent doing I/O.", 1027 | "fill": 1, 1028 | "gridPos": { 1029 | "h": 8, 1030 | "w": 12, 1031 | "x": 0, 1032 | "y": 36 1033 | }, 1034 | "id": 17, 1035 | "legend": { 1036 | "avg": false, 1037 | "current": false, 1038 | "max": false, 1039 | "min": false, 1040 | "show": true, 1041 | "total": false, 1042 | "values": false 1043 | }, 1044 | "lines": true, 1045 | "linewidth": 1, 1046 | "links": [], 1047 | "nullPointMode": "null", 1048 | "paceLength": 10, 1049 | "percentage": false, 1050 | "pointradius": 2, 1051 | "points": false, 1052 | "renderer": "flot", 1053 | "seriesOverrides": [], 1054 | "spaceLength": 10, 1055 | "stack": false, 1056 | "steppedLine": false, 1057 | "targets": [ 1058 | { 1059 | "expr": "avg(kafka_consumer_consumer_metrics_io_ratio{name=~\"$App\"}) by (kubernetes_pod_name)", 1060 | "format": "time_series", 1061 | "intervalFactor": 1, 1062 | "legendFormat": "{{kubernetes_pod_name}}", 1063 | "refId": "A" 1064 | } 1065 | ], 1066 | "thresholds": [], 1067 | "timeFrom": null, 1068 | "timeRegions": [], 1069 | "timeShift": null, 1070 | "title": "IO Ratio (avg)", 1071 | "tooltip": { 1072 | "shared": true, 1073 | "sort": 0, 1074 | "value_type": "individual" 1075 | }, 1076 | "type": "graph", 1077 | "xaxis": { 1078 | "buckets": null, 1079 | "mode": "time", 1080 | "name": null, 1081 | "show": true, 1082 | "values": [] 1083 | }, 1084 | "yaxes": [ 1085 | { 1086 | "format": "s", 1087 | "label": null, 1088 | "logBase": 1, 1089 | "max": null, 1090 | "min": null, 1091 | "show": true 1092 | }, 1093 | { 1094 | "format": "short", 1095 | "label": null, 1096 | "logBase": 1, 1097 | "max": null, 1098 | "min": null, 1099 | "show": true 1100 | } 1101 | ], 1102 | "yaxis": { 1103 | "align": false, 1104 | "alignLevel": null 1105 | } 1106 | }, 1107 | { 1108 | "aliasColors": {}, 1109 | "bars": false, 1110 | "dashLength": 10, 1111 | "dashes": false, 1112 | "datasource": "prometheus", 1113 | "description": "The fraction of time the I/O thread spent waiting.", 1114 | "fill": 1, 1115 | "gridPos": { 1116 | "h": 8, 1117 | "w": 12, 1118 | "x": 12, 1119 | "y": 36 1120 | }, 1121 | "id": 19, 1122 | "legend": { 1123 | "avg": false, 1124 | "current": false, 1125 | "max": false, 1126 | "min": false, 1127 | "show": true, 1128 | "total": false, 1129 | "values": false 1130 | }, 1131 | "lines": true, 1132 | "linewidth": 1, 1133 | "links": [], 1134 | "nullPointMode": "null", 1135 | "paceLength": 10, 1136 | "percentage": false, 1137 | "pointradius": 2, 1138 | "points": false, 1139 | "renderer": "flot", 1140 | "seriesOverrides": [], 1141 | "spaceLength": 10, 1142 | "stack": false, 1143 | "steppedLine": false, 1144 | "targets": [ 1145 | { 1146 | "expr": "avg(kafka_consumer_consumer_metrics_io_wait_ratio{name=~\"$App\"}) by (kubernetes_pod_name)", 1147 | "format": "time_series", 1148 | "intervalFactor": 1, 1149 | "legendFormat": "{{kubernetes_pod_name}}", 1150 | "refId": "A" 1151 | } 1152 | ], 1153 | "thresholds": [], 1154 | "timeFrom": null, 1155 | "timeRegions": [], 1156 | "timeShift": null, 1157 | "title": "IO Wait Ratio (avg)", 1158 | "tooltip": { 1159 | "shared": true, 1160 | "sort": 0, 1161 | "value_type": "individual" 1162 | }, 1163 | "type": "graph", 1164 | "xaxis": { 1165 | "buckets": null, 1166 | "mode": "time", 1167 | "name": null, 1168 | "show": true, 1169 | "values": [] 1170 | }, 1171 | "yaxes": [ 1172 | { 1173 | "format": "s", 1174 | "label": null, 1175 | "logBase": 1, 1176 | "max": null, 1177 | "min": null, 1178 | "show": true 1179 | }, 1180 | { 1181 | "format": "short", 1182 | "label": null, 1183 | "logBase": 1, 1184 | "max": null, 1185 | "min": null, 1186 | "show": true 1187 | } 1188 | ], 1189 | "yaxis": { 1190 | "align": false, 1191 | "alignLevel": null 1192 | } 1193 | }, 1194 | { 1195 | "aliasColors": {}, 1196 | "bars": false, 1197 | "dashLength": 10, 1198 | "dashes": false, 1199 | "datasource": "prometheus", 1200 | "description": "The average number of heartbeats per second. After a rebalance, the consumer sends heartbeats to the coordinator to keep itself active in the group. You can control this using the heartbeat.interval.ms setting for the consumer. You may see a lower rate than configured if the processing loop is taking more time to handle message batches. Usually this is OK as long as you see no increase in the join rate.", 1201 | "editable": true, 1202 | "error": false, 1203 | "fill": 1, 1204 | "grid": {}, 1205 | "gridPos": { 1206 | "h": 6, 1207 | "w": 24, 1208 | "x": 0, 1209 | "y": 44 1210 | }, 1211 | "id": 12, 1212 | "isNew": true, 1213 | "legend": { 1214 | "avg": false, 1215 | "current": false, 1216 | "max": false, 1217 | "min": false, 1218 | "show": true, 1219 | "total": false, 1220 | "values": false 1221 | }, 1222 | "lines": true, 1223 | "linewidth": 2, 1224 | "links": [], 1225 | "nullPointMode": "connected", 1226 | "paceLength": 10, 1227 | "percentage": false, 1228 | "pointradius": 5, 1229 | "points": false, 1230 | "renderer": "flot", 1231 | "seriesOverrides": [], 1232 | "spaceLength": 10, 1233 | "stack": false, 1234 | "steppedLine": false, 1235 | "targets": [ 1236 | { 1237 | "expr": "avg(kafka_consumer_consumer_coordinator_metrics_heartbeat_rate{name=~\"$App\"}) by (kubernetes_pod_name)", 1238 | "format": "time_series", 1239 | "intervalFactor": 1, 1240 | "legendFormat": "{{kubernetes_pod_name}}", 1241 | "refId": "A" 1242 | } 1243 | ], 1244 | "thresholds": [], 1245 | "timeFrom": null, 1246 | "timeRegions": [], 1247 | "timeShift": null, 1248 | "title": "Heartbeat Rate", 1249 | "tooltip": { 1250 | "msResolution": false, 1251 | "shared": true, 1252 | "sort": 0, 1253 | "value_type": "cumulative" 1254 | }, 1255 | "type": "graph", 1256 | "xaxis": { 1257 | "buckets": null, 1258 | "mode": "time", 1259 | "name": null, 1260 | "show": true, 1261 | "values": [] 1262 | }, 1263 | "yaxes": [ 1264 | { 1265 | "format": "short", 1266 | "label": "heartbeats/s", 1267 | "logBase": 1, 1268 | "max": null, 1269 | "min": null, 1270 | "show": true 1271 | }, 1272 | { 1273 | "format": "short", 1274 | "label": null, 1275 | "logBase": 1, 1276 | "max": null, 1277 | "min": null, 1278 | "show": true 1279 | } 1280 | ], 1281 | "yaxis": { 1282 | "align": false, 1283 | "alignLevel": null 1284 | } 1285 | }, 1286 | { 1287 | "aliasColors": {}, 1288 | "bars": false, 1289 | "dashLength": 10, 1290 | "dashes": false, 1291 | "datasource": "prometheus", 1292 | "description": "The number of group syncs per second. Group synchronization is the second and last phase of the rebalance protocol. Similar to join-rate, a large value indicates group instability.", 1293 | "editable": true, 1294 | "error": false, 1295 | "fill": 1, 1296 | "grid": {}, 1297 | "gridPos": { 1298 | "h": 7, 1299 | "w": 12, 1300 | "x": 0, 1301 | "y": 50 1302 | }, 1303 | "id": 10, 1304 | "isNew": true, 1305 | "legend": { 1306 | "avg": false, 1307 | "current": false, 1308 | "max": false, 1309 | "min": false, 1310 | "show": true, 1311 | "total": false, 1312 | "values": false 1313 | }, 1314 | "lines": true, 1315 | "linewidth": 2, 1316 | "links": [], 1317 | "nullPointMode": "connected", 1318 | "paceLength": 10, 1319 | "percentage": false, 1320 | "pointradius": 5, 1321 | "points": false, 1322 | "renderer": "flot", 1323 | "seriesOverrides": [], 1324 | "spaceLength": 10, 1325 | "stack": false, 1326 | "steppedLine": false, 1327 | "targets": [ 1328 | { 1329 | "expr": "avg(kafka_consumer_consumer_coordinator_metrics_sync_rate{name=~\"$App\"}) by (kubernetes_pod_name)", 1330 | "format": "time_series", 1331 | "intervalFactor": 1, 1332 | "legendFormat": "{{kubernetes_pod_name}}", 1333 | "refId": "A" 1334 | } 1335 | ], 1336 | "thresholds": [], 1337 | "timeFrom": null, 1338 | "timeRegions": [], 1339 | "timeShift": null, 1340 | "title": "Sync Rate", 1341 | "tooltip": { 1342 | "msResolution": false, 1343 | "shared": true, 1344 | "sort": 0, 1345 | "value_type": "cumulative" 1346 | }, 1347 | "type": "graph", 1348 | "xaxis": { 1349 | "buckets": null, 1350 | "mode": "time", 1351 | "name": null, 1352 | "show": true, 1353 | "values": [] 1354 | }, 1355 | "yaxes": [ 1356 | { 1357 | "format": "short", 1358 | "label": "syncs/s", 1359 | "logBase": 1, 1360 | "max": null, 1361 | "min": null, 1362 | "show": true 1363 | }, 1364 | { 1365 | "format": "short", 1366 | "label": null, 1367 | "logBase": 1, 1368 | "max": null, 1369 | "min": null, 1370 | "show": true 1371 | } 1372 | ], 1373 | "yaxis": { 1374 | "align": false, 1375 | "alignLevel": null 1376 | } 1377 | }, 1378 | { 1379 | "aliasColors": {}, 1380 | "bars": false, 1381 | "dashLength": 10, 1382 | "dashes": false, 1383 | "datasource": "prometheus", 1384 | "description": "The number of group joins per second. Group joining is the first phase of the rebalance protocol. A large value indicates that the consumer group is unstable and will likely be coupled with increased lag.", 1385 | "editable": true, 1386 | "error": false, 1387 | "fill": 1, 1388 | "grid": {}, 1389 | "gridPos": { 1390 | "h": 7, 1391 | "w": 12, 1392 | "x": 12, 1393 | "y": 50 1394 | }, 1395 | "id": 11, 1396 | "isNew": true, 1397 | "legend": { 1398 | "avg": false, 1399 | "current": false, 1400 | "max": false, 1401 | "min": false, 1402 | "show": true, 1403 | "total": false, 1404 | "values": false 1405 | }, 1406 | "lines": true, 1407 | "linewidth": 2, 1408 | "links": [], 1409 | "nullPointMode": "connected", 1410 | "paceLength": 10, 1411 | "percentage": false, 1412 | "pointradius": 5, 1413 | "points": false, 1414 | "renderer": "flot", 1415 | "seriesOverrides": [], 1416 | "spaceLength": 10, 1417 | "stack": false, 1418 | "steppedLine": false, 1419 | "targets": [ 1420 | { 1421 | "expr": "avg(kafka_consumer_consumer_coordinator_metrics_join_rate{name=~\"$App\"}) by (kubernetes_pod_name)", 1422 | "format": "time_series", 1423 | "intervalFactor": 1, 1424 | "legendFormat": "{{kubernetes_pod_name}}", 1425 | "refId": "A" 1426 | } 1427 | ], 1428 | "thresholds": [], 1429 | "timeFrom": null, 1430 | "timeRegions": [], 1431 | "timeShift": null, 1432 | "title": "Join Rate", 1433 | "tooltip": { 1434 | "msResolution": false, 1435 | "shared": true, 1436 | "sort": 0, 1437 | "value_type": "cumulative" 1438 | }, 1439 | "type": "graph", 1440 | "xaxis": { 1441 | "buckets": null, 1442 | "mode": "time", 1443 | "name": null, 1444 | "show": true, 1445 | "values": [] 1446 | }, 1447 | "yaxes": [ 1448 | { 1449 | "format": "short", 1450 | "label": "joins/s", 1451 | "logBase": 1, 1452 | "max": null, 1453 | "min": null, 1454 | "show": true 1455 | }, 1456 | { 1457 | "format": "short", 1458 | "label": null, 1459 | "logBase": 1, 1460 | "max": null, 1461 | "min": null, 1462 | "show": true 1463 | } 1464 | ], 1465 | "yaxis": { 1466 | "align": false, 1467 | "alignLevel": null 1468 | } 1469 | } 1470 | ], 1471 | "refresh": "10s", 1472 | "schemaVersion": 18, 1473 | "style": "dark", 1474 | "tags": [], 1475 | "templating": { 1476 | "list": [ 1477 | { 1478 | "allValue": null, 1479 | "current": { 1480 | "text": "All", 1481 | "value": [ 1482 | "$__all" 1483 | ] 1484 | }, 1485 | "datasource": "prometheus", 1486 | "definition": "label_values(kafka_consumer_kafka_metrics_count_count, name)", 1487 | "hide": 0, 1488 | "includeAll": true, 1489 | "label": null, 1490 | "multi": true, 1491 | "name": "App", 1492 | "options": [], 1493 | "query": "label_values(kafka_consumer_kafka_metrics_count_count, name)", 1494 | "refresh": 1, 1495 | "regex": "", 1496 | "skipUrlSync": false, 1497 | "sort": 0, 1498 | "tagValuesQuery": "", 1499 | "tags": [], 1500 | "tagsQuery": "", 1501 | "type": "query", 1502 | "useTags": false 1503 | } 1504 | ] 1505 | }, 1506 | "time": { 1507 | "from": "now-3h", 1508 | "to": "now" 1509 | }, 1510 | "timepicker": { 1511 | "refresh_intervals": [ 1512 | "5s", 1513 | "10s", 1514 | "30s", 1515 | "1m", 1516 | "5m", 1517 | "15m", 1518 | "30m", 1519 | "1h", 1520 | "2h", 1521 | "1d" 1522 | ], 1523 | "time_options": [ 1524 | "5m", 1525 | "15m", 1526 | "1h", 1527 | "6h", 1528 | "12h", 1529 | "24h", 1530 | "2d", 1531 | "7d", 1532 | "30d" 1533 | ] 1534 | }, 1535 | "timezone": "browser", 1536 | "title": "Kafka Consumer Metrics (Prometheus)", 1537 | "uid": "b49SoXjizw43", 1538 | "version": 2 1539 | } -------------------------------------------------------------------------------- /dashboards/influxdb-kafka-broker-metrics.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "editable": true, 16 | "gnetId": null, 17 | "graphTooltip": 0, 18 | "id": 1, 19 | "links": [], 20 | "panels": [ 21 | { 22 | "aliasColors": {}, 23 | "bars": false, 24 | "dashLength": 10, 25 | "dashes": false, 26 | "datasource": "influxdb-kafka", 27 | "fill": 1, 28 | "gridPos": { 29 | "h": 10, 30 | "w": 24, 31 | "x": 0, 32 | "y": 0 33 | }, 34 | "id": 24, 35 | "legend": { 36 | "avg": false, 37 | "current": false, 38 | "max": false, 39 | "min": false, 40 | "show": true, 41 | "total": false, 42 | "values": false 43 | }, 44 | "lines": true, 45 | "linewidth": 1, 46 | "links": [], 47 | "nullPointMode": "null", 48 | "paceLength": 10, 49 | "percentage": false, 50 | "pointradius": 5, 51 | "points": false, 52 | "renderer": "flot", 53 | "seriesOverrides": [], 54 | "spaceLength": 10, 55 | "stack": false, 56 | "steppedLine": false, 57 | "targets": [ 58 | { 59 | "alias": "$tag_typeName", 60 | "groupBy": [ 61 | { 62 | "params": [ 63 | "typeName" 64 | ], 65 | "type": "tag" 66 | } 67 | ], 68 | "measurement": "MessagesInPerSecPerTopic", 69 | "orderByTime": "ASC", 70 | "policy": "default", 71 | "refId": "A", 72 | "resultFormat": "time_series", 73 | "select": [ 74 | [ 75 | { 76 | "params": [ 77 | "OneMinuteRate" 78 | ], 79 | "type": "field" 80 | } 81 | ] 82 | ], 83 | "tags": [] 84 | } 85 | ], 86 | "thresholds": [], 87 | "timeFrom": null, 88 | "timeRegions": [], 89 | "timeShift": null, 90 | "title": "Messages per topic", 91 | "tooltip": { 92 | "shared": true, 93 | "sort": 0, 94 | "value_type": "individual" 95 | }, 96 | "type": "graph", 97 | "xaxis": { 98 | "buckets": null, 99 | "mode": "time", 100 | "name": null, 101 | "show": true, 102 | "values": [] 103 | }, 104 | "yaxes": [ 105 | { 106 | "format": "short", 107 | "label": null, 108 | "logBase": 1, 109 | "max": null, 110 | "min": null, 111 | "show": true 112 | }, 113 | { 114 | "format": "short", 115 | "label": null, 116 | "logBase": 1, 117 | "max": null, 118 | "min": null, 119 | "show": true 120 | } 121 | ], 122 | "yaxis": { 123 | "align": false, 124 | "alignLevel": null 125 | } 126 | }, 127 | { 128 | "collapsed": false, 129 | "gridPos": { 130 | "h": 1, 131 | "w": 24, 132 | "x": 0, 133 | "y": 10 134 | }, 135 | "id": 19, 136 | "panels": [], 137 | "title": "Critical", 138 | "type": "row" 139 | }, 140 | { 141 | "alert": { 142 | "conditions": [ 143 | { 144 | "evaluator": { 145 | "params": [ 146 | 0 147 | ], 148 | "type": "gt" 149 | }, 150 | "operator": { 151 | "type": "and" 152 | }, 153 | "query": { 154 | "params": [ 155 | "A", 156 | "5m", 157 | "now" 158 | ] 159 | }, 160 | "reducer": { 161 | "params": [], 162 | "type": "sum" 163 | }, 164 | "type": "query" 165 | } 166 | ], 167 | "executionErrorState": "alerting", 168 | "for": "20s", 169 | "frequency": "5s", 170 | "handler": 1, 171 | "name": "Under Replicated Partitions alert", 172 | "noDataState": "no_data", 173 | "notifications": [] 174 | }, 175 | "aliasColors": {}, 176 | "bars": false, 177 | "dashLength": 10, 178 | "dashes": false, 179 | "datasource": "influxdb-kafka", 180 | "editable": true, 181 | "error": false, 182 | "fill": 1, 183 | "grid": {}, 184 | "gridPos": { 185 | "h": 7, 186 | "w": 12, 187 | "x": 0, 188 | "y": 11 189 | }, 190 | "id": 4, 191 | "isNew": true, 192 | "legend": { 193 | "avg": false, 194 | "current": false, 195 | "max": false, 196 | "min": false, 197 | "show": true, 198 | "total": false, 199 | "values": false 200 | }, 201 | "lines": true, 202 | "linewidth": 2, 203 | "links": [], 204 | "nullPointMode": "connected", 205 | "paceLength": 10, 206 | "percentage": false, 207 | "pointradius": 5, 208 | "points": false, 209 | "renderer": "flot", 210 | "seriesOverrides": [], 211 | "spaceLength": 10, 212 | "stack": false, 213 | "steppedLine": false, 214 | "targets": [ 215 | { 216 | "dsType": "influxdb", 217 | "groupBy": [ 218 | { 219 | "params": [ 220 | "$interval" 221 | ], 222 | "type": "time" 223 | }, 224 | { 225 | "params": [ 226 | "null" 227 | ], 228 | "type": "fill" 229 | } 230 | ], 231 | "measurement": "UnderReplicatedPartitions", 232 | "orderByTime": "ASC", 233 | "policy": "default", 234 | "refId": "A", 235 | "resultFormat": "time_series", 236 | "select": [ 237 | [ 238 | { 239 | "params": [ 240 | "Value" 241 | ], 242 | "type": "field" 243 | }, 244 | { 245 | "params": [], 246 | "type": "sum" 247 | } 248 | ] 249 | ], 250 | "tags": [] 251 | } 252 | ], 253 | "thresholds": [ 254 | { 255 | "colorMode": "critical", 256 | "fill": true, 257 | "line": true, 258 | "op": "gt", 259 | "value": 0 260 | } 261 | ], 262 | "timeFrom": null, 263 | "timeRegions": [], 264 | "timeShift": null, 265 | "title": "Under Replicated Partitions", 266 | "tooltip": { 267 | "msResolution": false, 268 | "shared": true, 269 | "sort": 0, 270 | "value_type": "cumulative" 271 | }, 272 | "type": "graph", 273 | "xaxis": { 274 | "buckets": null, 275 | "mode": "time", 276 | "name": null, 277 | "show": true, 278 | "values": [] 279 | }, 280 | "yaxes": [ 281 | { 282 | "format": "short", 283 | "label": null, 284 | "logBase": 1, 285 | "max": null, 286 | "min": null, 287 | "show": true 288 | }, 289 | { 290 | "format": "short", 291 | "label": null, 292 | "logBase": 1, 293 | "max": null, 294 | "min": null, 295 | "show": true 296 | } 297 | ], 298 | "yaxis": { 299 | "align": false, 300 | "alignLevel": null 301 | } 302 | }, 303 | { 304 | "alert": { 305 | "conditions": [ 306 | { 307 | "evaluator": { 308 | "params": [ 309 | 0 310 | ], 311 | "type": "gt" 312 | }, 313 | "operator": { 314 | "type": "and" 315 | }, 316 | "query": { 317 | "params": [ 318 | "C", 319 | "5m", 320 | "now" 321 | ] 322 | }, 323 | "reducer": { 324 | "params": [], 325 | "type": "sum" 326 | }, 327 | "type": "query" 328 | } 329 | ], 330 | "executionErrorState": "alerting", 331 | "for": "30s", 332 | "frequency": "10s", 333 | "handler": 1, 334 | "name": "Offline Partitions Count > 0", 335 | "noDataState": "no_data", 336 | "notifications": [] 337 | }, 338 | "aliasColors": {}, 339 | "bars": false, 340 | "dashLength": 10, 341 | "dashes": false, 342 | "datasource": "influxdb-kafka", 343 | "editable": true, 344 | "error": false, 345 | "fill": 1, 346 | "grid": {}, 347 | "gridPos": { 348 | "h": 7, 349 | "w": 12, 350 | "x": 12, 351 | "y": 11 352 | }, 353 | "id": 8, 354 | "isNew": true, 355 | "legend": { 356 | "avg": false, 357 | "current": false, 358 | "max": false, 359 | "min": false, 360 | "show": true, 361 | "total": false, 362 | "values": false 363 | }, 364 | "lines": true, 365 | "linewidth": 2, 366 | "links": [], 367 | "nullPointMode": "connected", 368 | "paceLength": 10, 369 | "percentage": false, 370 | "pointradius": 5, 371 | "points": false, 372 | "renderer": "flot", 373 | "seriesOverrides": [], 374 | "spaceLength": 10, 375 | "stack": false, 376 | "steppedLine": false, 377 | "targets": [ 378 | { 379 | "dsType": "influxdb", 380 | "groupBy": [ 381 | { 382 | "params": [ 383 | "$interval" 384 | ], 385 | "type": "time" 386 | }, 387 | { 388 | "params": [ 389 | "null" 390 | ], 391 | "type": "fill" 392 | } 393 | ], 394 | "measurement": "PartitionCount", 395 | "orderByTime": "ASC", 396 | "policy": "default", 397 | "refId": "A", 398 | "resultFormat": "time_series", 399 | "select": [ 400 | [ 401 | { 402 | "params": [ 403 | "Value" 404 | ], 405 | "type": "field" 406 | }, 407 | { 408 | "params": [], 409 | "type": "last" 410 | } 411 | ] 412 | ], 413 | "tags": [] 414 | }, 415 | { 416 | "dsType": "influxdb", 417 | "groupBy": [ 418 | { 419 | "params": [ 420 | "$interval" 421 | ], 422 | "type": "time" 423 | }, 424 | { 425 | "params": [ 426 | "null" 427 | ], 428 | "type": "fill" 429 | } 430 | ], 431 | "measurement": "LeaderCount", 432 | "orderByTime": "ASC", 433 | "policy": "default", 434 | "refId": "B", 435 | "resultFormat": "time_series", 436 | "select": [ 437 | [ 438 | { 439 | "params": [ 440 | "Value" 441 | ], 442 | "type": "field" 443 | }, 444 | { 445 | "params": [], 446 | "type": "last" 447 | } 448 | ] 449 | ], 450 | "tags": [] 451 | }, 452 | { 453 | "dsType": "influxdb", 454 | "groupBy": [ 455 | { 456 | "params": [ 457 | "$interval" 458 | ], 459 | "type": "time" 460 | }, 461 | { 462 | "params": [ 463 | "null" 464 | ], 465 | "type": "fill" 466 | } 467 | ], 468 | "measurement": "OfflinePartitionsCount", 469 | "orderByTime": "ASC", 470 | "policy": "default", 471 | "refId": "C", 472 | "resultFormat": "time_series", 473 | "select": [ 474 | [ 475 | { 476 | "params": [ 477 | "Value" 478 | ], 479 | "type": "field" 480 | }, 481 | { 482 | "params": [], 483 | "type": "last" 484 | } 485 | ] 486 | ], 487 | "tags": [] 488 | } 489 | ], 490 | "thresholds": [ 491 | { 492 | "colorMode": "critical", 493 | "fill": true, 494 | "line": true, 495 | "op": "gt", 496 | "value": 0 497 | } 498 | ], 499 | "timeFrom": null, 500 | "timeRegions": [], 501 | "timeShift": null, 502 | "title": "Partition / Leader Count", 503 | "tooltip": { 504 | "msResolution": true, 505 | "shared": true, 506 | "sort": 0, 507 | "value_type": "cumulative" 508 | }, 509 | "type": "graph", 510 | "xaxis": { 511 | "buckets": null, 512 | "mode": "time", 513 | "name": null, 514 | "show": true, 515 | "values": [] 516 | }, 517 | "yaxes": [ 518 | { 519 | "format": "short", 520 | "label": null, 521 | "logBase": 1, 522 | "max": null, 523 | "min": null, 524 | "show": true 525 | }, 526 | { 527 | "format": "short", 528 | "label": null, 529 | "logBase": 1, 530 | "max": null, 531 | "min": null, 532 | "show": true 533 | } 534 | ], 535 | "yaxis": { 536 | "align": false, 537 | "alignLevel": null 538 | } 539 | }, 540 | { 541 | "aliasColors": {}, 542 | "bars": false, 543 | "dashLength": 10, 544 | "dashes": false, 545 | "datasource": "influxdb-kafka", 546 | "editable": true, 547 | "error": false, 548 | "fill": 1, 549 | "grid": {}, 550 | "gridPos": { 551 | "h": 7, 552 | "w": 12, 553 | "x": 0, 554 | "y": 18 555 | }, 556 | "id": 26, 557 | "isNew": true, 558 | "legend": { 559 | "avg": false, 560 | "current": false, 561 | "max": false, 562 | "min": false, 563 | "show": true, 564 | "total": false, 565 | "values": false 566 | }, 567 | "lines": true, 568 | "linewidth": 2, 569 | "links": [], 570 | "nullPointMode": "connected", 571 | "paceLength": 10, 572 | "percentage": false, 573 | "pointradius": 5, 574 | "points": false, 575 | "renderer": "flot", 576 | "seriesOverrides": [], 577 | "spaceLength": 10, 578 | "stack": false, 579 | "steppedLine": false, 580 | "targets": [ 581 | { 582 | "alias": "", 583 | "dsType": "influxdb", 584 | "groupBy": [], 585 | "measurement": "mmConsumerFetchMetrics", 586 | "orderByTime": "ASC", 587 | "policy": "default", 588 | "query": "SELECT \"Value\" FROM \"ConsumerLag\" WHERE (\"typeName\" != 'type=FetcherLagMetrics,name=ConsumerLag,clientId=ReplicaFetcherThread-0-10,topic=__consumer_offsets,partition=1' OR \"typeName\" != 'type=FetcherLagMetrics,name=ConsumerLag,clientId=ReplicaFetcherThread-0-10,topic=__consumer_offsets,partition=10' OR \"typeName\" != 'type=FetcherLagMetrics,name=ConsumerLag,clientId=ReplicaFetcherThread-0-10,topic=__consumer_offsets,partition=11') AND $timeFilter GROUP BY \"typeName\"", 589 | "rawQuery": false, 590 | "refId": "B", 591 | "resultFormat": "time_series", 592 | "select": [ 593 | [ 594 | { 595 | "params": [ 596 | "records-lag-max" 597 | ], 598 | "type": "field" 599 | } 600 | ] 601 | ], 602 | "tags": [] 603 | } 604 | ], 605 | "thresholds": [], 606 | "timeFrom": null, 607 | "timeRegions": [], 608 | "timeShift": null, 609 | "title": "Consumer Lag", 610 | "tooltip": { 611 | "msResolution": true, 612 | "shared": true, 613 | "sort": 0, 614 | "value_type": "cumulative" 615 | }, 616 | "type": "graph", 617 | "xaxis": { 618 | "buckets": null, 619 | "mode": "time", 620 | "name": null, 621 | "show": true, 622 | "values": [] 623 | }, 624 | "yaxes": [ 625 | { 626 | "format": "short", 627 | "label": null, 628 | "logBase": 1, 629 | "max": null, 630 | "min": null, 631 | "show": true 632 | }, 633 | { 634 | "format": "short", 635 | "label": null, 636 | "logBase": 1, 637 | "max": null, 638 | "min": null, 639 | "show": true 640 | } 641 | ], 642 | "yaxis": { 643 | "align": false, 644 | "alignLevel": null 645 | } 646 | }, 647 | { 648 | "collapsed": false, 649 | "gridPos": { 650 | "h": 1, 651 | "w": 24, 652 | "x": 0, 653 | "y": 25 654 | }, 655 | "id": 22, 656 | "panels": [], 657 | "title": "Network", 658 | "type": "row" 659 | }, 660 | { 661 | "aliasColors": {}, 662 | "bars": false, 663 | "dashLength": 10, 664 | "dashes": false, 665 | "datasource": "influxdb-kafka", 666 | "editable": true, 667 | "error": false, 668 | "fill": 1, 669 | "grid": {}, 670 | "gridPos": { 671 | "h": 7, 672 | "w": 12, 673 | "x": 0, 674 | "y": 26 675 | }, 676 | "id": 6, 677 | "isNew": true, 678 | "legend": { 679 | "avg": false, 680 | "current": false, 681 | "max": false, 682 | "min": false, 683 | "show": true, 684 | "total": false, 685 | "values": false 686 | }, 687 | "lines": true, 688 | "linewidth": 2, 689 | "links": [], 690 | "nullPointMode": "connected", 691 | "paceLength": 10, 692 | "percentage": false, 693 | "pointradius": 5, 694 | "points": false, 695 | "renderer": "flot", 696 | "seriesOverrides": [], 697 | "spaceLength": 10, 698 | "stack": false, 699 | "steppedLine": false, 700 | "targets": [ 701 | { 702 | "dsType": "influxdb", 703 | "groupBy": [ 704 | { 705 | "params": [ 706 | "$interval" 707 | ], 708 | "type": "time" 709 | }, 710 | { 711 | "params": [ 712 | "null" 713 | ], 714 | "type": "fill" 715 | } 716 | ], 717 | "measurement": "BytesInPerSec", 718 | "orderByTime": "ASC", 719 | "policy": "default", 720 | "refId": "A", 721 | "resultFormat": "time_series", 722 | "select": [ 723 | [ 724 | { 725 | "params": [ 726 | "OneMinuteRate" 727 | ], 728 | "type": "field" 729 | }, 730 | { 731 | "params": [], 732 | "type": "last" 733 | } 734 | ] 735 | ], 736 | "tags": [] 737 | }, 738 | { 739 | "dsType": "influxdb", 740 | "groupBy": [ 741 | { 742 | "params": [ 743 | "$interval" 744 | ], 745 | "type": "time" 746 | }, 747 | { 748 | "params": [ 749 | "null" 750 | ], 751 | "type": "fill" 752 | } 753 | ], 754 | "measurement": "BytesInPerSecPerTopic", 755 | "orderByTime": "ASC", 756 | "policy": "default", 757 | "refId": "B", 758 | "resultFormat": "time_series", 759 | "select": [ 760 | [ 761 | { 762 | "params": [ 763 | "OneMinuteRate" 764 | ], 765 | "type": "field" 766 | }, 767 | { 768 | "params": [], 769 | "type": "last" 770 | }, 771 | { 772 | "params": [ 773 | "topic_t1" 774 | ], 775 | "type": "alias" 776 | } 777 | ] 778 | ], 779 | "tags": [ 780 | { 781 | "key": "typeName", 782 | "operator": "=", 783 | "value": "type=BrokerTopicMetrics,name=BytesInPerSec,topic=t1" 784 | } 785 | ] 786 | }, 787 | { 788 | "dsType": "influxdb", 789 | "groupBy": [ 790 | { 791 | "params": [ 792 | "$interval" 793 | ], 794 | "type": "time" 795 | }, 796 | { 797 | "params": [ 798 | "null" 799 | ], 800 | "type": "fill" 801 | } 802 | ], 803 | "measurement": "BytesInPerSecPerTopic", 804 | "orderByTime": "ASC", 805 | "policy": "default", 806 | "refId": "C", 807 | "resultFormat": "time_series", 808 | "select": [ 809 | [ 810 | { 811 | "params": [ 812 | "Count" 813 | ], 814 | "type": "field" 815 | }, 816 | { 817 | "params": [], 818 | "type": "last" 819 | }, 820 | { 821 | "params": [ 822 | "topic_t2" 823 | ], 824 | "type": "alias" 825 | } 826 | ] 827 | ], 828 | "tags": [ 829 | { 830 | "key": "typeName", 831 | "operator": "=", 832 | "value": "type=BrokerTopicMetrics,name=BytesInPerSec,topic=t2" 833 | } 834 | ] 835 | } 836 | ], 837 | "thresholds": [], 838 | "timeFrom": null, 839 | "timeRegions": [], 840 | "timeShift": null, 841 | "title": "Bytes In", 842 | "tooltip": { 843 | "msResolution": true, 844 | "shared": true, 845 | "sort": 0, 846 | "value_type": "cumulative" 847 | }, 848 | "type": "graph", 849 | "xaxis": { 850 | "buckets": null, 851 | "mode": "time", 852 | "name": null, 853 | "show": true, 854 | "values": [] 855 | }, 856 | "yaxes": [ 857 | { 858 | "format": "bytes", 859 | "label": null, 860 | "logBase": 1, 861 | "max": null, 862 | "min": null, 863 | "show": true 864 | }, 865 | { 866 | "format": "short", 867 | "label": null, 868 | "logBase": 1, 869 | "max": null, 870 | "min": null, 871 | "show": true 872 | } 873 | ], 874 | "yaxis": { 875 | "align": false, 876 | "alignLevel": null 877 | } 878 | }, 879 | { 880 | "aliasColors": {}, 881 | "bars": false, 882 | "dashLength": 10, 883 | "dashes": false, 884 | "datasource": "influxdb-kafka", 885 | "editable": true, 886 | "error": false, 887 | "fill": 1, 888 | "grid": {}, 889 | "gridPos": { 890 | "h": 7, 891 | "w": 12, 892 | "x": 12, 893 | "y": 26 894 | }, 895 | "id": 16, 896 | "isNew": true, 897 | "legend": { 898 | "avg": false, 899 | "current": false, 900 | "max": false, 901 | "min": false, 902 | "show": true, 903 | "total": false, 904 | "values": false 905 | }, 906 | "lines": true, 907 | "linewidth": 2, 908 | "links": [], 909 | "nullPointMode": "connected", 910 | "paceLength": 10, 911 | "percentage": false, 912 | "pointradius": 5, 913 | "points": false, 914 | "renderer": "flot", 915 | "seriesOverrides": [], 916 | "spaceLength": 10, 917 | "stack": false, 918 | "steppedLine": false, 919 | "targets": [ 920 | { 921 | "dsType": "influxdb", 922 | "groupBy": [ 923 | { 924 | "params": [ 925 | "$interval" 926 | ], 927 | "type": "time" 928 | }, 929 | { 930 | "params": [ 931 | "null" 932 | ], 933 | "type": "fill" 934 | } 935 | ], 936 | "measurement": "BytesOutPerSec", 937 | "orderByTime": "ASC", 938 | "policy": "default", 939 | "refId": "A", 940 | "resultFormat": "time_series", 941 | "select": [ 942 | [ 943 | { 944 | "params": [ 945 | "OneMinuteRate" 946 | ], 947 | "type": "field" 948 | }, 949 | { 950 | "params": [], 951 | "type": "last" 952 | } 953 | ] 954 | ], 955 | "tags": [] 956 | }, 957 | { 958 | "dsType": "influxdb", 959 | "groupBy": [ 960 | { 961 | "params": [ 962 | "$interval" 963 | ], 964 | "type": "time" 965 | }, 966 | { 967 | "params": [ 968 | "null" 969 | ], 970 | "type": "fill" 971 | } 972 | ], 973 | "measurement": "BytesOutPerSecPerTopic", 974 | "orderByTime": "ASC", 975 | "policy": "default", 976 | "refId": "B", 977 | "resultFormat": "time_series", 978 | "select": [ 979 | [ 980 | { 981 | "params": [ 982 | "OneMinuteRate" 983 | ], 984 | "type": "field" 985 | }, 986 | { 987 | "params": [], 988 | "type": "last" 989 | }, 990 | { 991 | "params": [ 992 | "topic_t1" 993 | ], 994 | "type": "alias" 995 | } 996 | ] 997 | ], 998 | "tags": [ 999 | { 1000 | "key": "typeName", 1001 | "operator": "=", 1002 | "value": "type=BrokerTopicMetrics,name=BytesOutPerSec,topic=t1" 1003 | } 1004 | ] 1005 | }, 1006 | { 1007 | "dsType": "influxdb", 1008 | "groupBy": [ 1009 | { 1010 | "params": [ 1011 | "$interval" 1012 | ], 1013 | "type": "time" 1014 | }, 1015 | { 1016 | "params": [ 1017 | "null" 1018 | ], 1019 | "type": "fill" 1020 | } 1021 | ], 1022 | "measurement": "BytesOutPerSecPerTopic", 1023 | "orderByTime": "ASC", 1024 | "policy": "default", 1025 | "refId": "C", 1026 | "resultFormat": "time_series", 1027 | "select": [ 1028 | [ 1029 | { 1030 | "params": [ 1031 | "Count" 1032 | ], 1033 | "type": "field" 1034 | }, 1035 | { 1036 | "params": [], 1037 | "type": "last" 1038 | }, 1039 | { 1040 | "params": [ 1041 | "topic_t2" 1042 | ], 1043 | "type": "alias" 1044 | } 1045 | ] 1046 | ], 1047 | "tags": [ 1048 | { 1049 | "key": "typeName", 1050 | "operator": "=", 1051 | "value": "type=BrokerTopicMetrics,name=BytesOutPerSec,topic=t2" 1052 | } 1053 | ] 1054 | } 1055 | ], 1056 | "thresholds": [], 1057 | "timeFrom": null, 1058 | "timeRegions": [], 1059 | "timeShift": null, 1060 | "title": "Bytes Out", 1061 | "tooltip": { 1062 | "msResolution": true, 1063 | "shared": true, 1064 | "sort": 0, 1065 | "value_type": "cumulative" 1066 | }, 1067 | "type": "graph", 1068 | "xaxis": { 1069 | "buckets": null, 1070 | "mode": "time", 1071 | "name": null, 1072 | "show": true, 1073 | "values": [] 1074 | }, 1075 | "yaxes": [ 1076 | { 1077 | "format": "bytes", 1078 | "label": null, 1079 | "logBase": 1, 1080 | "max": null, 1081 | "min": null, 1082 | "show": true 1083 | }, 1084 | { 1085 | "format": "short", 1086 | "label": null, 1087 | "logBase": 1, 1088 | "max": null, 1089 | "min": null, 1090 | "show": true 1091 | } 1092 | ], 1093 | "yaxis": { 1094 | "align": false, 1095 | "alignLevel": null 1096 | } 1097 | }, 1098 | { 1099 | "aliasColors": {}, 1100 | "bars": false, 1101 | "dashLength": 10, 1102 | "dashes": false, 1103 | "datasource": "influxdb-kafka", 1104 | "editable": true, 1105 | "error": false, 1106 | "fill": 1, 1107 | "grid": {}, 1108 | "gridPos": { 1109 | "h": 7, 1110 | "w": 12, 1111 | "x": 0, 1112 | "y": 33 1113 | }, 1114 | "id": 7, 1115 | "isNew": true, 1116 | "legend": { 1117 | "avg": false, 1118 | "current": false, 1119 | "max": false, 1120 | "min": false, 1121 | "show": true, 1122 | "total": false, 1123 | "values": false 1124 | }, 1125 | "lines": true, 1126 | "linewidth": 2, 1127 | "links": [], 1128 | "nullPointMode": "connected", 1129 | "paceLength": 10, 1130 | "percentage": false, 1131 | "pointradius": 5, 1132 | "points": false, 1133 | "renderer": "flot", 1134 | "seriesOverrides": [], 1135 | "spaceLength": 10, 1136 | "stack": false, 1137 | "steppedLine": false, 1138 | "targets": [ 1139 | { 1140 | "dsType": "influxdb", 1141 | "groupBy": [ 1142 | { 1143 | "params": [ 1144 | "$interval" 1145 | ], 1146 | "type": "time" 1147 | }, 1148 | { 1149 | "params": [ 1150 | "null" 1151 | ], 1152 | "type": "fill" 1153 | } 1154 | ], 1155 | "measurement": "BytesInPerSec", 1156 | "policy": "default", 1157 | "refId": "A", 1158 | "resultFormat": "time_series", 1159 | "select": [ 1160 | [ 1161 | { 1162 | "params": [ 1163 | "FiveMinuteRate" 1164 | ], 1165 | "type": "field" 1166 | }, 1167 | { 1168 | "params": [], 1169 | "type": "last" 1170 | } 1171 | ] 1172 | ], 1173 | "tags": [] 1174 | }, 1175 | { 1176 | "dsType": "influxdb", 1177 | "groupBy": [ 1178 | { 1179 | "params": [ 1180 | "$interval" 1181 | ], 1182 | "type": "time" 1183 | }, 1184 | { 1185 | "params": [ 1186 | "null" 1187 | ], 1188 | "type": "fill" 1189 | } 1190 | ], 1191 | "measurement": "BytesInPerSecPerTopic", 1192 | "policy": "default", 1193 | "refId": "B", 1194 | "resultFormat": "time_series", 1195 | "select": [ 1196 | [ 1197 | { 1198 | "params": [ 1199 | "FiveMinuteRate" 1200 | ], 1201 | "type": "field" 1202 | }, 1203 | { 1204 | "params": [], 1205 | "type": "last" 1206 | }, 1207 | { 1208 | "params": [ 1209 | "topic_t1" 1210 | ], 1211 | "type": "alias" 1212 | } 1213 | ] 1214 | ], 1215 | "tags": [ 1216 | { 1217 | "key": "typeName", 1218 | "operator": "=", 1219 | "value": "type=BrokerTopicMetrics,name=BytesInPerSec,topic=t1" 1220 | } 1221 | ] 1222 | }, 1223 | { 1224 | "dsType": "influxdb", 1225 | "groupBy": [ 1226 | { 1227 | "params": [ 1228 | "$interval" 1229 | ], 1230 | "type": "time" 1231 | }, 1232 | { 1233 | "params": [ 1234 | "null" 1235 | ], 1236 | "type": "fill" 1237 | } 1238 | ], 1239 | "measurement": "BytesInPerSecPerTopic", 1240 | "policy": "default", 1241 | "refId": "C", 1242 | "resultFormat": "time_series", 1243 | "select": [ 1244 | [ 1245 | { 1246 | "params": [ 1247 | "FiveMinuteRate" 1248 | ], 1249 | "type": "field" 1250 | }, 1251 | { 1252 | "params": [], 1253 | "type": "last" 1254 | }, 1255 | { 1256 | "params": [ 1257 | "topic_t2" 1258 | ], 1259 | "type": "alias" 1260 | } 1261 | ] 1262 | ], 1263 | "tags": [ 1264 | { 1265 | "key": "typeName", 1266 | "operator": "=", 1267 | "value": "type=BrokerTopicMetrics,name=BytesInPerSec,topic=t2" 1268 | } 1269 | ] 1270 | } 1271 | ], 1272 | "thresholds": [], 1273 | "timeFrom": null, 1274 | "timeRegions": [], 1275 | "timeShift": null, 1276 | "title": "Bytes In (5 minute rate)", 1277 | "tooltip": { 1278 | "msResolution": true, 1279 | "shared": true, 1280 | "sort": 0, 1281 | "value_type": "cumulative" 1282 | }, 1283 | "type": "graph", 1284 | "xaxis": { 1285 | "buckets": null, 1286 | "mode": "time", 1287 | "name": null, 1288 | "show": true, 1289 | "values": [] 1290 | }, 1291 | "yaxes": [ 1292 | { 1293 | "format": "bytes", 1294 | "label": null, 1295 | "logBase": 1, 1296 | "max": null, 1297 | "min": null, 1298 | "show": true 1299 | }, 1300 | { 1301 | "format": "short", 1302 | "label": null, 1303 | "logBase": 1, 1304 | "max": null, 1305 | "min": null, 1306 | "show": true 1307 | } 1308 | ], 1309 | "yaxis": { 1310 | "align": false, 1311 | "alignLevel": null 1312 | } 1313 | }, 1314 | { 1315 | "aliasColors": {}, 1316 | "bars": false, 1317 | "dashLength": 10, 1318 | "dashes": false, 1319 | "datasource": "influxdb-kafka", 1320 | "editable": true, 1321 | "error": false, 1322 | "fill": 1, 1323 | "grid": {}, 1324 | "gridPos": { 1325 | "h": 7, 1326 | "w": 12, 1327 | "x": 12, 1328 | "y": 33 1329 | }, 1330 | "id": 17, 1331 | "isNew": true, 1332 | "legend": { 1333 | "avg": false, 1334 | "current": false, 1335 | "max": false, 1336 | "min": false, 1337 | "show": true, 1338 | "total": false, 1339 | "values": false 1340 | }, 1341 | "lines": true, 1342 | "linewidth": 2, 1343 | "links": [], 1344 | "nullPointMode": "connected", 1345 | "paceLength": 10, 1346 | "percentage": false, 1347 | "pointradius": 5, 1348 | "points": false, 1349 | "renderer": "flot", 1350 | "seriesOverrides": [], 1351 | "spaceLength": 10, 1352 | "stack": false, 1353 | "steppedLine": false, 1354 | "targets": [ 1355 | { 1356 | "dsType": "influxdb", 1357 | "groupBy": [ 1358 | { 1359 | "params": [ 1360 | "$interval" 1361 | ], 1362 | "type": "time" 1363 | }, 1364 | { 1365 | "params": [ 1366 | "null" 1367 | ], 1368 | "type": "fill" 1369 | } 1370 | ], 1371 | "measurement": "BytesOutPerSec", 1372 | "policy": "default", 1373 | "refId": "A", 1374 | "resultFormat": "time_series", 1375 | "select": [ 1376 | [ 1377 | { 1378 | "params": [ 1379 | "FiveMinuteRate" 1380 | ], 1381 | "type": "field" 1382 | }, 1383 | { 1384 | "params": [], 1385 | "type": "last" 1386 | } 1387 | ] 1388 | ], 1389 | "tags": [] 1390 | }, 1391 | { 1392 | "dsType": "influxdb", 1393 | "groupBy": [ 1394 | { 1395 | "params": [ 1396 | "$interval" 1397 | ], 1398 | "type": "time" 1399 | }, 1400 | { 1401 | "params": [ 1402 | "null" 1403 | ], 1404 | "type": "fill" 1405 | } 1406 | ], 1407 | "measurement": "BytesOutPerSecPerTopic", 1408 | "policy": "default", 1409 | "refId": "B", 1410 | "resultFormat": "time_series", 1411 | "select": [ 1412 | [ 1413 | { 1414 | "params": [ 1415 | "FiveMinuteRate" 1416 | ], 1417 | "type": "field" 1418 | }, 1419 | { 1420 | "params": [], 1421 | "type": "last" 1422 | }, 1423 | { 1424 | "params": [ 1425 | "topic_t1" 1426 | ], 1427 | "type": "alias" 1428 | } 1429 | ] 1430 | ], 1431 | "tags": [ 1432 | { 1433 | "key": "typeName", 1434 | "operator": "=", 1435 | "value": "type=BrokerTopicMetrics,name=BytesOutPerSec,topic=t1" 1436 | } 1437 | ] 1438 | }, 1439 | { 1440 | "dsType": "influxdb", 1441 | "groupBy": [ 1442 | { 1443 | "params": [ 1444 | "$interval" 1445 | ], 1446 | "type": "time" 1447 | }, 1448 | { 1449 | "params": [ 1450 | "null" 1451 | ], 1452 | "type": "fill" 1453 | } 1454 | ], 1455 | "measurement": "BytesOutPerSecPerTopic", 1456 | "policy": "default", 1457 | "refId": "C", 1458 | "resultFormat": "time_series", 1459 | "select": [ 1460 | [ 1461 | { 1462 | "params": [ 1463 | "FiveMinuteRate" 1464 | ], 1465 | "type": "field" 1466 | }, 1467 | { 1468 | "params": [], 1469 | "type": "last" 1470 | }, 1471 | { 1472 | "params": [ 1473 | "topic_t2" 1474 | ], 1475 | "type": "alias" 1476 | } 1477 | ] 1478 | ], 1479 | "tags": [ 1480 | { 1481 | "key": "typeName", 1482 | "operator": "=", 1483 | "value": "type=BrokerTopicMetrics,name=BytesOutPerSec,topic=t2" 1484 | } 1485 | ] 1486 | } 1487 | ], 1488 | "thresholds": [], 1489 | "timeFrom": null, 1490 | "timeRegions": [], 1491 | "timeShift": null, 1492 | "title": "Bytes Out (5 minute rate)", 1493 | "tooltip": { 1494 | "msResolution": true, 1495 | "shared": true, 1496 | "sort": 0, 1497 | "value_type": "cumulative" 1498 | }, 1499 | "type": "graph", 1500 | "xaxis": { 1501 | "buckets": null, 1502 | "mode": "time", 1503 | "name": null, 1504 | "show": true, 1505 | "values": [] 1506 | }, 1507 | "yaxes": [ 1508 | { 1509 | "format": "bytes", 1510 | "label": null, 1511 | "logBase": 1, 1512 | "max": null, 1513 | "min": null, 1514 | "show": true 1515 | }, 1516 | { 1517 | "format": "short", 1518 | "label": null, 1519 | "logBase": 1, 1520 | "max": null, 1521 | "min": null, 1522 | "show": true 1523 | } 1524 | ], 1525 | "yaxis": { 1526 | "align": false, 1527 | "alignLevel": null 1528 | } 1529 | }, 1530 | { 1531 | "aliasColors": {}, 1532 | "bars": false, 1533 | "dashLength": 10, 1534 | "dashes": false, 1535 | "datasource": "influxdb-kafka", 1536 | "editable": true, 1537 | "error": false, 1538 | "fill": 1, 1539 | "grid": {}, 1540 | "gridPos": { 1541 | "h": 7, 1542 | "w": 8, 1543 | "x": 0, 1544 | "y": 40 1545 | }, 1546 | "id": 9, 1547 | "isNew": true, 1548 | "legend": { 1549 | "avg": false, 1550 | "current": false, 1551 | "max": false, 1552 | "min": false, 1553 | "show": true, 1554 | "total": false, 1555 | "values": false 1556 | }, 1557 | "lines": true, 1558 | "linewidth": 2, 1559 | "links": [], 1560 | "nullPointMode": "connected", 1561 | "paceLength": 10, 1562 | "percentage": false, 1563 | "pointradius": 5, 1564 | "points": false, 1565 | "renderer": "flot", 1566 | "seriesOverrides": [], 1567 | "spaceLength": 10, 1568 | "stack": false, 1569 | "steppedLine": false, 1570 | "targets": [ 1571 | { 1572 | "dsType": "influxdb", 1573 | "groupBy": [ 1574 | { 1575 | "params": [ 1576 | "$interval" 1577 | ], 1578 | "type": "time" 1579 | }, 1580 | { 1581 | "params": [ 1582 | "null" 1583 | ], 1584 | "type": "fill" 1585 | } 1586 | ], 1587 | "measurement": "RequestHandlerAvgIdlePercent", 1588 | "orderByTime": "ASC", 1589 | "policy": "default", 1590 | "refId": "A", 1591 | "resultFormat": "time_series", 1592 | "select": [ 1593 | [ 1594 | { 1595 | "params": [ 1596 | "MeanRate" 1597 | ], 1598 | "type": "field" 1599 | }, 1600 | { 1601 | "params": [], 1602 | "type": "mean" 1603 | } 1604 | ] 1605 | ], 1606 | "tags": [] 1607 | } 1608 | ], 1609 | "thresholds": [], 1610 | "timeFrom": null, 1611 | "timeRegions": [], 1612 | "timeShift": null, 1613 | "title": "Request Handler avg idle", 1614 | "tooltip": { 1615 | "msResolution": false, 1616 | "shared": true, 1617 | "sort": 0, 1618 | "value_type": "cumulative" 1619 | }, 1620 | "type": "graph", 1621 | "xaxis": { 1622 | "buckets": null, 1623 | "mode": "time", 1624 | "name": null, 1625 | "show": true, 1626 | "values": [] 1627 | }, 1628 | "yaxes": [ 1629 | { 1630 | "format": "percentunit", 1631 | "label": null, 1632 | "logBase": 1, 1633 | "max": null, 1634 | "min": null, 1635 | "show": true 1636 | }, 1637 | { 1638 | "format": "short", 1639 | "label": null, 1640 | "logBase": 1, 1641 | "max": null, 1642 | "min": null, 1643 | "show": true 1644 | } 1645 | ], 1646 | "yaxis": { 1647 | "align": false, 1648 | "alignLevel": null 1649 | } 1650 | }, 1651 | { 1652 | "aliasColors": {}, 1653 | "bars": false, 1654 | "dashLength": 10, 1655 | "dashes": false, 1656 | "datasource": "influxdb-kafka", 1657 | "editable": true, 1658 | "error": false, 1659 | "fill": 1, 1660 | "grid": {}, 1661 | "gridPos": { 1662 | "h": 7, 1663 | "w": 8, 1664 | "x": 8, 1665 | "y": 40 1666 | }, 1667 | "id": 13, 1668 | "isNew": true, 1669 | "legend": { 1670 | "avg": false, 1671 | "current": false, 1672 | "max": false, 1673 | "min": false, 1674 | "show": true, 1675 | "total": false, 1676 | "values": false 1677 | }, 1678 | "lines": true, 1679 | "linewidth": 2, 1680 | "links": [], 1681 | "nullPointMode": "connected", 1682 | "paceLength": 10, 1683 | "percentage": false, 1684 | "pointradius": 5, 1685 | "points": false, 1686 | "renderer": "flot", 1687 | "seriesOverrides": [], 1688 | "spaceLength": 10, 1689 | "stack": false, 1690 | "steppedLine": false, 1691 | "targets": [ 1692 | { 1693 | "dsType": "influxdb", 1694 | "groupBy": [ 1695 | { 1696 | "params": [ 1697 | "$interval" 1698 | ], 1699 | "type": "time" 1700 | }, 1701 | { 1702 | "params": [ 1703 | "null" 1704 | ], 1705 | "type": "fill" 1706 | } 1707 | ], 1708 | "measurement": "ProduceTotalTimeMs", 1709 | "policy": "default", 1710 | "refId": "A", 1711 | "resultFormat": "time_series", 1712 | "select": [ 1713 | [ 1714 | { 1715 | "params": [ 1716 | "Mean" 1717 | ], 1718 | "type": "field" 1719 | }, 1720 | { 1721 | "params": [], 1722 | "type": "mean" 1723 | } 1724 | ] 1725 | ], 1726 | "tags": [] 1727 | }, 1728 | { 1729 | "dsType": "influxdb", 1730 | "groupBy": [ 1731 | { 1732 | "params": [ 1733 | "$interval" 1734 | ], 1735 | "type": "time" 1736 | }, 1737 | { 1738 | "params": [ 1739 | "null" 1740 | ], 1741 | "type": "fill" 1742 | } 1743 | ], 1744 | "measurement": "FetchConsumerTotalTimeMs", 1745 | "policy": "default", 1746 | "refId": "B", 1747 | "resultFormat": "time_series", 1748 | "select": [ 1749 | [ 1750 | { 1751 | "params": [ 1752 | "Mean" 1753 | ], 1754 | "type": "field" 1755 | }, 1756 | { 1757 | "params": [], 1758 | "type": "mean" 1759 | } 1760 | ] 1761 | ], 1762 | "tags": [] 1763 | }, 1764 | { 1765 | "dsType": "influxdb", 1766 | "groupBy": [ 1767 | { 1768 | "params": [ 1769 | "$interval" 1770 | ], 1771 | "type": "time" 1772 | }, 1773 | { 1774 | "params": [ 1775 | "null" 1776 | ], 1777 | "type": "fill" 1778 | } 1779 | ], 1780 | "measurement": "FetchFollowerTotalTimeMs", 1781 | "policy": "default", 1782 | "refId": "C", 1783 | "resultFormat": "time_series", 1784 | "select": [ 1785 | [ 1786 | { 1787 | "params": [ 1788 | "Mean" 1789 | ], 1790 | "type": "field" 1791 | }, 1792 | { 1793 | "params": [], 1794 | "type": "mean" 1795 | } 1796 | ] 1797 | ], 1798 | "tags": [] 1799 | } 1800 | ], 1801 | "thresholds": [], 1802 | "timeFrom": null, 1803 | "timeRegions": [], 1804 | "timeShift": null, 1805 | "title": "Requests Total Time", 1806 | "tooltip": { 1807 | "msResolution": false, 1808 | "shared": true, 1809 | "sort": 0, 1810 | "value_type": "cumulative" 1811 | }, 1812 | "type": "graph", 1813 | "xaxis": { 1814 | "buckets": null, 1815 | "mode": "time", 1816 | "name": null, 1817 | "show": true, 1818 | "values": [] 1819 | }, 1820 | "yaxes": [ 1821 | { 1822 | "format": "short", 1823 | "label": null, 1824 | "logBase": 1, 1825 | "max": null, 1826 | "min": null, 1827 | "show": true 1828 | }, 1829 | { 1830 | "format": "short", 1831 | "label": null, 1832 | "logBase": 1, 1833 | "max": null, 1834 | "min": null, 1835 | "show": true 1836 | } 1837 | ], 1838 | "yaxis": { 1839 | "align": false, 1840 | "alignLevel": null 1841 | } 1842 | }, 1843 | { 1844 | "aliasColors": {}, 1845 | "bars": false, 1846 | "dashLength": 10, 1847 | "dashes": false, 1848 | "datasource": "influxdb-kafka", 1849 | "editable": true, 1850 | "error": false, 1851 | "fill": 1, 1852 | "grid": {}, 1853 | "gridPos": { 1854 | "h": 7, 1855 | "w": 8, 1856 | "x": 16, 1857 | "y": 40 1858 | }, 1859 | "id": 14, 1860 | "isNew": true, 1861 | "legend": { 1862 | "avg": false, 1863 | "current": false, 1864 | "max": false, 1865 | "min": false, 1866 | "show": true, 1867 | "total": false, 1868 | "values": false 1869 | }, 1870 | "lines": true, 1871 | "linewidth": 2, 1872 | "links": [], 1873 | "nullPointMode": "connected", 1874 | "paceLength": 10, 1875 | "percentage": false, 1876 | "pointradius": 5, 1877 | "points": false, 1878 | "renderer": "flot", 1879 | "seriesOverrides": [], 1880 | "spaceLength": 10, 1881 | "stack": false, 1882 | "steppedLine": false, 1883 | "targets": [ 1884 | { 1885 | "dsType": "influxdb", 1886 | "groupBy": [ 1887 | { 1888 | "params": [ 1889 | "$interval" 1890 | ], 1891 | "type": "time" 1892 | }, 1893 | { 1894 | "params": [ 1895 | "null" 1896 | ], 1897 | "type": "fill" 1898 | } 1899 | ], 1900 | "measurement": "ProduceRequestQueueTimeMs", 1901 | "orderByTime": "ASC", 1902 | "policy": "default", 1903 | "refId": "A", 1904 | "resultFormat": "time_series", 1905 | "select": [ 1906 | [ 1907 | { 1908 | "params": [ 1909 | "Mean" 1910 | ], 1911 | "type": "field" 1912 | }, 1913 | { 1914 | "params": [], 1915 | "type": "mean" 1916 | } 1917 | ] 1918 | ], 1919 | "tags": [] 1920 | }, 1921 | { 1922 | "dsType": "influxdb", 1923 | "groupBy": [ 1924 | { 1925 | "params": [ 1926 | "$interval" 1927 | ], 1928 | "type": "time" 1929 | }, 1930 | { 1931 | "params": [ 1932 | "null" 1933 | ], 1934 | "type": "fill" 1935 | } 1936 | ], 1937 | "measurement": "FetchConsumerRequestQueueTimeMs", 1938 | "orderByTime": "ASC", 1939 | "policy": "default", 1940 | "refId": "B", 1941 | "resultFormat": "time_series", 1942 | "select": [ 1943 | [ 1944 | { 1945 | "params": [ 1946 | "Mean" 1947 | ], 1948 | "type": "field" 1949 | }, 1950 | { 1951 | "params": [], 1952 | "type": "mean" 1953 | } 1954 | ] 1955 | ], 1956 | "tags": [] 1957 | }, 1958 | { 1959 | "dsType": "influxdb", 1960 | "groupBy": [ 1961 | { 1962 | "params": [ 1963 | "$interval" 1964 | ], 1965 | "type": "time" 1966 | }, 1967 | { 1968 | "params": [ 1969 | "null" 1970 | ], 1971 | "type": "fill" 1972 | } 1973 | ], 1974 | "measurement": "FetchFollowerRequestQueueTimeMs", 1975 | "orderByTime": "ASC", 1976 | "policy": "default", 1977 | "refId": "C", 1978 | "resultFormat": "time_series", 1979 | "select": [ 1980 | [ 1981 | { 1982 | "params": [ 1983 | "Mean" 1984 | ], 1985 | "type": "field" 1986 | }, 1987 | { 1988 | "params": [], 1989 | "type": "mean" 1990 | } 1991 | ] 1992 | ], 1993 | "tags": [] 1994 | } 1995 | ], 1996 | "thresholds": [], 1997 | "timeFrom": null, 1998 | "timeRegions": [], 1999 | "timeShift": null, 2000 | "title": "Request Queue Times", 2001 | "tooltip": { 2002 | "msResolution": false, 2003 | "shared": true, 2004 | "sort": 0, 2005 | "value_type": "cumulative" 2006 | }, 2007 | "type": "graph", 2008 | "xaxis": { 2009 | "buckets": null, 2010 | "mode": "time", 2011 | "name": null, 2012 | "show": true, 2013 | "values": [] 2014 | }, 2015 | "yaxes": [ 2016 | { 2017 | "format": "short", 2018 | "label": null, 2019 | "logBase": 1, 2020 | "max": null, 2021 | "min": null, 2022 | "show": true 2023 | }, 2024 | { 2025 | "format": "short", 2026 | "label": null, 2027 | "logBase": 1, 2028 | "max": null, 2029 | "min": null, 2030 | "show": true 2031 | } 2032 | ], 2033 | "yaxis": { 2034 | "align": false, 2035 | "alignLevel": null 2036 | } 2037 | }, 2038 | { 2039 | "collapsed": false, 2040 | "gridPos": { 2041 | "h": 1, 2042 | "w": 24, 2043 | "x": 0, 2044 | "y": 47 2045 | }, 2046 | "id": 21, 2047 | "panels": [], 2048 | "title": "Kafka Internals", 2049 | "type": "row" 2050 | }, 2051 | { 2052 | "aliasColors": {}, 2053 | "bars": false, 2054 | "dashLength": 10, 2055 | "dashes": false, 2056 | "datasource": "influxdb-kafka", 2057 | "editable": true, 2058 | "error": false, 2059 | "fill": 1, 2060 | "grid": {}, 2061 | "gridPos": { 2062 | "h": 7, 2063 | "w": 12, 2064 | "x": 0, 2065 | "y": 48 2066 | }, 2067 | "id": 10, 2068 | "isNew": true, 2069 | "legend": { 2070 | "avg": false, 2071 | "current": false, 2072 | "max": false, 2073 | "min": false, 2074 | "show": true, 2075 | "total": false, 2076 | "values": false 2077 | }, 2078 | "lines": true, 2079 | "linewidth": 2, 2080 | "links": [], 2081 | "nullPointMode": "connected", 2082 | "paceLength": 10, 2083 | "percentage": false, 2084 | "pointradius": 5, 2085 | "points": false, 2086 | "renderer": "flot", 2087 | "seriesOverrides": [], 2088 | "spaceLength": 10, 2089 | "stack": false, 2090 | "steppedLine": false, 2091 | "targets": [ 2092 | { 2093 | "dsType": "influxdb", 2094 | "groupBy": [ 2095 | { 2096 | "params": [ 2097 | "$interval" 2098 | ], 2099 | "type": "time" 2100 | } 2101 | ], 2102 | "measurement": "LeaderElectionRateAndTimeMs", 2103 | "orderByTime": "ASC", 2104 | "policy": "default", 2105 | "refId": "A", 2106 | "resultFormat": "time_series", 2107 | "select": [ 2108 | [ 2109 | { 2110 | "params": [ 2111 | "OneMinuteRate" 2112 | ], 2113 | "type": "field" 2114 | }, 2115 | { 2116 | "params": [], 2117 | "type": "mean" 2118 | } 2119 | ] 2120 | ], 2121 | "tags": [] 2122 | }, 2123 | { 2124 | "dsType": "influxdb", 2125 | "groupBy": [ 2126 | { 2127 | "params": [ 2128 | "$interval" 2129 | ], 2130 | "type": "time" 2131 | }, 2132 | { 2133 | "params": [ 2134 | "null" 2135 | ], 2136 | "type": "fill" 2137 | } 2138 | ], 2139 | "measurement": "UncleanLeaderElectionsPerSec", 2140 | "orderByTime": "ASC", 2141 | "policy": "default", 2142 | "refId": "B", 2143 | "resultFormat": "time_series", 2144 | "select": [ 2145 | [ 2146 | { 2147 | "params": [ 2148 | "OneMinuteRate" 2149 | ], 2150 | "type": "field" 2151 | }, 2152 | { 2153 | "params": [], 2154 | "type": "mean" 2155 | } 2156 | ] 2157 | ], 2158 | "tags": [] 2159 | } 2160 | ], 2161 | "thresholds": [], 2162 | "timeFrom": null, 2163 | "timeRegions": [], 2164 | "timeShift": null, 2165 | "title": "Leader Election", 2166 | "tooltip": { 2167 | "msResolution": false, 2168 | "shared": true, 2169 | "sort": 0, 2170 | "value_type": "cumulative" 2171 | }, 2172 | "type": "graph", 2173 | "xaxis": { 2174 | "buckets": null, 2175 | "mode": "time", 2176 | "name": null, 2177 | "show": true, 2178 | "values": [] 2179 | }, 2180 | "yaxes": [ 2181 | { 2182 | "format": "short", 2183 | "label": null, 2184 | "logBase": 1, 2185 | "max": null, 2186 | "min": null, 2187 | "show": true 2188 | }, 2189 | { 2190 | "format": "short", 2191 | "label": null, 2192 | "logBase": 1, 2193 | "max": null, 2194 | "min": null, 2195 | "show": true 2196 | } 2197 | ], 2198 | "yaxis": { 2199 | "align": false, 2200 | "alignLevel": null 2201 | } 2202 | }, 2203 | { 2204 | "aliasColors": {}, 2205 | "bars": false, 2206 | "dashLength": 10, 2207 | "dashes": false, 2208 | "datasource": "influxdb-kafka", 2209 | "editable": true, 2210 | "error": false, 2211 | "fill": 1, 2212 | "grid": {}, 2213 | "gridPos": { 2214 | "h": 7, 2215 | "w": 12, 2216 | "x": 12, 2217 | "y": 48 2218 | }, 2219 | "id": 11, 2220 | "isNew": true, 2221 | "legend": { 2222 | "avg": false, 2223 | "current": false, 2224 | "max": false, 2225 | "min": false, 2226 | "show": true, 2227 | "total": false, 2228 | "values": false 2229 | }, 2230 | "lines": true, 2231 | "linewidth": 2, 2232 | "links": [], 2233 | "nullPointMode": "connected", 2234 | "paceLength": 10, 2235 | "percentage": false, 2236 | "pointradius": 5, 2237 | "points": false, 2238 | "renderer": "flot", 2239 | "seriesOverrides": [], 2240 | "spaceLength": 10, 2241 | "stack": false, 2242 | "steppedLine": false, 2243 | "targets": [ 2244 | { 2245 | "dsType": "influxdb", 2246 | "groupBy": [ 2247 | { 2248 | "params": [ 2249 | "$interval" 2250 | ], 2251 | "type": "time" 2252 | } 2253 | ], 2254 | "measurement": "ActiveControllerCount", 2255 | "orderByTime": "ASC", 2256 | "policy": "default", 2257 | "refId": "A", 2258 | "resultFormat": "time_series", 2259 | "select": [ 2260 | [ 2261 | { 2262 | "params": [ 2263 | "Value" 2264 | ], 2265 | "type": "field" 2266 | }, 2267 | { 2268 | "params": [], 2269 | "type": "sum" 2270 | } 2271 | ] 2272 | ], 2273 | "tags": [] 2274 | } 2275 | ], 2276 | "thresholds": [], 2277 | "timeFrom": null, 2278 | "timeRegions": [], 2279 | "timeShift": null, 2280 | "title": "Active Controllers", 2281 | "tooltip": { 2282 | "msResolution": false, 2283 | "shared": true, 2284 | "sort": 0, 2285 | "value_type": "cumulative" 2286 | }, 2287 | "type": "graph", 2288 | "xaxis": { 2289 | "buckets": null, 2290 | "mode": "time", 2291 | "name": null, 2292 | "show": true, 2293 | "values": [] 2294 | }, 2295 | "yaxes": [ 2296 | { 2297 | "format": "short", 2298 | "label": null, 2299 | "logBase": 1, 2300 | "max": null, 2301 | "min": null, 2302 | "show": true 2303 | }, 2304 | { 2305 | "format": "short", 2306 | "label": null, 2307 | "logBase": 1, 2308 | "max": null, 2309 | "min": null, 2310 | "show": true 2311 | } 2312 | ], 2313 | "yaxis": { 2314 | "align": false, 2315 | "alignLevel": null 2316 | } 2317 | }, 2318 | { 2319 | "aliasColors": {}, 2320 | "bars": false, 2321 | "dashLength": 10, 2322 | "dashes": false, 2323 | "datasource": "influxdb-kafka", 2324 | "editable": true, 2325 | "error": false, 2326 | "fill": 1, 2327 | "grid": {}, 2328 | "gridPos": { 2329 | "h": 7, 2330 | "w": 12, 2331 | "x": 0, 2332 | "y": 55 2333 | }, 2334 | "id": 3, 2335 | "isNew": true, 2336 | "legend": { 2337 | "avg": false, 2338 | "current": false, 2339 | "max": false, 2340 | "min": false, 2341 | "show": true, 2342 | "total": false, 2343 | "values": false 2344 | }, 2345 | "lines": true, 2346 | "linewidth": 2, 2347 | "links": [], 2348 | "nullPointMode": "connected", 2349 | "paceLength": 10, 2350 | "percentage": false, 2351 | "pointradius": 5, 2352 | "points": false, 2353 | "renderer": "flot", 2354 | "seriesOverrides": [], 2355 | "spaceLength": 10, 2356 | "stack": false, 2357 | "steppedLine": false, 2358 | "targets": [ 2359 | { 2360 | "dsType": "influxdb", 2361 | "groupBy": [ 2362 | { 2363 | "params": [ 2364 | "$interval" 2365 | ], 2366 | "type": "time" 2367 | }, 2368 | { 2369 | "params": [ 2370 | "null" 2371 | ], 2372 | "type": "fill" 2373 | } 2374 | ], 2375 | "measurement": "LogFlush", 2376 | "orderByTime": "ASC", 2377 | "policy": "default", 2378 | "refId": "A", 2379 | "resultFormat": "time_series", 2380 | "select": [ 2381 | [ 2382 | { 2383 | "params": [ 2384 | "OneMinuteRate" 2385 | ], 2386 | "type": "field" 2387 | }, 2388 | { 2389 | "params": [], 2390 | "type": "last" 2391 | } 2392 | ] 2393 | ], 2394 | "tags": [] 2395 | } 2396 | ], 2397 | "thresholds": [], 2398 | "timeFrom": null, 2399 | "timeRegions": [], 2400 | "timeShift": null, 2401 | "title": "Log Flush latency", 2402 | "tooltip": { 2403 | "msResolution": false, 2404 | "shared": true, 2405 | "sort": 0, 2406 | "value_type": "cumulative" 2407 | }, 2408 | "type": "graph", 2409 | "xaxis": { 2410 | "buckets": null, 2411 | "mode": "time", 2412 | "name": null, 2413 | "show": true, 2414 | "values": [] 2415 | }, 2416 | "yaxes": [ 2417 | { 2418 | "format": "short", 2419 | "label": null, 2420 | "logBase": 1, 2421 | "max": null, 2422 | "min": null, 2423 | "show": true 2424 | }, 2425 | { 2426 | "format": "short", 2427 | "label": null, 2428 | "logBase": 1, 2429 | "max": null, 2430 | "min": null, 2431 | "show": true 2432 | } 2433 | ], 2434 | "yaxis": { 2435 | "align": false, 2436 | "alignLevel": null 2437 | } 2438 | }, 2439 | { 2440 | "aliasColors": {}, 2441 | "bars": false, 2442 | "dashLength": 10, 2443 | "dashes": false, 2444 | "datasource": "influxdb-kafka", 2445 | "editable": true, 2446 | "error": false, 2447 | "fill": 1, 2448 | "grid": {}, 2449 | "gridPos": { 2450 | "h": 7, 2451 | "w": 12, 2452 | "x": 12, 2453 | "y": 55 2454 | }, 2455 | "id": 5, 2456 | "isNew": true, 2457 | "legend": { 2458 | "avg": false, 2459 | "current": false, 2460 | "max": false, 2461 | "min": false, 2462 | "show": true, 2463 | "total": false, 2464 | "values": false 2465 | }, 2466 | "lines": true, 2467 | "linewidth": 2, 2468 | "links": [], 2469 | "nullPointMode": "connected", 2470 | "paceLength": 10, 2471 | "percentage": false, 2472 | "pointradius": 5, 2473 | "points": false, 2474 | "renderer": "flot", 2475 | "seriesOverrides": [], 2476 | "spaceLength": 10, 2477 | "stack": false, 2478 | "steppedLine": false, 2479 | "targets": [ 2480 | { 2481 | "dsType": "influxdb", 2482 | "groupBy": [ 2483 | { 2484 | "params": [ 2485 | "$interval" 2486 | ], 2487 | "type": "time" 2488 | }, 2489 | { 2490 | "params": [ 2491 | "null" 2492 | ], 2493 | "type": "fill" 2494 | } 2495 | ], 2496 | "hide": false, 2497 | "measurement": "MessagesInPerSec", 2498 | "orderByTime": "ASC", 2499 | "policy": "default", 2500 | "refId": "A", 2501 | "resultFormat": "time_series", 2502 | "select": [ 2503 | [ 2504 | { 2505 | "params": [ 2506 | "OneMinuteRate" 2507 | ], 2508 | "type": "field" 2509 | }, 2510 | { 2511 | "params": [], 2512 | "type": "last" 2513 | } 2514 | ] 2515 | ], 2516 | "tags": [] 2517 | } 2518 | ], 2519 | "thresholds": [], 2520 | "timeFrom": null, 2521 | "timeRegions": [], 2522 | "timeShift": null, 2523 | "title": "Messages In per sec", 2524 | "tooltip": { 2525 | "msResolution": false, 2526 | "shared": true, 2527 | "sort": 0, 2528 | "value_type": "cumulative" 2529 | }, 2530 | "type": "graph", 2531 | "xaxis": { 2532 | "buckets": null, 2533 | "mode": "time", 2534 | "name": null, 2535 | "show": true, 2536 | "values": [] 2537 | }, 2538 | "yaxes": [ 2539 | { 2540 | "format": "short", 2541 | "label": null, 2542 | "logBase": 1, 2543 | "max": null, 2544 | "min": null, 2545 | "show": true 2546 | }, 2547 | { 2548 | "format": "short", 2549 | "label": null, 2550 | "logBase": 1, 2551 | "max": null, 2552 | "min": null, 2553 | "show": true 2554 | } 2555 | ], 2556 | "yaxis": { 2557 | "align": false, 2558 | "alignLevel": null 2559 | } 2560 | }, 2561 | { 2562 | "collapsed": false, 2563 | "gridPos": { 2564 | "h": 1, 2565 | "w": 24, 2566 | "x": 0, 2567 | "y": 62 2568 | }, 2569 | "id": 18, 2570 | "panels": [], 2571 | "title": "OS Level", 2572 | "type": "row" 2573 | }, 2574 | { 2575 | "aliasColors": {}, 2576 | "bars": false, 2577 | "dashLength": 10, 2578 | "dashes": false, 2579 | "datasource": "influxdb-kafka", 2580 | "editable": true, 2581 | "error": false, 2582 | "fill": 1, 2583 | "grid": {}, 2584 | "gridPos": { 2585 | "h": 7, 2586 | "w": 12, 2587 | "x": 0, 2588 | "y": 63 2589 | }, 2590 | "id": 1, 2591 | "isNew": true, 2592 | "legend": { 2593 | "avg": false, 2594 | "current": false, 2595 | "max": false, 2596 | "min": false, 2597 | "show": true, 2598 | "total": false, 2599 | "values": false 2600 | }, 2601 | "lines": true, 2602 | "linewidth": 2, 2603 | "links": [], 2604 | "nullPointMode": "connected", 2605 | "paceLength": 10, 2606 | "percentage": false, 2607 | "pointradius": 5, 2608 | "points": false, 2609 | "renderer": "flot", 2610 | "seriesOverrides": [], 2611 | "spaceLength": 10, 2612 | "stack": false, 2613 | "steppedLine": false, 2614 | "targets": [ 2615 | { 2616 | "dsType": "influxdb", 2617 | "groupBy": [ 2618 | { 2619 | "params": [ 2620 | "$interval" 2621 | ], 2622 | "type": "time" 2623 | }, 2624 | { 2625 | "params": [ 2626 | "null" 2627 | ], 2628 | "type": "fill" 2629 | } 2630 | ], 2631 | "measurement": "jvmMemory", 2632 | "orderByTime": "ASC", 2633 | "policy": "default", 2634 | "refId": "A", 2635 | "resultFormat": "time_series", 2636 | "select": [ 2637 | [ 2638 | { 2639 | "params": [ 2640 | "HeapMemoryUsage_init" 2641 | ], 2642 | "type": "field" 2643 | }, 2644 | { 2645 | "params": [], 2646 | "type": "last" 2647 | }, 2648 | { 2649 | "params": [ 2650 | "heap" 2651 | ], 2652 | "type": "alias" 2653 | } 2654 | ] 2655 | ], 2656 | "tags": [ 2657 | { 2658 | "key": "attributeName", 2659 | "operator": "=", 2660 | "value": "HeapMemoryUsage" 2661 | } 2662 | ] 2663 | }, 2664 | { 2665 | "dsType": "influxdb", 2666 | "groupBy": [ 2667 | { 2668 | "params": [ 2669 | "$interval" 2670 | ], 2671 | "type": "time" 2672 | }, 2673 | { 2674 | "params": [ 2675 | "null" 2676 | ], 2677 | "type": "fill" 2678 | } 2679 | ], 2680 | "measurement": "jvmMemory", 2681 | "orderByTime": "ASC", 2682 | "policy": "default", 2683 | "refId": "B", 2684 | "resultFormat": "time_series", 2685 | "select": [ 2686 | [ 2687 | { 2688 | "params": [ 2689 | "NonHeapMemoryUsage_init" 2690 | ], 2691 | "type": "field" 2692 | }, 2693 | { 2694 | "params": [], 2695 | "type": "last" 2696 | }, 2697 | { 2698 | "params": [ 2699 | "non-heap" 2700 | ], 2701 | "type": "alias" 2702 | } 2703 | ] 2704 | ], 2705 | "tags": [ 2706 | { 2707 | "key": "attributeName", 2708 | "operator": "=", 2709 | "value": "NonHeapMemoryUsage" 2710 | } 2711 | ] 2712 | }, 2713 | { 2714 | "dsType": "influxdb", 2715 | "groupBy": [ 2716 | { 2717 | "params": [ 2718 | "$interval" 2719 | ], 2720 | "type": "time" 2721 | }, 2722 | { 2723 | "params": [ 2724 | "null" 2725 | ], 2726 | "type": "fill" 2727 | } 2728 | ], 2729 | "measurement": "os", 2730 | "orderByTime": "ASC", 2731 | "policy": "default", 2732 | "refId": "C", 2733 | "resultFormat": "time_series", 2734 | "select": [ 2735 | [ 2736 | { 2737 | "params": [ 2738 | "FreePhysicalMemorySize" 2739 | ], 2740 | "type": "field" 2741 | }, 2742 | { 2743 | "params": [], 2744 | "type": "last" 2745 | }, 2746 | { 2747 | "params": [ 2748 | "physical" 2749 | ], 2750 | "type": "alias" 2751 | } 2752 | ] 2753 | ], 2754 | "tags": [] 2755 | } 2756 | ], 2757 | "thresholds": [], 2758 | "timeFrom": null, 2759 | "timeRegions": [], 2760 | "timeShift": null, 2761 | "title": "Memory (JVM & Physical)", 2762 | "tooltip": { 2763 | "msResolution": false, 2764 | "shared": true, 2765 | "sort": 0, 2766 | "value_type": "cumulative" 2767 | }, 2768 | "type": "graph", 2769 | "xaxis": { 2770 | "buckets": null, 2771 | "mode": "time", 2772 | "name": null, 2773 | "show": true, 2774 | "values": [] 2775 | }, 2776 | "yaxes": [ 2777 | { 2778 | "format": "bytes", 2779 | "label": null, 2780 | "logBase": 1, 2781 | "max": null, 2782 | "min": null, 2783 | "show": true 2784 | }, 2785 | { 2786 | "format": "short", 2787 | "label": null, 2788 | "logBase": 1, 2789 | "max": null, 2790 | "min": null, 2791 | "show": true 2792 | } 2793 | ], 2794 | "yaxis": { 2795 | "align": false, 2796 | "alignLevel": null 2797 | } 2798 | } 2799 | ], 2800 | "refresh": "5s", 2801 | "schemaVersion": 18, 2802 | "style": "dark", 2803 | "tags": [], 2804 | "templating": { 2805 | "list": [] 2806 | }, 2807 | "time": { 2808 | "from": "now-30m", 2809 | "to": "now" 2810 | }, 2811 | "timepicker": { 2812 | "refresh_intervals": [ 2813 | "5s", 2814 | "10s", 2815 | "30s", 2816 | "1m", 2817 | "5m", 2818 | "15m", 2819 | "30m", 2820 | "1h", 2821 | "2h", 2822 | "1d" 2823 | ], 2824 | "time_options": [ 2825 | "5m", 2826 | "15m", 2827 | "1h", 2828 | "6h", 2829 | "12h", 2830 | "24h", 2831 | "2d", 2832 | "7d", 2833 | "30d" 2834 | ] 2835 | }, 2836 | "timezone": "browser", 2837 | "title": "Kafka Broker Metrics", 2838 | "uid": "4I2-Khwik", 2839 | "version": 2 2840 | } --------------------------------------------------------------------------------