├── .github └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── .gitignore ├── .java-version ├── .travis.yml ├── CODE_OF_CONDUCT.md ├── HOW_TO_PUBLISH_TO_MAVEN.md ├── LICENSE ├── README.md ├── build.gradle ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── jetbrains.svg ├── monitoring-client ├── bin │ ├── main │ │ ├── com │ │ │ └── omarsmak │ │ │ │ └── kafka │ │ │ │ └── consumer │ │ │ │ └── lag │ │ │ │ └── monitoring │ │ │ │ └── client │ │ │ │ ├── KafkaConsumerLagClient.kt │ │ │ │ ├── KafkaConsumerLagClientFactory.kt │ │ │ │ ├── data │ │ │ │ └── entities.kt │ │ │ │ ├── exceptions │ │ │ │ └── KafkaConsumerLagClientException.kt │ │ │ │ └── impl │ │ │ │ ├── AbstractKafkaConsumerLagClient.kt │ │ │ │ └── KafkaConsumerLagJavaClient.kt │ │ └── config │ │ │ └── consumer-monitoring.properties │ └── test │ │ └── com │ │ └── omarsmak │ │ └── kafka │ │ └── consumer │ │ └── lag │ │ └── monitoring │ │ └── client │ │ └── impl │ │ └── KafkaConsumerLagJavaClientTest.kt ├── build.gradle └── src │ ├── main │ ├── kotlin │ │ └── com │ │ │ └── omarsmak │ │ │ └── kafka │ │ │ └── consumer │ │ │ └── lag │ │ │ └── monitoring │ │ │ └── client │ │ │ ├── KafkaConsumerLagClient.kt │ │ │ ├── KafkaConsumerLagClientFactory.kt │ │ │ ├── data │ │ │ └── entities.kt │ │ │ ├── exceptions │ │ │ └── KafkaConsumerLagClientException.kt │ │ │ └── impl │ │ │ ├── AbstractKafkaConsumerLagClient.kt │ │ │ └── KafkaConsumerLagJavaClient.kt │ └── resources │ │ └── config │ │ └── consumer-monitoring.properties │ └── test │ └── kotlin │ └── com │ └── omarsmak │ └── kafka │ └── consumer │ └── lag │ └── monitoring │ └── client │ └── impl │ └── KafkaConsumerLagJavaClientTest.kt ├── monitoring-component-console ├── bin │ └── main │ │ ├── META-INF │ │ └── native-image │ │ │ ├── jni-config.json │ │ │ ├── proxy-config.json │ │ │ ├── reflect-config.json │ │ │ └── resource-config.json │ │ ├── com │ │ └── omarsmak │ │ │ └── kafka │ │ │ └── consumer │ │ │ └── lag │ │ │ └── monitoring │ │ │ └── component │ │ │ └── console │ │ │ ├── ConsoleMonitoringComponent.kt │ │ │ └── Main.kt │ │ └── simplelogger.properties ├── build.gradle └── src │ └── main │ ├── kotlin │ └── com │ │ └── omarsmak │ │ └── kafka │ │ └── consumer │ │ └── lag │ │ └── monitoring │ │ └── component │ │ └── console │ │ ├── ConsoleMonitoringComponent.kt │ │ └── Main.kt │ └── resources │ ├── META-INF │ └── native-image │ │ ├── jni-config.json │ │ ├── proxy-config.json │ │ ├── reflect-config.json │ │ └── resource-config.json │ └── simplelogger.properties ├── monitoring-component-prometheus ├── bin │ └── main │ │ ├── META-INF │ │ └── native-image │ │ │ ├── jni-config.json │ │ │ ├── proxy-config.json │ │ │ ├── reflect-config.json │ │ │ └── resource-config.json │ │ ├── com │ │ └── omarsmak │ │ │ └── kafka │ │ │ └── consumer │ │ │ └── lag │ │ │ └── monitoring │ │ │ └── component │ │ │ └── prometheus │ │ │ ├── Main.kt │ │ │ └── PrometheusMonitoringComponent.kt │ │ └── configs.properties ├── build.gradle └── src │ └── main │ ├── kotlin │ └── com │ │ └── omarsmak │ │ └── kafka │ │ └── consumer │ │ └── lag │ │ └── monitoring │ │ └── component │ │ └── prometheus │ │ ├── Main.kt │ │ └── PrometheusMonitoringComponent.kt │ └── resources │ ├── META-INF │ └── native-image │ │ ├── jni-config.json │ │ ├── proxy-config.json │ │ ├── reflect-config.json │ │ └── resource-config.json │ └── configs.properties ├── monitoring-core ├── bin │ ├── main │ │ ├── com │ │ │ └── omarsmak │ │ │ │ └── kafka │ │ │ │ └── consumer │ │ │ │ └── lag │ │ │ │ └── monitoring │ │ │ │ ├── component │ │ │ │ └── MonitoringComponent.kt │ │ │ │ ├── data │ │ │ │ └── ConsumerGroupLag.kt │ │ │ │ ├── engine │ │ │ │ └── MonitoringEngine.kt │ │ │ │ └── support │ │ │ │ ├── Utils.kt │ │ │ │ └── extensions.kt │ │ └── config │ │ │ └── log4j2.template │ └── test │ │ └── com │ │ └── omarsmak │ │ └── kafka │ │ └── consumer │ │ └── lag │ │ └── monitoring │ │ ├── engine │ │ ├── MonitoringEngineTest.kt │ │ └── TestMonitoringComponent.kt │ │ └── support │ │ ├── ExtensionsTest.kt │ │ └── UtilsTest.kt ├── build.gradle └── src │ ├── main │ ├── kotlin │ │ └── com │ │ │ └── omarsmak │ │ │ └── kafka │ │ │ └── consumer │ │ │ └── lag │ │ │ └── monitoring │ │ │ ├── component │ │ │ └── MonitoringComponent.kt │ │ │ ├── data │ │ │ └── ConsumerGroupLag.kt │ │ │ ├── engine │ │ │ └── MonitoringEngine.kt │ │ │ └── support │ │ │ ├── Utils.kt │ │ │ └── extensions.kt │ └── resources │ │ └── config │ │ └── log4j2.template │ └── test │ └── kotlin │ └── com │ └── omarsmak │ └── kafka │ └── consumer │ └── lag │ └── monitoring │ ├── engine │ ├── MonitoringEngineTest.kt │ └── TestMonitoringComponent.kt │ └── support │ ├── ExtensionsTest.kt │ └── UtilsTest.kt └── settings.gradle /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/java,kotlin,gradle,intellij 3 | # Edit at https://www.gitignore.io/?templates=java,kotlin,gradle,intellij 4 | 5 | ### Intellij ### 6 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 7 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 8 | 9 | # User-specific stuff 10 | .idea/**/workspace.xml 11 | .idea/**/tasks.xml 12 | .idea/**/usage.statistics.xml 13 | .idea/**/dictionaries 14 | .idea/**/shelf 15 | 16 | # Generated files 17 | .idea/**/contentModel.xml 18 | 19 | # Sensitive or high-churn files 20 | .idea/**/dataSources/ 21 | .idea/**/dataSources.ids 22 | .idea/**/dataSources.local.xml 23 | .idea/**/sqlDataSources.xml 24 | .idea/**/dynamic.xml 25 | .idea/**/uiDesigner.xml 26 | .idea/**/dbnavigator.xml 27 | 28 | # Gradle 29 | .idea/**/gradle.xml 30 | .idea/**/libraries 31 | 32 | .idea/ 33 | 34 | # Gradle and Maven with auto-import 35 | # When using Gradle or Maven with auto-import, you should exclude module files, 36 | # since they will be recreated, and may cause churn. Uncomment if using 37 | # auto-import. 38 | # .idea/modules.xml 39 | # .idea/*.iml 40 | # .idea/modules 41 | 42 | # CMake 43 | cmake-build-*/ 44 | 45 | # Mongo Explorer plugin 46 | .idea/**/mongoSettings.xml 47 | 48 | # File-based project format 49 | *.iws 50 | 51 | # IntelliJ 52 | out/ 53 | 54 | # mpeltonen/sbt-idea plugin 55 | .idea_modules/ 56 | 57 | # JIRA plugin 58 | atlassian-ide-plugin.xml 59 | 60 | # Cursive Clojure plugin 61 | .idea/replstate.xml 62 | 63 | # Crashlytics plugin (for Android Studio and IntelliJ) 64 | com_crashlytics_export_strings.xml 65 | crashlytics.properties 66 | crashlytics-build.properties 67 | fabric.properties 68 | 69 | # Editor-based Rest Client 70 | .idea/httpRequests 71 | 72 | # Android studio 3.1+ serialized cache file 73 | .idea/caches/build_file_checksums.ser 74 | 75 | ### Intellij Patch ### 76 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 77 | 78 | # *.iml 79 | # modules.xml 80 | # .idea/misc.xml 81 | # *.ipr 82 | 83 | # Sonarlint plugin 84 | .idea/sonarlint 85 | 86 | ### Java ### 87 | # Compiled class file 88 | *.class 89 | 90 | # Log file 91 | *.log 92 | 93 | # BlueJ files 94 | *.ctxt 95 | 96 | # Mobile Tools for Java (J2ME) 97 | .mtj.tmp/ 98 | 99 | # Package Files # 100 | *.jar 101 | *.war 102 | *.nar 103 | *.ear 104 | *.zip 105 | *.tar.gz 106 | *.rar 107 | 108 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 109 | hs_err_pid* 110 | 111 | ### Kotlin ### 112 | # Compiled class file 113 | 114 | # Log file 115 | 116 | # BlueJ files 117 | 118 | # Mobile Tools for Java (J2ME) 119 | 120 | # Package Files # 121 | 122 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 123 | 124 | ### Gradle ### 125 | .gradle 126 | build/ 127 | 128 | # Ignore Gradle GUI config 129 | gradle-app.setting 130 | 131 | # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) 132 | !gradle-wrapper.jar 133 | 134 | # Cache of project 135 | .gradletasknamecache 136 | 137 | # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 138 | # gradle/wrapper/gradle-wrapper.properties 139 | 140 | ### Gradle Patch ### 141 | **/build/ 142 | 143 | # End of https://www.gitignore.io/api/java,kotlin,gradle,intellij 144 | ### macOS ### 145 | # General 146 | .DS_Store 147 | .AppleDouble 148 | .LSOverride 149 | 150 | # Icon must end with two \r 151 | Icon 152 | 153 | # Thumbnails 154 | ._* 155 | 156 | # Files that might appear in the root of a volume 157 | .DocumentRevisions-V100 158 | .fseventsd 159 | .Spotlight-V100 160 | .TemporaryItems 161 | .Trashes 162 | .VolumeIcon.icns 163 | .com.apple.timemachine.donotpresent 164 | 165 | # Directories potentially created on remote AFP share 166 | .AppleDB 167 | .AppleDesktop 168 | Network Trash Folder 169 | Temporary Items 170 | .apdisk 171 | 172 | # End of https://www.toptal.com/developers/gitignore/api/macos 173 | 174 | # ignore gpg keys 175 | *.gpg 176 | 177 | 178 | -------------------------------------------------------------------------------- /.java-version: -------------------------------------------------------------------------------- 1 | graalvm64-11.0.8 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | jobs: 2 | include: 3 | - stage: "Tests" 4 | os: linux 5 | language: java 6 | jdk: 7 | - openjdk11 8 | script: ./gradlew clean test 9 | 10 | - stage: "Deploy Jar and Native Linux application and Docker" 11 | if: tag IS present 12 | os: 13 | - linux 14 | language: java 15 | jdk: 16 | - openjdk11 17 | script: ./gradlew clean shadowJar buildNativeApplication 18 | deploy: 19 | - provider: releases 20 | api_key: $GITHUB_KEY 21 | file_glob: true 22 | skip_cleanup: true 23 | file: 24 | - "*/build/libs/kafka-consumer-lag-monitoring-*.jar" 25 | - "*/build/dist/kafka-consumer-lag-monitoring-*.tar.gz" 26 | on: 27 | tags: true 28 | 29 | - provider: script 30 | script: ./gradlew jib 31 | skip_cleanup: true 32 | on: 33 | tags: true 34 | 35 | - provider: script 36 | script: ./gradlew buildAndPushNativeApplicationDockerImage 37 | skip_cleanup: true 38 | on: 39 | tags: true 40 | 41 | 42 | - stage: "Deploy Native Mac application" 43 | if: tag IS present 44 | os: osx 45 | osx_image: xcode9.4 46 | language: java 47 | jdk: 48 | - openjdk11 49 | script: ./gradlew clean buildNativeApplication 50 | deploy: 51 | - provider: releases 52 | api_key: $GITHUB_KEY 53 | file_glob: true 54 | skip_cleanup: true 55 | file: 56 | - "*/build/dist/kafka-consumer-lag-monitoring-*.tar.gz" 57 | on: 58 | tags: true 59 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at omarsmak@gmail.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /HOW_TO_PUBLISH_TO_MAVEN.md: -------------------------------------------------------------------------------- 1 | # How to publish this project to Maven Nexus repository 2 | 3 | This project has moved from JCenter to Maven Central (Nexus repository), therefore publishing this project has now different workflow. To publish, make sure to follow these steps based on this [blog](https://proandroiddev.com/publishing-a-maven-artifact-3-3-step-by-step-instructions-to-mavencentral-publishing-bd661081645d): 4 | 5 | 1. Configure the the library version in `build.gradle`. 6 | 7 | 1. Create a GPG key: 8 | ``` 9 | > gpg --full-generate-key 10 | > gpg --export-secret-keys 1D9BB9FE > secert-keys.gpg 11 | 12 | # upload your public key to a public server so that sonatype can find it. If that doesn't work, you can also upload manually at http://keys.gnupg.net:11371/ 13 | > gpg --keyserver hkp://pool.sks-keyservers.net --send-keys 1D9BB9FE 14 | ``` 15 | 16 | 1. Configure your credentials in your `~/.gradle/gradle.properties` file: 17 | 18 | ``` 19 | signing.keyId=[gpg key id] 20 | signing.password=[gpg key password] 21 | signing.secretKeyRingFile=[gpg key file location to `secert-keys.gpg`] 22 | 23 | ossrhUsername=[sonatype username e.g: JIRA username] 24 | ossrhPassword=[sonatype password e.g: JIRA password] 25 | ``` 26 | 27 | 1. Upload the artifacts: 28 | ``` 29 | ./gradlew uploadArchives 30 | ``` -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Omar Al-Safi 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Kafka Consumer Lag Monitoring - Lightweight and Cloud Native Ready 2 | ==== 3 | [![Build Status](https://travis-ci.com/omarsmak/kafka-consumer-lag-monitoring.svg?token=ACVRSYGMw5EM3tmwPiBz&branch=master)](https://travis-ci.com/omarsmak/kafka-consumer-lag-monitoring) 4 | ![Download](https://maven-badges.herokuapp.com/maven-central/com.omarsmak.kafka/consumer-lag-monitoring/badge.svg) 5 | 6 | 7 | A client tool that exports the consumer lag of a Kafka consumer group to different implementations such as Prometheus or your terminal. It utlizes Kafka's AdminClient and Kafka's Consumer's client in order to fetch such 8 | metrics. 9 | Consumer lag calculated as follows: 10 | 11 | sum(topic_offset_per_partition-consumer_offset_per_partition) 12 | 13 | 14 | #### What is Consumer Lag and why is important? 15 | Quoting this [article](https://sematext.com/blog/kafka-consumer-lag-offsets-monitoring/): 16 | > What is Kafka Consumer Lag? Kafka Consumer Lag is the indicator of how much lag there is between Kafka producers and consumers.... 17 | 18 | > Why is Consumer Lag Important? Many applications today are based on being able to process (near) real-time data. Think about performance monitoring system like Sematext Monitoring or log management service like Sematext Logs. They continuously process infinite streams of near real-time data. If they were to show you metrics or logs with too much delay – if the Consumer Lag were too big – they’d be nearly useless. This Consumer Lag tells us how far behind each Consumer (Group) is in each Partition. **The smaller the lag the more real-time the data consumption**. 19 | 20 | In summary, consumer lag tells us 2 things: 21 | * The closer the lag to 0, the more confidence we are on processing messages nearer to real-time. Therefore, it _could_ indicate that our consumers are processing messages in a healthy manner. 22 | * The further the lag from 0, the less confidence we are on processing messages nearer to real-time. Therefore, it _could_ indicate that our consumers are not processing messages in a healthy manner. 23 | 24 | ### Supported Kafka Versions 25 | Since this client uses Kafka Admin Client and Kafka Consumer client version of *2+*, therefore this client supportes Kafka brokers from version **0.10.2+**. 26 | 27 | ## Features 28 | * Rich metrics that show detailed consumer lags on both levels, on the **consumer group level** and on the **consumer member level** for more granularity. 29 | * Metrics are available for both, **console and Prometheus**. 30 | * **Very fast** due to the *native compilation* by GraalVM Native Image. 31 | * Highly configurable through either properties configurations or through environment variables. 32 | * Configurable logging through log4j, currently supports JSON as well the standard logging. 33 | * Ready to use thin Docker images either for *Jar* or *native* application for your cloud deployments such as **Kubernetes**. 34 | * The tool is also available as **maven package** in case you want to be embedded it into your application. 35 | 36 | ## Changelog 37 | #### 0.1.1 38 | - Issue #29: Publish the artifacts in Maven Central instead of bintray 39 | - Update Kafka clients to version `2.8.0`. 40 | #### 0.1.0 41 | **Major Release:** 42 | - Issue #27: Refactor the client in order to minimize the usage of dependencies and remove any reflections. 43 | - Issue #24: Support native compilation via GraalVM Native Image. 44 | - Issue #15: Configurable log4j support for either JSON or standard logging. 45 | - Issue #14: Support of configurations through environment variables. 46 | - Update all the dependencies to the latest version. 47 | #### 0.0.8: 48 | - Issue #23: Extend Lag stats on consumer member level. 49 | - Issue #20: Support consumer group and topic deletion on the fly. 50 | - Issue #21: Change default port to 9739 51 | #### 0.0.7: 52 | - Issue #17: Now this client will show newly joined consumer groups as well **without the need to restart the client**. You should start it once and it will always refresh the consumer groups list according to the poll interval. 53 | - Kafka client updated to version 2.5.0. 54 | 55 | #### 0.0.6: 56 | - Issue #8: Support configuration file as parameter 57 | - Kafka client updated to version 2.4.1. 58 | 59 | 60 | ## Installation and Usage 61 | #### Native Application 62 | You can downland the latest release of the Native application from [here](https://github.com/omarsmak/kafka-consumer-lag-monitoring/releases), currently it only supports **Mac** and **Linux**. An example from Prometheus component: 63 | ``` 64 | ./kafka-consumer-lag-monitoring-prometheus-0.1.0 config.properties 65 | ``` 66 | 67 | **Note to Mac users**: You will need to verify the application, to do this, run: 68 | ``` 69 | xattr -r -d com.apple.quarantine kafka-consumer-lag-monitoring-prometheus-0.1.0 70 | ``` 71 | 72 | #### Uber JAR 73 | You can downland the latest release of the Uber JAR from [here](https://github.com/omarsmak/kafka-consumer-lag-monitoring/releases). This client requires at least Java 8 in order to run. You can run it like this for example from Console component: 74 | ``` 75 | java -jar kafka-consumer-lag-monitoring-console-0.1.0-all.jar -b kafka1:9092,kafka2:9092,kafka3:9092 -c "my_awesome_consumer_group_01" -p 5000 76 | ``` 77 | 78 | #### Docker 79 | There two types of docker images: 80 | 1. Docker images based on the **native application**: This docker image is built using the natively compiled application, the benefit is, you will get **faster** and **small** image which 81 | is beneficial for your cloud native environment. However, since the native compilation is pretty new to this client, is still an evolving work. 82 | 2. Docker images based on the **Uber Jar**: This docker image is built using the uber Jar. Although it might be slower and larger, it is the more stable than the Docker native images but is still optimized to run in container orchestration frameworks 83 | such as kubernetes as efficient as possible. 84 | 85 | Example: 86 | ``` 87 | docker run omarsmak/kafka-consumer-lag-monitoring-prometheus-native -p 9739:9739 \ 88 | -e kafka_bootstrap_servers=localhost:9092 \ 89 | -e kafka_retry_backoff.ms = 200 \ 90 | -e monitoring_lag_consumer_groups="test*" \ 91 | -e monitoring_lag_prometheus_http_port=9739 \ 92 | -e monitoring_lag_logging_rootLogger_appenderRef_stdout_ref=LogToConsole \ 93 | -e monitoring_lag_logging_rootLogger_level=info 94 | ``` 95 | 96 | ## Usage 97 | ### Console Component: 98 | This mode will print the consumer lag per partition and the total lag among all partitions and continuously refreshing the metrics per the value of `--poll.interval` startup parameter. It accepts the following parameters: 99 | ``` 100 | ./kafka-consumer-lag-monitoring-console-0.1.0 -h 101 | 130 ↵ omaral-safi@Omars-MBP-2 102 | Usage: kafka-consumer-lag-monitoring-console [-hV] [-b=] 103 | [-c=] [-f=] [-p=] 104 | Prints the kafka consumer lag to the console. 105 | -b, --bootstrap.servers= 106 | A list of host/port pairs to use for establishing the initial 107 | connection to the Kafka cluster 108 | -c, --consumer.groups= 109 | A list of Kafka consumer groups or list ending with star (*) 110 | to fetch all consumers with matching pattern, e.g: 'test_v*' 111 | -f, --properties.file= 112 | Optional. Properties file for Kafka AdminClient 113 | configurations, this is the typical Kafka properties file 114 | that can be used in the AdminClient. For more info, please 115 | take a look at Kafka AdminClient configurations 116 | documentation. 117 | -h, --help Show this help message and exit. 118 | -p, --poll.interval= 119 | Interval delay in ms to that refreshes the client lag 120 | metrics, default to 2000ms 121 | -V, --version Print version information and exit. 122 | ``` 123 | 124 | An example output: 125 | ``` 126 | ./kafka-consumer-lag-monitoring-console-0.1.0 -b kafka1:9092,kafka2:9092,kafka3:9092 -c "my_awesome_consumer_group_01" -p 5000 127 | Consumer group: my_awesome_consumer_group_01 128 | ============================================================================== 129 | 130 | Topic name: topic_example_1 131 | Total topic offsets: 211132248 132 | Total consumer offsets: 187689403 133 | Total lag: 23442845 134 | 135 | Topic name: topic_example_2 136 | Total topic offsets: 15763247 137 | Total consumer offsets: 15024564 138 | Total lag: 738683 139 | 140 | Topic name: topic_example_3 141 | Total topic offsets: 392 142 | Total consumer offsets: 392 143 | Total lag: 0 144 | 145 | Topic name: topic_example_4 146 | Total topic offsets: 24572 147 | Total consumer offsets: 24570 148 | Total lag: 2 149 | 150 | Topic name: topic_example_5 151 | Total topic offsets: 430 152 | Total consumer offsets: 430 153 | Total lag: 0 154 | 155 | Topic name: topic_example_6 156 | Total topic offsets: 6342 157 | Total consumer offsets: 6335 158 | Total lag: 7 159 | ``` 160 | 161 | ##### Example Usage Native Application: 162 | ``` 163 | ./kafka-consumer-lag-monitoring-console-0.1.0 -c "test*" -b localhost:9092 -p 500 164 | ``` 165 | 166 | ##### Example Usage Uber Jar Application: 167 | ``` 168 | java -jar kafka-consumer-lag-monitoring-console-0.1.0-all.jar -c "test*" -b localhost:9092 -p 500 169 | ``` 170 | 171 | ##### Example Usage Docker Native Application: 172 | ``` 173 | docker run omarsmak/kafka-consumer-lag-monitoring-console-native -c "test*" -b localhost:9092 -p 500 174 | ``` 175 | 176 | ##### Example Usage Docker Uber Jar Application: 177 | ``` 178 | docker run omarsmak/kafka-consumer-lag-monitoring-console -c "test*" -b localhost:9092 -p 500 179 | ``` 180 | 181 | ### Prometheus Component: 182 | In this mode, the tool will start an http server on a port that being set in `monitoring.lag.prometheus.http.port` config and it will expose an endpoint that is reachable via `localhost:/metrics` or `localhost:/prometheus` 183 | so prometheus server can scrap these metrics and expose them for example to grafana. You will need to pass the configuration as properties file or via environment variables. An example config file: 184 | ``` 185 | kafka.bootstrap.servers=localhost:9092 186 | kafka.retry.backoff.ms = 200 187 | monitoring.lag.consumer.groups=group-a-*,*-b-*,*-c 188 | monitoring.lag.consumer.groups.exclude=*test* 189 | monitoring.lag.prometheus.http.port=9772 190 | monitoring.lag.logging.rootLogger.appenderRef.stdout.ref=LogToConsole 191 | monitoring.lag.logging.rootLogger.level=info 192 | ``` 193 | This will include consumer groups that either start with `group-a-`, contain `-b-`, or end with `-c`, excluding those containing `test`. 194 | 195 | You can then run it like the following: 196 | ##### Example Usage Native Application: 197 | ``` 198 | ./kafka-consumer-lag-monitoring-prometheus-0.1.0 config.proprties 199 | ``` 200 | 201 | ##### Example Usage Uber Jar Application: 202 | ``` 203 | java -jar kafka-consumer-lag-monitoring-prometheus-0.1.0-all.jar config.proprties 204 | ``` 205 | 206 | ##### Example Usage Docker Native Application: 207 | For Docker, we will use the environment variables instead: 208 | ``` 209 | docker run omarsmak/kafka-consumer-lag-monitoring-prometheus-native -p 9739:9739 \ 210 | -e kafka_bootstrap_servers=localhost:9092 \ 211 | -e kafka_retry_backoff.ms = 200 \ 212 | -e monitoring_lag_consumer_groups="test*" \ 213 | -e monitoring_lag_prometheus_http_port=9739 \ 214 | -e monitoring_lag_logging_rootLogger_appenderRef_stdout_ref=LogToConsole \ 215 | -e monitoring_lag_logging_rootLogger_level=info 216 | ``` 217 | 218 | ##### Example Usage Docker Uber Jar Application: 219 | For Docker, we will use the environment variables instead: 220 | ``` 221 | docker run omarsmak/kafka-consumer-lag-monitoring-prometheus -p 9739:9739 \ 222 | -e kafka_bootstrap_servers=localhost:9092 \ 223 | -e kafka_retry_backoff.ms = 200 \ 224 | -e monitoring_lag_consumer_groups="test*" \ 225 | -e monitoring_lag_prometheus_http_port=9739 \ 226 | -e monitoring_lag_logging_rootLogger_appenderRef_stdout_ref=LogToConsole \ 227 | -e monitoring_lag_logging_rootLogger_level=info 228 | ``` 229 | 230 | **Note:** By default, port `9739` is exposed by the docker image, hence you **should avoid** overrding the client's HTTP port through the client's startup arguments (`--http.port`) as described below when you run the client through docker container and leave it to the default of `9739`. However you can still change the corresponding docker mapped port to anything of your choice. 231 | 232 | ##### Exposed Metrics: 233 | ##### `kafka_consumer_group_offset{group, topic, partition}` 234 | The latest committed offset of a consumer group in a given partition of a topic. 235 | 236 | ##### `kafka_consumer_group_partition_lag{group, topic, partition}` 237 | The lag of a consumer group behind the head of a given partition of a topic. Calculated like this: `current_topic_offset_per_partition - current_consumer_offset_per_partition`. 238 | 239 | ##### `kafka_topic_latest_offsets{group, topic, partition}` 240 | The latest committed offset of a topic in a given partition. 241 | 242 | ##### `kafka_consumer_group_total_lag{group, topic}` 243 | The total lag of a consumer group behind the head of a topic. This gives the total lags from all partitions over each topic, it provides good visibility but not a precise measurement since is not partition aware. 244 | 245 | ##### `kafka_consumer_group_member_lag{group, member, topic}` 246 | The total lag of a consumer group member behind the head of a topic. This gives the total lags over each consumer member within consumer group. 247 | 248 | ##### `kafka_consumer_group_member_partition_lag{group, member, topic, partition}` 249 | The lag of a consumer member within consumer group behind the head of a given partition of a topic. 250 | 251 | 252 | #### Configuration 253 | Majority of the components here, for example `Prometheus` components supports two types of configurations: 254 | 1. **Application Properties File**: You can provide the application a config properties file as argument e.g: `./kafka-consumer-lag-monitoring-prometheus-0.1.0 config.properties`, this is an example config: 255 | 256 | ``` 257 | kafka.bootstrap.servers=localhost:9092 258 | kafka.retry.backoff.ms = 200 259 | monitoring.lag.consumer.groups=test* 260 | monitoring.lag.prometheus.http.port=9772 261 | monitoring.lag.logging.rootLogger.appenderRef.stdout.ref=LogToConsole 262 | monitoring.lag.logging.rootLogger.level=info 263 | ``` 264 | Note here the application accepts configs with two prefixes: 265 | - `kafka.`: Use the `kafka` prefix for any config related to Kafka admin client, these configs are basically the same configs that you will find here: https://kafka.apache.org/documentation/#adminclientconfigs/. 266 | - `monitoring.lag.` : Use the `monitoring.lag` prefix to pass any config specific to this client, you will take a look which configs that the client will accept later. 267 | 268 | 2. **Environment Variables**: You can as well pass the configs as environment variables, this is useful when running the application in environment like Docker, for example: 269 | 270 | ``` 271 | docker run --rm -p 9739:9739 \ 272 | -e monitoring_lag_logging_rootLogger_appenderRef_stdout_ref=LogToConsole \ 273 | -e monitoring_lag_consumer_groups="test-*" \ 274 | -e kafka_bootstrap_servers=host.docker.internal:9092 \ 275 | omarsmak/kafka-consumer-lag-monitoring-prometheus-native:latest 276 | ``` 277 | Similar to the application properties file, it supports `kafka` and `monitoring.lag`. However, you will need to replace all dot `.` with underscore `_` for all the configs, for example the config `kafka.bootstrap.servers` its environment equivalent is `kafka_bootstrap_servers`. 278 | 279 | #### Available Configurations 280 | - `monitoring.lag.consumer.groups` : A list of Kafka consumer groups or list ending with star (\*\) to fetch all consumers with matching pattern, e.g: `test_v*`. 281 | - `monitoring.lag.poll.interval` : Interval delay in ms to that refreshes the client lag metrics, default to 2000ms. 282 | - `monitoring.lag.prometheus.http.port` : Http port that is used to expose metrics in case, default to 9739. 283 | 284 | 285 | #### Logging 286 | The client ships with Log4j bindings and supports JSON and standard logging. The default log4j properties that it uses: 287 | ``` 288 | # Log to console 289 | appender.console.type = Console 290 | appender.console.name = LogToConsole 291 | appender.console.layout.type = PatternLayout 292 | appender.console.layout.pattern = [%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n 293 | 294 | # Log to console as JSON 295 | appender.json.type = Console 296 | appender.json.name = LogInJSON 297 | appender.json.layout.type = JsonLayout 298 | appender.json.layout.complete = true 299 | appender.json.layout.compact = false 300 | 301 | rootLogger.level = info 302 | rootLogger.appenderRef.stdout.ref = LogInJSON 303 | ``` 304 | By default, `LogInJSON` is enabled. However, you can customtize all of this by providing these configurations prefixed with `monitoring.lag.logging.`. For example, to enable the standard logging, you will need to 305 | add this config `monitoring.lag.logging.rootLogger.appenderRef.stdout.ref=LogToConsole` or as environment variable: `monitoring_lag_logging_rootLogger_appenderRef_stdout_ref=LogToConsole`. 306 | 307 | **Note**: When configuring the logging through the environment variables, note that the configuration are **case sensitive**. 308 | 309 | 310 | ## Usage as Library 311 | If you want to use this client embedded into your application, you can achieve that by adding a dependency to this tool in your `pom.xml` or `gradle.build` as explained below: 312 | #### Maven 313 | ``` 314 | 315 | com.omarsmak.kafka 316 | consumer-lag-monitoring 317 | 0.1.1 318 | 319 | ``` 320 | 321 | #### Gradle 322 | ``` 323 | compile 'com.omarsmak.kafka:consumer-lag-monitoring:0.1.1' 324 | 325 | ``` 326 | 327 | ### Usage 328 | #### Java 329 | ``` 330 | import com.omarsmak.kafka.consumer.lag.monitoring.client.KafkaConsumerLagClient; 331 | import com.omarsmak.kafka.consumer.lag.monitoring.client.KafkaConsumerLagClientFactory; 332 | import org.apache.kafka.clients.admin.AdminClientConfig; 333 | 334 | import java.util.Properties; 335 | 336 | public class ConsumerLagClientTest { 337 | 338 | public static void main(String[] args){ 339 | // Create a Properties object to hold the Kafka bootstrap servers 340 | final Properties properties = new Properties(); 341 | properties.setProperty(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "kafka1:9092"); 342 | 343 | // Create the client, we will use the Java client 344 | final KafkaConsumerLagClient kafkaConsumerLagClient = KafkaConsumerLagClientFactory.create(properties); 345 | 346 | // Print the lag of a Kafka consumer 347 | System.out.println(kafkaConsumerLagClient.getConsumerLag("awesome-consumer")); 348 | } 349 | } 350 | ``` 351 | 352 | #### Kotlin 353 | ``` 354 | import com.omarsmak.kafka.consumer.lag.monitoring.client.KafkaConsumerLagClientFactory 355 | import org.apache.kafka.clients.admin.AdminClientConfig 356 | import java.util.Properties 357 | 358 | object ConsumerLagClientTest { 359 | 360 | @JvmStatic 361 | fun main(arg: Array) { 362 | // Create a Properties object to hold the Kafka bootstrap servers 363 | val properties = Properties().apply { 364 | this[AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG] = "kafka1:9092" 365 | } 366 | 367 | // Create the client, we will use the Kafka AdminClient Java client 368 | val kafkaConsumerLagClient = KafkaConsumerLagClientFactory.create(properties) 369 | 370 | // Print the lag of a Kafka consumer 371 | println(kafkaConsumerLagClient.getConsumerLag("awesome-consumer")) 372 | } 373 | } 374 | ``` 375 | 376 | ## Build The Project 377 | Run `./gradlew clean build` on the top project folder which is as result, it will run all tests and build the Uber jar. 378 | 379 | 380 | ## Project Sponsors 381 | [![Alt text](./jetbrains.svg)](https://www.jetbrains.com/?from=kafka-consumer-lag-monitoring) 382 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | buildscript { 2 | dependencies { 3 | classpath("org.jfrog.buildinfo:build-info-extractor-gradle:${jfrogBuildExtractorVersion}") 4 | } 5 | repositories { 6 | mavenCentral() 7 | jcenter() 8 | } 9 | } 10 | 11 | plugins { 12 | id "org.jetbrains.kotlin.jvm" version "${kotlinVersion}" 13 | id "com.adarshr.test-logger" version "${testLoggerPluginVersion}" apply false 14 | id 'com.github.johnrengelman.shadow' version "${shadowPluginVersion}" apply false 15 | id "com.palantir.graal" version "${palantirGraalNativeVersion}" apply false 16 | id 'com.bmuschko.docker-remote-api' version "${bmuschkoDockerPluginVersion}" apply false 17 | id 'com.google.cloud.tools.jib' version "${jibPluginVersion}" apply false 18 | } 19 | 20 | apply plugin: 'idea' 21 | 22 | subprojects { 23 | apply plugin: 'com.adarshr.test-logger' 24 | apply plugin: 'kotlin' 25 | 26 | group = "com.omarsmak.kafka.consumer.lag.monitoring" 27 | version = '0.1.3' 28 | 29 | ext { 30 | libraryName = "kafka-consumer-lag-monitoring" 31 | libraryVersion = version 32 | } 33 | 34 | idea { 35 | module { 36 | downloadJavadoc = true 37 | downloadSources = true 38 | } 39 | } 40 | 41 | compileKotlin { 42 | kotlinOptions.jvmTarget = "1.8" 43 | } 44 | 45 | compileTestKotlin { 46 | kotlinOptions.jvmTarget = "1.8" 47 | } 48 | 49 | test { 50 | useJUnitPlatform() 51 | } 52 | 53 | dependencies { 54 | compile "io.github.microutils:kotlin-logging:$kotlinLoggingVersion" 55 | 56 | testImplementation "org.junit.jupiter:junit-jupiter-api:${junitVersion}" 57 | testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${junitVersion}" 58 | testCompile "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" 59 | } 60 | } 61 | 62 | 63 | import com.bmuschko.gradle.docker.tasks.image.Dockerfile 64 | import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage 65 | import com.bmuschko.gradle.docker.tasks.image.DockerTagImage 66 | import com.bmuschko.gradle.docker.tasks.image.DockerPushImage 67 | 68 | // these tasks only applicable for any component 69 | configure(subprojects.findAll { it.name.contains("component") }) { 70 | apply plugin: 'com.github.johnrengelman.shadow' 71 | apply plugin: 'com.palantir.graal' 72 | apply plugin: 'com.bmuschko.docker-remote-api' 73 | apply plugin: 'com.google.cloud.tools.jib' 74 | 75 | afterEvaluate { 76 | ext { 77 | componentFullName = libraryName + "-" + project.componentName 78 | componentFullNameWithVersion = componentFullName + "-" + version 79 | componentDockerName = "omarsmak/" + componentFullName 80 | componentDockerNativeName = componentDockerName + "-native" 81 | } 82 | 83 | shadowJar { 84 | zip64 = true 85 | archiveBaseName.set(componentFullName) 86 | } 87 | 88 | jar { 89 | manifest { 90 | attributes( 91 | "Multi-Release": true 92 | ) 93 | } 94 | } 95 | 96 | graal { 97 | graalVersion graalNativeImageVersion 98 | outputName componentFullNameWithVersion 99 | mainClass project.mainClassName 100 | javaVersion '11' 101 | option '--no-fallback' 102 | option '--report-unsupported-elements-at-runtime' 103 | option '--allow-incomplete-classpath' 104 | } 105 | 106 | docker { 107 | registryCredentials { 108 | username = System.getenv('DOCKERHUB_USER') 109 | password = System.getenv('DOCKERHUB_PASSWORD') 110 | email = 'omarsmak@gmail.com' 111 | } 112 | } 113 | 114 | jib { 115 | from.image 'gcr.io/distroless/java:11' 116 | to{ 117 | image = componentDockerName 118 | tags = [libraryVersion, "latest"] 119 | auth { 120 | username = System.getenv('DOCKERHUB_USER') 121 | password = System.getenv('DOCKERHUB_PASSWORD') 122 | } 123 | } 124 | container { 125 | if (dockerExposedPort) { 126 | ports = [dockerExposedPort.toString()] 127 | } 128 | 129 | jvmFlags = [ 130 | "-XX:+UseCompressedOops", 131 | "-XX:MaxRAMPercentage=80", 132 | "-Dfile.encoding=UTF-8", 133 | "-Djava.security.egd=file:/dev/./urandom" 134 | ] 135 | } 136 | } 137 | 138 | task createNativeDockerfile(type: Dockerfile) { 139 | 140 | // First stage: build our native image inside a docker image 141 | 142 | from new Dockerfile.From("ghcr.io/graalvm/graalvm-ce:${graalNativeImageVersion}").withStage("build") 143 | workingDir "/app" 144 | copyFile componentFullName + "*all.jar", "/app/jar/application.jar" 145 | runCommand "gu install native-image" 146 | runCommand "native-image -jar jar/application.jar --no-fallback --report-unsupported-elements-at-runtime --allow-incomplete-classpath" 147 | 148 | // Second stage: Create our application docker image 149 | from "frolvlad/alpine-glibc" 150 | runCommand "apk update && apk add libstdc++" 151 | workingDir "/work/" 152 | copyFile new Dockerfile.CopyFile("/app/application", "/work/application").withStage("build") 153 | 154 | if (project.dockerExposedPort) { 155 | exposePort project.dockerExposedPort 156 | } 157 | 158 | entryPoint("./application") 159 | } 160 | 161 | task copyShadowJarToDockerContext(type: Copy) { 162 | dependsOn shadowJar 163 | 164 | from "$buildDir/libs" 165 | include componentFullName + "*all.jar" 166 | into "$buildDir/docker" 167 | } 168 | 169 | /** 170 | * Build a Docker image with native application 171 | */ 172 | task buildNativeApplicationDockerImage(type: DockerBuildImage) { 173 | dependsOn copyShadowJarToDockerContext, createNativeDockerfile 174 | 175 | remove = true 176 | noCache = false 177 | images.add(componentDockerNativeName + ":latest") 178 | images.add(componentDockerNativeName + ":$version") 179 | } 180 | 181 | /** 182 | * Build and push a Docker image with native application 183 | */ 184 | task buildAndPushNativeApplicationDockerImage(type: DockerPushImage) { 185 | dependsOn buildNativeApplicationDockerImage 186 | 187 | images.set(buildNativeApplicationDockerImage.images) 188 | } 189 | 190 | /** 191 | * Build a native application using Graal Native Image 192 | */ 193 | task buildNativeApplication(type: Tar) { 194 | dependsOn nativeImage 195 | 196 | OperatingSystem os = org.gradle.nativeplatform.platform.internal.DefaultNativePlatform.currentOperatingSystem; 197 | 198 | def osName = "generic" 199 | 200 | if (os.linux) { 201 | osName = "linux" 202 | } else if (os.macOsX) { 203 | osName = "mac" 204 | } else if (os.windows) { 205 | osName = "windows" 206 | } 207 | 208 | archiveFileName = "$componentFullNameWithVersion-${osName}.tar.gz" 209 | destinationDirectory = file("$buildDir/dist") 210 | compression = Compression.GZIP 211 | 212 | from "$buildDir/graal/$componentFullNameWithVersion" 213 | } 214 | 215 | /** 216 | * Build a Docker image with jar application 217 | */ 218 | task buildDockerImage { 219 | dependsOn jibDockerBuild 220 | } 221 | 222 | /** 223 | * Build and push a Docker image with jar application 224 | */ 225 | task buildAndPushDockerImage { 226 | dependsOn jib 227 | } 228 | } 229 | } 230 | 231 | allprojects { 232 | repositories { 233 | mavenCentral() 234 | jcenter() 235 | } 236 | } 237 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | kotlinVersion = 1.4.10 2 | kotlinLoggingVersion = 1.12.0 3 | kafkaVersion = 2.8.0 4 | junitVersion = 5.5.2 5 | prometheusVersion = 0.8.0 6 | jfrogBuildExtractorVersion = 4.5.4 7 | picocliVersion = 4.5.2 8 | mordantVersion = 1.2.1 9 | slf4jVersion = 1.7.30 10 | log4jSlf4jImplVersion = 2.17.1 11 | mockkVersion = 1.9.3 12 | graalNativeImageVersion = 21.1.0 13 | jacksonDatabindVersion = 2.11.3 14 | 15 | 16 | #plugins version 17 | shadowPluginVersion = 5.2.0 18 | testLoggerPluginVersion = 2.1.1 19 | palantirGitVersion = 0.12.2 20 | palantirGraalNativeVersion = 0.7.1 21 | jfrogBintrayVersion = 1.8.4 22 | jibPluginVersion = 1.8.0 23 | bmuschkoDockerPluginVersion = 6.6.1 24 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/omarsmak/kafka-consumer-lag-monitoring/29f82989f24be2fd952d99e554920fd2f1d4bbc3/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-all.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS="" 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn () { 37 | echo "$*" 38 | } 39 | 40 | die () { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save () { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /jetbrains.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 9 | 10 | 11 | 14 | 15 | 16 | 17 | 18 | 19 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 31 | 32 | 33 | 34 | 35 | 36 | 39 | 40 | 41 | 42 | 43 | 45 | 47 | 48 | 51 | 54 | 56 | 57 | 59 | 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /monitoring-client/bin/main/com/omarsmak/kafka/consumer/lag/monitoring/client/KafkaConsumerLagClient.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.client 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.client.data.Lag 4 | import com.omarsmak.kafka.consumer.lag.monitoring.client.data.Offsets 5 | 6 | /** 7 | * Interface for KafkaOffsetClient public API 8 | * 9 | * @author oalsafi 10 | */ 11 | 12 | interface KafkaConsumerLagClient : AutoCloseable { 13 | /** 14 | * Return consumers groups list 15 | */ 16 | fun getConsumerGroupsList(): List 17 | 18 | /** 19 | * Return current offsets for a consumer 20 | */ 21 | fun getConsumerOffsets(consumerGroup: String): List 22 | 23 | /** 24 | * Return current offsets for consumer members of a consumer group 25 | */ 26 | fun getConsumerGroupMembersOffsets(consumerGroup: String): Map> 27 | 28 | /** 29 | * Return topic offset per partition 30 | */ 31 | fun getTopicOffsets(topicName: String): Offsets 32 | 33 | /** 34 | * Return consumer lag per topic, the way to calculate this as follows: 35 | * lag = current_topic_offset - current_consumer_offset 36 | */ 37 | fun getConsumerLag(consumerGroup: String): List 38 | 39 | /** 40 | * Return consumer lag per member per topic, the way to calculate this as follows: 41 | * lag = current_topic_offset - current_consumer_offset 42 | */ 43 | fun getConsumerMemberLag(consumerGroup: String): Map> 44 | } 45 | -------------------------------------------------------------------------------- /monitoring-client/bin/main/com/omarsmak/kafka/consumer/lag/monitoring/client/KafkaConsumerLagClientFactory.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.client 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.client.impl.KafkaConsumerLagJavaClient 4 | import java.util.Properties 5 | import org.apache.kafka.clients.admin.AdminClient 6 | import org.apache.kafka.clients.consumer.KafkaConsumer 7 | import org.apache.kafka.common.serialization.Serdes 8 | 9 | /** 10 | * Main KafkaOffset factory, this should be the entry 11 | * 12 | * @author oalsafi 13 | */ 14 | object KafkaConsumerLagClientFactory { 15 | @JvmStatic 16 | fun create(prop: Properties): KafkaConsumerLagClient = createJavaClient(prop) 17 | 18 | @JvmStatic 19 | fun create(map: Map): KafkaConsumerLagClient = createJavaClient(Properties().apply { 20 | putAll(map) 21 | }) 22 | 23 | private fun createJavaClient(prop: Properties): KafkaConsumerLagJavaClient { 24 | val adminClient = AdminClient.create(prop) 25 | val consumerClient = KafkaConsumer(prop, Serdes.String().deserializer(), Serdes.String().deserializer()) 26 | 27 | return KafkaConsumerLagJavaClient(adminClient, consumerClient) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /monitoring-client/bin/main/com/omarsmak/kafka/consumer/lag/monitoring/client/data/entities.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.client.data 2 | 3 | /** 4 | * Entities representations 5 | * 6 | * @author oalsafi 7 | */ 8 | 9 | data class Offsets( 10 | val topicName: String, 11 | val offsetPerPartition: Map 12 | ) 13 | 14 | data class Lag( 15 | val topicName: String, 16 | val totalLag: Long, 17 | val lagPerPartition: Map, 18 | val latestTopicOffsets: Map, 19 | val latestConsumerOffsets: Map 20 | ) 21 | -------------------------------------------------------------------------------- /monitoring-client/bin/main/com/omarsmak/kafka/consumer/lag/monitoring/client/exceptions/KafkaConsumerLagClientException.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.client.exceptions 2 | 3 | /** 4 | * Indicates some error while performing an operation 5 | * 6 | * @author oalsafi 7 | */ 8 | open class KafkaConsumerLagClientException : RuntimeException { 9 | constructor(message: String, cause: Throwable) : super(message, cause) 10 | 11 | constructor(message: String) : super(message) 12 | 13 | constructor(cause: Throwable) : super(cause) 14 | 15 | constructor() : super() 16 | } 17 | -------------------------------------------------------------------------------- /monitoring-client/bin/main/com/omarsmak/kafka/consumer/lag/monitoring/client/impl/AbstractKafkaConsumerLagClient.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("MaximumLineLength", "MaxLineLength") 2 | 3 | package com.omarsmak.kafka.consumer.lag.monitoring.client.impl 4 | 5 | import com.omarsmak.kafka.consumer.lag.monitoring.client.KafkaConsumerLagClient 6 | import com.omarsmak.kafka.consumer.lag.monitoring.client.data.Lag 7 | import com.omarsmak.kafka.consumer.lag.monitoring.client.data.Offsets 8 | import com.omarsmak.kafka.consumer.lag.monitoring.client.exceptions.KafkaConsumerLagClientException 9 | import org.apache.kafka.clients.consumer.KafkaConsumer 10 | import org.apache.kafka.common.TopicPartition 11 | 12 | /** 13 | * Base client class 14 | * 15 | * @author oalsafi 16 | */ 17 | 18 | internal abstract class AbstractKafkaConsumerLagClient( 19 | private val kafkaConsumerClient: KafkaConsumer 20 | ) : KafkaConsumerLagClient { 21 | 22 | protected abstract fun closeClients() 23 | 24 | override fun getTopicOffsets(topicName: String): Offsets { 25 | val partitions = kafkaConsumerClient.partitionsFor(topicName).orEmpty() 26 | if (partitions.isEmpty()) throw KafkaConsumerLagClientException("Topic `$topicName` does not exist in the Kafka cluster.") 27 | val topicPartition = partitions.map { 28 | TopicPartition(it.topic(), it.partition()) 29 | } 30 | 31 | val topicOffsetsMap = kafkaConsumerClient.endOffsets(topicPartition).map { 32 | it.key.partition() to it.value 33 | }.toMap() 34 | 35 | return Offsets(topicName, topicOffsetsMap) 36 | } 37 | 38 | override fun getConsumerLag(consumerGroup: String): List { 39 | val consumerOffsets = getConsumerOffsets(consumerGroup) 40 | return consumerOffsets.map { 41 | getConsumerLagPerTopic(it) 42 | } 43 | } 44 | 45 | override fun close() { 46 | kafkaConsumerClient.wakeup() 47 | closeClients() 48 | } 49 | 50 | override fun getConsumerMemberLag(consumerGroup: String): Map> { 51 | val consumerMemberOffsets = getConsumerGroupMembersOffsets(consumerGroup) 52 | 53 | return consumerMemberOffsets.mapValues { it -> it.value.map { getConsumerLagPerTopic(it) } } 54 | } 55 | 56 | private fun getConsumerLagPerTopic(consumerOffsets: Offsets): Lag { 57 | val topicOffsets = getTopicOffsets(consumerOffsets.topicName) 58 | 59 | val lagPerPartitionAndTotalLag = calculateLagPerPartitionAndTotalLag(topicOffsets, consumerOffsets) 60 | val lagPerPartition = lagPerPartitionAndTotalLag.first 61 | val totalLag = lagPerPartitionAndTotalLag.second 62 | 63 | return Lag( 64 | topicOffsets.topicName, 65 | totalLag, 66 | lagPerPartition, 67 | topicOffsets.offsetPerPartition, 68 | consumerOffsets.offsetPerPartition 69 | ) 70 | } 71 | 72 | private fun calculateLagPerPartitionAndTotalLag(topicOffsets: Offsets, consumerOffsets: Offsets): Pair, Long>{ 73 | var totalLag = 0L 74 | val lagPerPartition = consumerOffsets.offsetPerPartition.map { (k, v) -> 75 | val lag = topicOffsets.offsetPerPartition.getValue(k) - v 76 | totalLag += lag 77 | k to lag 78 | }.toMap() 79 | 80 | return (lagPerPartition to totalLag) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /monitoring-client/bin/main/com/omarsmak/kafka/consumer/lag/monitoring/client/impl/KafkaConsumerLagJavaClient.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("MaximumLineLength", "MaxLineLength") 2 | 3 | package com.omarsmak.kafka.consumer.lag.monitoring.client.impl 4 | 5 | import com.omarsmak.kafka.consumer.lag.monitoring.client.data.Offsets 6 | import com.omarsmak.kafka.consumer.lag.monitoring.client.exceptions.KafkaConsumerLagClientException 7 | import org.apache.kafka.clients.admin.AdminClient 8 | import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsOptions 9 | import org.apache.kafka.clients.consumer.KafkaConsumer 10 | import org.apache.kafka.clients.consumer.OffsetAndMetadata 11 | import org.apache.kafka.common.TopicPartition 12 | 13 | /** 14 | * An abstraction over Kafka Java clients 15 | * 16 | * @author oalsafi 17 | */ 18 | 19 | internal class KafkaConsumerLagJavaClient( 20 | private val javaAdminClient: AdminClient, 21 | kafkaConsumerClient: KafkaConsumer 22 | ) : AbstractKafkaConsumerLagClient(kafkaConsumerClient) { 23 | 24 | override fun getConsumerGroupsList(): List { 25 | val consumerList = javaAdminClient.listConsumerGroups().all().get().map { it.groupId() } 26 | if (consumerList.isEmpty()) throw KafkaConsumerLagClientException("No consumers existing in the Kafka cluster.") 27 | return consumerList 28 | } 29 | 30 | override fun getConsumerOffsets(consumerGroup: String): List { 31 | val offsets = javaAdminClient.listConsumerGroupOffsets(consumerGroup) 32 | .partitionsToOffsetAndMetadata() 33 | .get() 34 | ?: throw KafkaConsumerLagClientException("Consumer group `$consumerGroup` does not exist in the Kafka cluster.") 35 | 36 | return getConsumerOffsetsPerTopic(offsets) 37 | } 38 | 39 | override fun getConsumerGroupMembersOffsets(consumerGroup: String): Map> { 40 | // to get offsets per member, we need to do the following: 41 | // 1. fetch consumer members for a consumer group 42 | // 2. fetch group offsets for consumer members once we have the topic partition 43 | val consumerGroupMembersEnvelope = javaAdminClient.describeConsumerGroups(listOf(consumerGroup)) 44 | .all() 45 | .get() 46 | ?: throw KafkaConsumerLagClientException("Consumer group `$consumerGroup` does not exist in the Kafka cluster.") 47 | 48 | val consumerGroupMembers = consumerGroupMembersEnvelope.values.firstOrNull() 49 | ?: throw KafkaConsumerLagClientException("Consumer group `$consumerGroup` and its members does not exist in the Kafka cluster.") 50 | 51 | return consumerGroupMembers 52 | .members() 53 | .map { it.consumerId() to getConsumerOffsetPerMember(consumerGroup, it.assignment().topicPartitions().toList()) } 54 | .toMap() 55 | } 56 | 57 | private fun getConsumerOffsetPerMember(consumerGroup: String, topicPartition: List): List { 58 | val offsets = javaAdminClient.listConsumerGroupOffsets(consumerGroup, ListConsumerGroupOffsetsOptions().topicPartitions(topicPartition)) 59 | .partitionsToOffsetAndMetadata() 60 | .get() 61 | ?: throw KafkaConsumerLagClientException("Consumer group `$consumerGroup` does not exist in the Kafka cluster.") 62 | 63 | return getConsumerOffsetsPerTopic(offsets) 64 | } 65 | 66 | private fun getConsumerOffsetsPerTopic(offsets: Map): List { 67 | val rawOffsets = mutableMapOf>() 68 | offsets.filterValues { it != null } 69 | .forEach { (t, u) -> 70 | // First we get the key of the topic 71 | val offsetPerPartition = rawOffsets.getOrPut(t.topic()) { mutableMapOf() } 72 | // Add the updated map 73 | offsetPerPartition.putIfAbsent(t.partition(), u!!.offset()) 74 | rawOffsets.replace(t.topic(), offsetPerPartition) 75 | } 76 | 77 | return rawOffsets.map { 78 | Offsets(it.key, it.value) 79 | } 80 | } 81 | 82 | 83 | override fun closeClients() { 84 | javaAdminClient.close() 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /monitoring-client/bin/main/config/consumer-monitoring.properties: -------------------------------------------------------------------------------- 1 | bootstrap.servers=test:9092 -------------------------------------------------------------------------------- /monitoring-client/bin/test/com/omarsmak/kafka/consumer/lag/monitoring/client/impl/KafkaConsumerLagJavaClientTest.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.client.impl 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.client.KafkaConsumerLagClient 4 | import com.omarsmak.kafka.consumer.lag.monitoring.client.KafkaConsumerLagClientFactory 5 | import com.omarsmak.kafka.consumer.lag.monitoring.client.exceptions.KafkaConsumerLagClientException 6 | import io.mockk.every 7 | import io.mockk.mockk 8 | import io.mockk.mockkClass 9 | import org.apache.kafka.clients.admin.AdminClient 10 | import org.apache.kafka.clients.admin.AdminClientConfig 11 | import org.apache.kafka.clients.admin.ConsumerGroupListing 12 | import org.apache.kafka.clients.consumer.KafkaConsumer 13 | import org.apache.kafka.clients.consumer.OffsetAndMetadata 14 | import org.apache.kafka.common.Node 15 | import org.apache.kafka.common.PartitionInfo 16 | import org.apache.kafka.common.TopicPartition 17 | import org.junit.jupiter.api.Assertions.* 18 | import org.junit.jupiter.api.BeforeAll 19 | import org.junit.jupiter.api.Disabled 20 | import org.junit.jupiter.api.Test 21 | import org.junit.jupiter.api.TestInstance 22 | import java.util.* 23 | import kotlin.random.Random 24 | import kotlin.test.assertFailsWith 25 | 26 | @TestInstance(TestInstance.Lifecycle.PER_CLASS) 27 | internal class KafkaConsumerLagJavaClientTest { 28 | 29 | private val TEST_TOPIC_NAME = "test-topic" 30 | private val INVALID_TOPIC_NAME = "blah_blah_topic" 31 | private val TEST_CONSUMER_NAME = "consumer-test" 32 | private val INVALID_TEST_CONSUMER_NAME = "blah_blah_consumer" 33 | 34 | private lateinit var kafkaOffsetClient: KafkaConsumerLagClient 35 | 36 | // Mocks 37 | private lateinit var adminClient: AdminClient 38 | private lateinit var kafkaConsumerClient: KafkaConsumer 39 | 40 | @BeforeAll 41 | @Suppress("UNCHECKED_CAST") 42 | fun initialize() { 43 | // Mocking classes 44 | adminClient = mockkClass(AdminClient::class) 45 | kafkaConsumerClient = mockkClass(KafkaConsumer::class) as KafkaConsumer 46 | 47 | mockClasses() 48 | 49 | // Initialize admin client with injected mocks 50 | kafkaOffsetClient = KafkaConsumerLagJavaClient(adminClient, kafkaConsumerClient) 51 | } 52 | 53 | private fun mockClasses(){ 54 | every { adminClient.listConsumerGroups().all().get() } returns mockGroupListings(10) 55 | 56 | every { adminClient.listConsumerGroupOffsets(TEST_CONSUMER_NAME) 57 | .partitionsToOffsetAndMetadata() 58 | .get() } returns mapOf(mockTopicPartition(0, TEST_TOPIC_NAME).first() to OffsetAndMetadata(1L)) 59 | 60 | every { adminClient.listConsumerGroupOffsets(INVALID_TEST_CONSUMER_NAME) 61 | .partitionsToOffsetAndMetadata() 62 | .get() } returns null 63 | 64 | every { kafkaConsumerClient.partitionsFor(TEST_TOPIC_NAME) } returns mockListPartitionInfo(9, TEST_TOPIC_NAME) 65 | every { kafkaConsumerClient.partitionsFor(INVALID_TOPIC_NAME) } returns null 66 | every { kafkaConsumerClient.endOffsets(mockTopicPartition(9, TEST_TOPIC_NAME))} returns mockTopicPartitionToEndOffset(9, TEST_TOPIC_NAME) 67 | } 68 | 69 | private fun mockGroupListings(numOfGroups: Int): List { 70 | val resultList = mutableListOf() 71 | for (i in 0 .. numOfGroups){ 72 | resultList.add(ConsumerGroupListing("group-$i", true)) 73 | } 74 | return resultList 75 | } 76 | 77 | private fun mockTopicPartition(numPartition: Int, topicName: String): List { 78 | val resultList = mutableListOf() 79 | for (i in 0 .. numPartition){ 80 | resultList.add(TopicPartition(topicName, i)) 81 | } 82 | return resultList 83 | } 84 | 85 | private fun mockListPartitionInfo(numPartition: Int, topicName: String): List { 86 | val resultList = mutableListOf() 87 | for (i in 0 .. numPartition) { 88 | resultList.add(PartitionInfo( 89 | topicName, 90 | i, 91 | mockArrayNodes().first(), 92 | mockArrayNodes(), 93 | mockArrayNodes() 94 | )) 95 | } 96 | return resultList 97 | } 98 | 99 | private fun mockArrayNodes() = arrayOf( Node(1,"node-1",1234), Node(2,"node-2",1234)) 100 | 101 | private fun mockTopicPartitionToEndOffset(numPartition: Int, topicName: String): Map { 102 | val topicPartition = mockTopicPartition(numPartition, topicName) 103 | return topicPartition.map { 104 | it to Random.nextLong() 105 | }.toMap() 106 | } 107 | 108 | @Test 109 | fun `it should return all consumer groups`(){ 110 | val consumerGroups = kafkaOffsetClient.getConsumerGroupsList() 111 | assertTrue(consumerGroups.isNotEmpty()) 112 | } 113 | 114 | @Test 115 | fun `it should return all the consumer offsets per partition`() { 116 | val consumerOffsets = kafkaOffsetClient.getConsumerOffsets(TEST_CONSUMER_NAME) 117 | 118 | assertTrue(consumerOffsets.isNotEmpty()) 119 | 120 | // Test if it throws error if it does not find an existing consumer group in Kafka 121 | assertFailsWith(KafkaConsumerLagClientException::class) { 122 | kafkaOffsetClient.getConsumerOffsets(INVALID_TEST_CONSUMER_NAME) 123 | } 124 | } 125 | 126 | @Test 127 | fun `it should return all the latest topic offsets per partition`() { 128 | val numPartitions = 10 129 | val topicOffsets = kafkaOffsetClient.getTopicOffsets(TEST_TOPIC_NAME) 130 | 131 | assertEquals(topicOffsets.topicName, TEST_TOPIC_NAME) 132 | assertEquals(topicOffsets.offsetPerPartition.size, numPartitions) 133 | 134 | // Test if it throws error if it does not find an existing topic in Kafka 135 | assertFailsWith(KafkaConsumerLagClientException::class) { 136 | kafkaOffsetClient.getTopicOffsets(INVALID_TOPIC_NAME) 137 | } 138 | } 139 | 140 | @Test 141 | fun `it should return the consumer lag per partition`() { 142 | val consumerLag = kafkaOffsetClient.getConsumerLag(TEST_CONSUMER_NAME) 143 | 144 | assertTrue(consumerLag.isNotEmpty()) 145 | assertNotNull(consumerLag.first().totalLag) 146 | assertTrue(consumerLag.first().lagPerPartition.isNotEmpty()) 147 | assertTrue(consumerLag.first().latestConsumerOffsets.isNotEmpty()) 148 | assertTrue(consumerLag.first().latestTopicOffsets.isNotEmpty()) 149 | 150 | // Test if it throws error if it does not find an existing consumer group in Kafka 151 | assertFailsWith(KafkaConsumerLagClientException::class) { 152 | kafkaOffsetClient.getConsumerLag(INVALID_TEST_CONSUMER_NAME) 153 | } 154 | } 155 | } -------------------------------------------------------------------------------- /monitoring-client/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'maven' 3 | id 'signing' 4 | } 5 | 6 | ext { 7 | bintrayRepo = "kafka" 8 | bintrayName = "consumer-lag-monitoring" 9 | 10 | publishingGroupId = "com.omarsmak.kafka" 11 | publishingArtifactId = "consumer-lag-monitoring" 12 | 13 | libraryDescription = "Client tool that exports the consumer lag of Kafka consumer groups to Prometheus or your terminal" 14 | sitUrl = "https://github.com/omarsmak/kafka-consumer-lag-monitoring" 15 | gitUrl = "https://github.com/omarsmak/kafka-consumer-lag-monitoring.git" 16 | developerId = "omarsmak" 17 | developerName = "Omar Al-Safi" 18 | developerEmail = "omarsmak@gmail.com" 19 | licenceName = "The MIT License" 20 | licenseUrl = "https://opensource.org/licenses/MIT" 21 | libraryLicences = ['MIT'] 22 | } 23 | 24 | def ossrhUsernameValue = hasProperty('ossrhUsername') ? ossrhUsername : System.getenv('ossrhUsername') 25 | def ossrhPasswordValue = hasProperty('ossrhPassword') ? ossrhPassword : System.getenv('ossrhPassword') 26 | 27 | task sourceJar(type: Jar) { 28 | classifier "sources" 29 | from sourceSets.main.allSource 30 | } 31 | 32 | javadoc.failOnError = false 33 | 34 | task javadocJar(type: Jar, dependsOn: javadoc) { 35 | classifier = 'javadoc' 36 | from javadoc.destinationDir 37 | } 38 | 39 | artifacts { 40 | archives sourceJar 41 | archives javadocJar 42 | } 43 | 44 | signing { 45 | sign configurations.archives 46 | } 47 | 48 | tasks.withType(Sign) { 49 | onlyIf { gradle.taskGraph.hasTask(":uploadArchives") } 50 | } 51 | 52 | 53 | dependencies { 54 | compile "org.apache.kafka:kafka-clients:$kafkaVersion" 55 | 56 | testImplementation "org.junit.jupiter:junit-jupiter-api:${junitVersion}" 57 | testImplementation "io.mockk:mockk:${mockkVersion}" 58 | testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${junitVersion}" 59 | testCompile "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" 60 | } 61 | 62 | // Create the publication with the pom configuration: 63 | uploadArchives { 64 | repositories { 65 | mavenDeployer { 66 | beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } 67 | 68 | repository(url: "https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/") { 69 | authentication(userName: ossrhUsernameValue, password: ossrhPasswordValue) 70 | } 71 | 72 | snapshotRepository(url: "https://s01.oss.sonatype.org/content/repositories/snapshots/") { 73 | authentication(userName: ossrhUsernameValue, password: ossrhPasswordValue) 74 | } 75 | 76 | pom.project { 77 | name libraryName 78 | groupId publishingGroupId 79 | artifactId publishingArtifactId 80 | version libraryVersion 81 | packaging 'jar' 82 | description libraryDescription 83 | url sitUrl 84 | 85 | scm { 86 | url gitUrl 87 | } 88 | 89 | licenses { 90 | license { 91 | name licenceName 92 | url licenseUrl 93 | } 94 | } 95 | 96 | developers { 97 | developer { 98 | id developerId 99 | name developerName 100 | email developerEmail 101 | } 102 | } 103 | } 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /monitoring-client/src/main/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/client/KafkaConsumerLagClient.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.client 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.client.data.Lag 4 | import com.omarsmak.kafka.consumer.lag.monitoring.client.data.Offsets 5 | 6 | /** 7 | * Interface for KafkaOffsetClient public API 8 | * 9 | * @author oalsafi 10 | */ 11 | 12 | interface KafkaConsumerLagClient : AutoCloseable { 13 | /** 14 | * Return consumers groups list 15 | */ 16 | fun getConsumerGroupsList(): List 17 | 18 | /** 19 | * Return current offsets for a consumer 20 | */ 21 | fun getConsumerOffsets(consumerGroup: String): List 22 | 23 | /** 24 | * Return current offsets for consumer members of a consumer group 25 | */ 26 | fun getConsumerGroupMembersOffsets(consumerGroup: String): Map> 27 | 28 | /** 29 | * Return topic offset per partition 30 | */ 31 | fun getTopicOffsets(topicName: String): Offsets 32 | 33 | /** 34 | * Return consumer lag per topic, the way to calculate this as follows: 35 | * lag = current_topic_offset - current_consumer_offset 36 | */ 37 | fun getConsumerLag(consumerGroup: String): List 38 | 39 | /** 40 | * Return consumer lag per member per topic, the way to calculate this as follows: 41 | * lag = current_topic_offset - current_consumer_offset 42 | */ 43 | fun getConsumerMemberLag(consumerGroup: String): Map> 44 | } 45 | -------------------------------------------------------------------------------- /monitoring-client/src/main/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/client/KafkaConsumerLagClientFactory.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.client 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.client.impl.KafkaConsumerLagJavaClient 4 | import java.util.Properties 5 | import org.apache.kafka.clients.admin.AdminClient 6 | import org.apache.kafka.clients.consumer.KafkaConsumer 7 | import org.apache.kafka.common.serialization.Serdes 8 | 9 | /** 10 | * Main KafkaOffset factory, this should be the entry 11 | * 12 | * @author oalsafi 13 | */ 14 | object KafkaConsumerLagClientFactory { 15 | @JvmStatic 16 | fun create(prop: Properties): KafkaConsumerLagClient = createJavaClient(prop) 17 | 18 | @JvmStatic 19 | fun create(map: Map): KafkaConsumerLagClient = createJavaClient(Properties().apply { 20 | putAll(map) 21 | }) 22 | 23 | private fun createJavaClient(prop: Properties): KafkaConsumerLagJavaClient { 24 | val adminClient = AdminClient.create(prop) 25 | val consumerClient = KafkaConsumer(prop, Serdes.String().deserializer(), Serdes.String().deserializer()) 26 | 27 | return KafkaConsumerLagJavaClient(adminClient, consumerClient) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /monitoring-client/src/main/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/client/data/entities.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.client.data 2 | 3 | /** 4 | * Entities representations 5 | * 6 | * @author oalsafi 7 | */ 8 | 9 | data class Offsets( 10 | val topicName: String, 11 | val offsetPerPartition: Map 12 | ) 13 | 14 | data class Lag( 15 | val topicName: String, 16 | val totalLag: Long, 17 | val lagPerPartition: Map, 18 | val latestTopicOffsets: Map, 19 | val latestConsumerOffsets: Map 20 | ) 21 | -------------------------------------------------------------------------------- /monitoring-client/src/main/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/client/exceptions/KafkaConsumerLagClientException.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.client.exceptions 2 | 3 | /** 4 | * Indicates some error while performing an operation 5 | * 6 | * @author oalsafi 7 | */ 8 | open class KafkaConsumerLagClientException : RuntimeException { 9 | constructor(message: String, cause: Throwable) : super(message, cause) 10 | 11 | constructor(message: String) : super(message) 12 | 13 | constructor(cause: Throwable) : super(cause) 14 | 15 | constructor() : super() 16 | } 17 | -------------------------------------------------------------------------------- /monitoring-client/src/main/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/client/impl/AbstractKafkaConsumerLagClient.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("MaximumLineLength", "MaxLineLength") 2 | 3 | package com.omarsmak.kafka.consumer.lag.monitoring.client.impl 4 | 5 | import com.omarsmak.kafka.consumer.lag.monitoring.client.KafkaConsumerLagClient 6 | import com.omarsmak.kafka.consumer.lag.monitoring.client.data.Lag 7 | import com.omarsmak.kafka.consumer.lag.monitoring.client.data.Offsets 8 | import com.omarsmak.kafka.consumer.lag.monitoring.client.exceptions.KafkaConsumerLagClientException 9 | import org.apache.kafka.clients.consumer.KafkaConsumer 10 | import org.apache.kafka.common.TopicPartition 11 | 12 | /** 13 | * Base client class 14 | * 15 | * @author oalsafi 16 | */ 17 | 18 | internal abstract class AbstractKafkaConsumerLagClient( 19 | private val kafkaConsumerClient: KafkaConsumer 20 | ) : KafkaConsumerLagClient { 21 | 22 | protected abstract fun closeClients() 23 | 24 | override fun getTopicOffsets(topicName: String): Offsets { 25 | val partitions = kafkaConsumerClient.partitionsFor(topicName).orEmpty() 26 | if (partitions.isEmpty()) throw KafkaConsumerLagClientException("Topic `$topicName` does not exist in the Kafka cluster.") 27 | val topicPartition = partitions.map { 28 | TopicPartition(it.topic(), it.partition()) 29 | } 30 | 31 | val topicOffsetsMap = kafkaConsumerClient.endOffsets(topicPartition).map { 32 | it.key.partition() to it.value 33 | }.toMap() 34 | 35 | return Offsets(topicName, topicOffsetsMap) 36 | } 37 | 38 | override fun getConsumerLag(consumerGroup: String): List { 39 | val consumerOffsets = getConsumerOffsets(consumerGroup) 40 | return consumerOffsets.map { 41 | getConsumerLagPerTopic(it) 42 | } 43 | } 44 | 45 | override fun close() { 46 | kafkaConsumerClient.wakeup() 47 | closeClients() 48 | } 49 | 50 | override fun getConsumerMemberLag(consumerGroup: String): Map> { 51 | val consumerMemberOffsets = getConsumerGroupMembersOffsets(consumerGroup) 52 | 53 | return consumerMemberOffsets.mapValues { it -> it.value.map { getConsumerLagPerTopic(it) } } 54 | } 55 | 56 | private fun getConsumerLagPerTopic(consumerOffsets: Offsets): Lag { 57 | val topicOffsets = getTopicOffsets(consumerOffsets.topicName) 58 | 59 | val lagPerPartitionAndTotalLag = calculateLagPerPartitionAndTotalLag(topicOffsets, consumerOffsets) 60 | val lagPerPartition = lagPerPartitionAndTotalLag.first 61 | val totalLag = lagPerPartitionAndTotalLag.second 62 | 63 | return Lag( 64 | topicOffsets.topicName, 65 | totalLag, 66 | lagPerPartition, 67 | topicOffsets.offsetPerPartition, 68 | consumerOffsets.offsetPerPartition 69 | ) 70 | } 71 | 72 | private fun calculateLagPerPartitionAndTotalLag(topicOffsets: Offsets, consumerOffsets: Offsets): Pair, Long>{ 73 | var totalLag = 0L 74 | val lagPerPartition = consumerOffsets.offsetPerPartition.map { (k, v) -> 75 | val lag = topicOffsets.offsetPerPartition.getValue(k) - v 76 | totalLag += lag 77 | k to lag 78 | }.toMap() 79 | 80 | return (lagPerPartition to totalLag) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /monitoring-client/src/main/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/client/impl/KafkaConsumerLagJavaClient.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("MaximumLineLength", "MaxLineLength") 2 | 3 | package com.omarsmak.kafka.consumer.lag.monitoring.client.impl 4 | 5 | import com.omarsmak.kafka.consumer.lag.monitoring.client.data.Offsets 6 | import com.omarsmak.kafka.consumer.lag.monitoring.client.exceptions.KafkaConsumerLagClientException 7 | import org.apache.kafka.clients.admin.AdminClient 8 | import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsOptions 9 | import org.apache.kafka.clients.consumer.KafkaConsumer 10 | import org.apache.kafka.clients.consumer.OffsetAndMetadata 11 | import org.apache.kafka.common.TopicPartition 12 | 13 | /** 14 | * An abstraction over Kafka Java clients 15 | * 16 | * @author oalsafi 17 | */ 18 | 19 | internal class KafkaConsumerLagJavaClient( 20 | private val javaAdminClient: AdminClient, 21 | kafkaConsumerClient: KafkaConsumer 22 | ) : AbstractKafkaConsumerLagClient(kafkaConsumerClient) { 23 | 24 | override fun getConsumerGroupsList(): List { 25 | val consumerList = javaAdminClient.listConsumerGroups().all().get().map { it.groupId() } 26 | if (consumerList.isEmpty()) throw KafkaConsumerLagClientException("No consumers existing in the Kafka cluster.") 27 | return consumerList 28 | } 29 | 30 | override fun getConsumerOffsets(consumerGroup: String): List { 31 | val offsets = javaAdminClient.listConsumerGroupOffsets(consumerGroup) 32 | .partitionsToOffsetAndMetadata() 33 | .get() 34 | ?: throw KafkaConsumerLagClientException("Consumer group `$consumerGroup` does not exist in the Kafka cluster.") 35 | 36 | return getConsumerOffsetsPerTopic(offsets) 37 | } 38 | 39 | override fun getConsumerGroupMembersOffsets(consumerGroup: String): Map> { 40 | // to get offsets per member, we need to do the following: 41 | // 1. fetch consumer members for a consumer group 42 | // 2. fetch group offsets for consumer members once we have the topic partition 43 | val consumerGroupMembersEnvelope = javaAdminClient.describeConsumerGroups(listOf(consumerGroup)) 44 | .all() 45 | .get() 46 | ?: throw KafkaConsumerLagClientException("Consumer group `$consumerGroup` does not exist in the Kafka cluster.") 47 | 48 | val consumerGroupMembers = consumerGroupMembersEnvelope.values.firstOrNull() 49 | ?: throw KafkaConsumerLagClientException("Consumer group `$consumerGroup` and its members does not exist in the Kafka cluster.") 50 | 51 | return consumerGroupMembers 52 | .members() 53 | .map { it.consumerId() to getConsumerOffsetPerMember(consumerGroup, it.assignment().topicPartitions().toList()) } 54 | .toMap() 55 | } 56 | 57 | private fun getConsumerOffsetPerMember(consumerGroup: String, topicPartition: List): List { 58 | val offsets = javaAdminClient.listConsumerGroupOffsets(consumerGroup, ListConsumerGroupOffsetsOptions().topicPartitions(topicPartition)) 59 | .partitionsToOffsetAndMetadata() 60 | .get() 61 | ?: throw KafkaConsumerLagClientException("Consumer group `$consumerGroup` does not exist in the Kafka cluster.") 62 | 63 | return getConsumerOffsetsPerTopic(offsets) 64 | } 65 | 66 | private fun getConsumerOffsetsPerTopic(offsets: Map): List { 67 | val rawOffsets = mutableMapOf>() 68 | offsets.filterValues { it != null } 69 | .forEach { (t, u) -> 70 | // First we get the key of the topic 71 | val offsetPerPartition = rawOffsets.getOrPut(t.topic()) { mutableMapOf() } 72 | // Add the updated map 73 | offsetPerPartition.putIfAbsent(t.partition(), u!!.offset()) 74 | rawOffsets.replace(t.topic(), offsetPerPartition) 75 | } 76 | 77 | return rawOffsets.map { 78 | Offsets(it.key, it.value) 79 | } 80 | } 81 | 82 | 83 | override fun closeClients() { 84 | javaAdminClient.close() 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /monitoring-client/src/main/resources/config/consumer-monitoring.properties: -------------------------------------------------------------------------------- 1 | bootstrap.servers=test:9092 -------------------------------------------------------------------------------- /monitoring-client/src/test/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/client/impl/KafkaConsumerLagJavaClientTest.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.client.impl 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.client.KafkaConsumerLagClient 4 | import com.omarsmak.kafka.consumer.lag.monitoring.client.KafkaConsumerLagClientFactory 5 | import com.omarsmak.kafka.consumer.lag.monitoring.client.exceptions.KafkaConsumerLagClientException 6 | import io.mockk.every 7 | import io.mockk.mockk 8 | import io.mockk.mockkClass 9 | import org.apache.kafka.clients.admin.AdminClient 10 | import org.apache.kafka.clients.admin.AdminClientConfig 11 | import org.apache.kafka.clients.admin.ConsumerGroupListing 12 | import org.apache.kafka.clients.consumer.KafkaConsumer 13 | import org.apache.kafka.clients.consumer.OffsetAndMetadata 14 | import org.apache.kafka.common.Node 15 | import org.apache.kafka.common.PartitionInfo 16 | import org.apache.kafka.common.TopicPartition 17 | import org.junit.jupiter.api.Assertions.* 18 | import org.junit.jupiter.api.BeforeAll 19 | import org.junit.jupiter.api.Disabled 20 | import org.junit.jupiter.api.Test 21 | import org.junit.jupiter.api.TestInstance 22 | import java.util.* 23 | import kotlin.random.Random 24 | import kotlin.test.assertFailsWith 25 | 26 | @TestInstance(TestInstance.Lifecycle.PER_CLASS) 27 | internal class KafkaConsumerLagJavaClientTest { 28 | 29 | private val TEST_TOPIC_NAME = "test-topic" 30 | private val INVALID_TOPIC_NAME = "blah_blah_topic" 31 | private val TEST_CONSUMER_NAME = "consumer-test" 32 | private val INVALID_TEST_CONSUMER_NAME = "blah_blah_consumer" 33 | 34 | private lateinit var kafkaOffsetClient: KafkaConsumerLagClient 35 | 36 | // Mocks 37 | private lateinit var adminClient: AdminClient 38 | private lateinit var kafkaConsumerClient: KafkaConsumer 39 | 40 | @BeforeAll 41 | @Suppress("UNCHECKED_CAST") 42 | fun initialize() { 43 | // Mocking classes 44 | adminClient = mockkClass(AdminClient::class) 45 | kafkaConsumerClient = mockkClass(KafkaConsumer::class) as KafkaConsumer 46 | 47 | mockClasses() 48 | 49 | // Initialize admin client with injected mocks 50 | kafkaOffsetClient = KafkaConsumerLagJavaClient(adminClient, kafkaConsumerClient) 51 | } 52 | 53 | private fun mockClasses(){ 54 | every { adminClient.listConsumerGroups().all().get() } returns mockGroupListings(10) 55 | 56 | every { adminClient.listConsumerGroupOffsets(TEST_CONSUMER_NAME) 57 | .partitionsToOffsetAndMetadata() 58 | .get() } returns mapOf(mockTopicPartition(0, TEST_TOPIC_NAME).first() to OffsetAndMetadata(1L)) 59 | 60 | every { adminClient.listConsumerGroupOffsets(INVALID_TEST_CONSUMER_NAME) 61 | .partitionsToOffsetAndMetadata() 62 | .get() } returns null 63 | 64 | every { kafkaConsumerClient.partitionsFor(TEST_TOPIC_NAME) } returns mockListPartitionInfo(9, TEST_TOPIC_NAME) 65 | every { kafkaConsumerClient.partitionsFor(INVALID_TOPIC_NAME) } returns null 66 | every { kafkaConsumerClient.endOffsets(mockTopicPartition(9, TEST_TOPIC_NAME))} returns mockTopicPartitionToEndOffset(9, TEST_TOPIC_NAME) 67 | } 68 | 69 | private fun mockGroupListings(numOfGroups: Int): List { 70 | val resultList = mutableListOf() 71 | for (i in 0 .. numOfGroups){ 72 | resultList.add(ConsumerGroupListing("group-$i", true)) 73 | } 74 | return resultList 75 | } 76 | 77 | private fun mockTopicPartition(numPartition: Int, topicName: String): List { 78 | val resultList = mutableListOf() 79 | for (i in 0 .. numPartition){ 80 | resultList.add(TopicPartition(topicName, i)) 81 | } 82 | return resultList 83 | } 84 | 85 | private fun mockListPartitionInfo(numPartition: Int, topicName: String): List { 86 | val resultList = mutableListOf() 87 | for (i in 0 .. numPartition) { 88 | resultList.add(PartitionInfo( 89 | topicName, 90 | i, 91 | mockArrayNodes().first(), 92 | mockArrayNodes(), 93 | mockArrayNodes() 94 | )) 95 | } 96 | return resultList 97 | } 98 | 99 | private fun mockArrayNodes() = arrayOf( Node(1,"node-1",1234), Node(2,"node-2",1234)) 100 | 101 | private fun mockTopicPartitionToEndOffset(numPartition: Int, topicName: String): Map { 102 | val topicPartition = mockTopicPartition(numPartition, topicName) 103 | return topicPartition.map { 104 | it to Random.nextLong() 105 | }.toMap() 106 | } 107 | 108 | @Test 109 | fun `it should return all consumer groups`(){ 110 | val consumerGroups = kafkaOffsetClient.getConsumerGroupsList() 111 | assertTrue(consumerGroups.isNotEmpty()) 112 | } 113 | 114 | @Test 115 | fun `it should return all the consumer offsets per partition`() { 116 | val consumerOffsets = kafkaOffsetClient.getConsumerOffsets(TEST_CONSUMER_NAME) 117 | 118 | assertTrue(consumerOffsets.isNotEmpty()) 119 | 120 | // Test if it throws error if it does not find an existing consumer group in Kafka 121 | assertFailsWith(KafkaConsumerLagClientException::class) { 122 | kafkaOffsetClient.getConsumerOffsets(INVALID_TEST_CONSUMER_NAME) 123 | } 124 | } 125 | 126 | @Test 127 | fun `it should return all the latest topic offsets per partition`() { 128 | val numPartitions = 10 129 | val topicOffsets = kafkaOffsetClient.getTopicOffsets(TEST_TOPIC_NAME) 130 | 131 | assertEquals(topicOffsets.topicName, TEST_TOPIC_NAME) 132 | assertEquals(topicOffsets.offsetPerPartition.size, numPartitions) 133 | 134 | // Test if it throws error if it does not find an existing topic in Kafka 135 | assertFailsWith(KafkaConsumerLagClientException::class) { 136 | kafkaOffsetClient.getTopicOffsets(INVALID_TOPIC_NAME) 137 | } 138 | } 139 | 140 | @Test 141 | fun `it should return the consumer lag per partition`() { 142 | val consumerLag = kafkaOffsetClient.getConsumerLag(TEST_CONSUMER_NAME) 143 | 144 | assertTrue(consumerLag.isNotEmpty()) 145 | assertNotNull(consumerLag.first().totalLag) 146 | assertTrue(consumerLag.first().lagPerPartition.isNotEmpty()) 147 | assertTrue(consumerLag.first().latestConsumerOffsets.isNotEmpty()) 148 | assertTrue(consumerLag.first().latestTopicOffsets.isNotEmpty()) 149 | 150 | // Test if it throws error if it does not find an existing consumer group in Kafka 151 | assertFailsWith(KafkaConsumerLagClientException::class) { 152 | kafkaOffsetClient.getConsumerLag(INVALID_TEST_CONSUMER_NAME) 153 | } 154 | } 155 | } -------------------------------------------------------------------------------- /monitoring-component-console/bin/main/META-INF/native-image/jni-config.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name":"com.sun.management.internal.DiagnosticCommandArgumentInfo", 4 | "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","boolean","boolean","int"] }] 5 | }, 6 | { 7 | "name":"com.sun.management.internal.DiagnosticCommandArgumentInfo[]" 8 | }, 9 | { 10 | "name":"com.sun.management.internal.DiagnosticCommandInfo", 11 | "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","java.util.List"] }] 12 | }, 13 | { 14 | "name":"com.sun.management.internal.DiagnosticCommandInfo[]" 15 | }, 16 | { 17 | "name":"java.lang.ClassLoader", 18 | "methods":[ 19 | {"name":"getPlatformClassLoader","parameterTypes":[] }, 20 | {"name":"loadClass","parameterTypes":["java.lang.String"] } 21 | ] 22 | }, 23 | { 24 | "name":"java.lang.ClassNotFoundException" 25 | }, 26 | { 27 | "name":"java.lang.NoSuchMethodError" 28 | }, 29 | { 30 | "name":"java.util.Arrays", 31 | "methods":[{"name":"asList","parameterTypes":["java.lang.Object[]"] }] 32 | }, 33 | { 34 | "name":"sun.management.VMManagementImpl", 35 | "fields":[ 36 | {"name":"compTimeMonitoringSupport"}, 37 | {"name":"currentThreadCpuTimeSupport"}, 38 | {"name":"objectMonitorUsageSupport"}, 39 | {"name":"otherThreadCpuTimeSupport"}, 40 | {"name":"remoteDiagnosticCommandsSupport"}, 41 | {"name":"synchronizerUsageSupport"}, 42 | {"name":"threadAllocatedMemorySupport"}, 43 | {"name":"threadContentionMonitoringSupport"} 44 | ] 45 | } 46 | ] 47 | -------------------------------------------------------------------------------- /monitoring-component-console/bin/main/META-INF/native-image/proxy-config.json: -------------------------------------------------------------------------------- 1 | [ 2 | ] 3 | -------------------------------------------------------------------------------- /monitoring-component-console/bin/main/META-INF/native-image/reflect-config.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name":"com.omarsmak.kafka.consumer.lag.monitoring.component.console.ClientCli", 4 | "allDeclaredFields":true, 5 | "allDeclaredMethods":true 6 | }, 7 | { 8 | "name":"com.omarsmak.kafka.consumer.lag.monitoring.component.console.VersionProvider", 9 | "allDeclaredFields":true, 10 | "allDeclaredMethods":true, 11 | "methods":[{"name":"","parameterTypes":[] }] 12 | }, 13 | { 14 | "name":"com.sun.management.GarbageCollectorMXBean", 15 | "allPublicMethods":true 16 | }, 17 | { 18 | "name":"com.sun.management.GcInfo", 19 | "allPublicMethods":true 20 | }, 21 | { 22 | "name":"com.sun.management.HotSpotDiagnosticMXBean", 23 | "allPublicMethods":true 24 | }, 25 | { 26 | "name":"com.sun.management.ThreadMXBean", 27 | "allPublicMethods":true 28 | }, 29 | { 30 | "name":"com.sun.management.UnixOperatingSystemMXBean", 31 | "allPublicMethods":true 32 | }, 33 | { 34 | "name":"com.sun.management.VMOption", 35 | "allPublicMethods":true 36 | }, 37 | { 38 | "name":"com.sun.management.internal.GarbageCollectorExtImpl", 39 | "allPublicConstructors":true 40 | }, 41 | { 42 | "name":"com.sun.management.internal.HotSpotDiagnostic", 43 | "allPublicConstructors":true 44 | }, 45 | { 46 | "name":"com.sun.management.internal.HotSpotThreadImpl", 47 | "allPublicConstructors":true 48 | }, 49 | { 50 | "name":"com.sun.management.internal.OperatingSystemImpl", 51 | "allPublicConstructors":true 52 | }, 53 | { 54 | "name":"java.lang.Boolean", 55 | "fields":[{"name":"TYPE"}] 56 | }, 57 | { 58 | "name":"java.lang.Byte", 59 | "fields":[{"name":"TYPE"}] 60 | }, 61 | { 62 | "name":"java.lang.Character", 63 | "fields":[{"name":"TYPE"}] 64 | }, 65 | { 66 | "name":"java.lang.Double", 67 | "fields":[{"name":"TYPE"}] 68 | }, 69 | { 70 | "name":"java.lang.Float", 71 | "fields":[{"name":"TYPE"}] 72 | }, 73 | { 74 | "name":"java.lang.Integer", 75 | "fields":[{"name":"TYPE"}] 76 | }, 77 | { 78 | "name":"java.lang.Long", 79 | "fields":[{"name":"TYPE"}] 80 | }, 81 | { 82 | "name":"java.lang.Object", 83 | "allDeclaredFields":true, 84 | "allDeclaredMethods":true 85 | }, 86 | { 87 | "name":"java.lang.Short", 88 | "fields":[{"name":"TYPE"}] 89 | }, 90 | { 91 | "name":"java.lang.StackTraceElement", 92 | "allPublicMethods":true 93 | }, 94 | { 95 | "name":"java.lang.String" 96 | }, 97 | { 98 | "name":"java.lang.String[]" 99 | }, 100 | { 101 | "name":"java.lang.Void", 102 | "fields":[{"name":"TYPE"}] 103 | }, 104 | { 105 | "name":"java.lang.management.BufferPoolMXBean", 106 | "allPublicMethods":true 107 | }, 108 | { 109 | "name":"java.lang.management.ClassLoadingMXBean", 110 | "allPublicMethods":true 111 | }, 112 | { 113 | "name":"java.lang.management.CompilationMXBean", 114 | "allPublicMethods":true 115 | }, 116 | { 117 | "name":"java.lang.management.LockInfo", 118 | "allPublicMethods":true 119 | }, 120 | { 121 | "name":"java.lang.management.ManagementPermission", 122 | "methods":[{"name":"","parameterTypes":["java.lang.String"] }] 123 | }, 124 | { 125 | "name":"java.lang.management.MemoryMXBean", 126 | "allPublicMethods":true 127 | }, 128 | { 129 | "name":"java.lang.management.MemoryManagerMXBean", 130 | "allPublicMethods":true 131 | }, 132 | { 133 | "name":"java.lang.management.MemoryPoolMXBean", 134 | "allPublicMethods":true 135 | }, 136 | { 137 | "name":"java.lang.management.MemoryUsage", 138 | "allPublicMethods":true 139 | }, 140 | { 141 | "name":"java.lang.management.MonitorInfo", 142 | "allPublicMethods":true 143 | }, 144 | { 145 | "name":"java.lang.management.PlatformLoggingMXBean", 146 | "allPublicMethods":true 147 | }, 148 | { 149 | "name":"java.lang.management.RuntimeMXBean", 150 | "allPublicMethods":true 151 | }, 152 | { 153 | "name":"java.lang.management.ThreadInfo", 154 | "allPublicMethods":true 155 | }, 156 | { 157 | "name":"java.math.BigDecimal" 158 | }, 159 | { 160 | "name":"java.math.BigInteger" 161 | }, 162 | { 163 | "name":"java.nio.file.Path" 164 | }, 165 | { 166 | "name":"java.nio.file.Paths", 167 | "methods":[{"name":"get","parameterTypes":["java.lang.String","java.lang.String[]"] }] 168 | }, 169 | { 170 | "name":"java.sql.Connection" 171 | }, 172 | { 173 | "name":"java.sql.Driver" 174 | }, 175 | { 176 | "name":"java.sql.DriverManager", 177 | "methods":[ 178 | {"name":"getConnection","parameterTypes":["java.lang.String"] }, 179 | {"name":"getDriver","parameterTypes":["java.lang.String"] } 180 | ] 181 | }, 182 | { 183 | "name":"java.sql.Time", 184 | "methods":[{"name":"","parameterTypes":["long"] }] 185 | }, 186 | { 187 | "name":"java.sql.Timestamp", 188 | "methods":[{"name":"valueOf","parameterTypes":["java.lang.String"] }] 189 | }, 190 | { 191 | "name":"java.time.Duration", 192 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 193 | }, 194 | { 195 | "name":"java.time.Instant", 196 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 197 | }, 198 | { 199 | "name":"java.time.LocalDate", 200 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 201 | }, 202 | { 203 | "name":"java.time.LocalDateTime", 204 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 205 | }, 206 | { 207 | "name":"java.time.LocalTime", 208 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 209 | }, 210 | { 211 | "name":"java.time.MonthDay", 212 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 213 | }, 214 | { 215 | "name":"java.time.OffsetDateTime", 216 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 217 | }, 218 | { 219 | "name":"java.time.OffsetTime", 220 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 221 | }, 222 | { 223 | "name":"java.time.Period", 224 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 225 | }, 226 | { 227 | "name":"java.time.Year", 228 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 229 | }, 230 | { 231 | "name":"java.time.YearMonth", 232 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 233 | }, 234 | { 235 | "name":"java.time.ZoneId", 236 | "methods":[{"name":"of","parameterTypes":["java.lang.String"] }] 237 | }, 238 | { 239 | "name":"java.time.ZoneOffset", 240 | "methods":[{"name":"of","parameterTypes":["java.lang.String"] }] 241 | }, 242 | { 243 | "name":"java.time.ZonedDateTime", 244 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 245 | }, 246 | { 247 | "name":"java.util.Date" 248 | }, 249 | { 250 | "name":"java.util.PropertyPermission", 251 | "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String"] }] 252 | }, 253 | { 254 | "name":"java.util.logging.LogManager", 255 | "methods":[{"name":"getLoggingMXBean","parameterTypes":[] }] 256 | }, 257 | { 258 | "name":"java.util.logging.LoggingMXBean", 259 | "allPublicMethods":true 260 | }, 261 | { 262 | "name":"javax.management.MBeanOperationInfo", 263 | "allPublicMethods":true 264 | }, 265 | { 266 | "name":"javax.management.MBeanServerBuilder", 267 | "methods":[{"name":"","parameterTypes":[] }] 268 | }, 269 | { 270 | "name":"javax.management.ObjectName" 271 | }, 272 | { 273 | "name":"javax.management.openmbean.CompositeData" 274 | }, 275 | { 276 | "name":"javax.management.openmbean.CompositeData[]" 277 | }, 278 | { 279 | "name":"javax.management.openmbean.TabularData" 280 | }, 281 | { 282 | "name":"jdk.management.jfr.ConfigurationInfo", 283 | "allPublicMethods":true 284 | }, 285 | { 286 | "name":"jdk.management.jfr.EventTypeInfo", 287 | "allPublicMethods":true 288 | }, 289 | { 290 | "name":"jdk.management.jfr.FlightRecorderMXBean", 291 | "allPublicMethods":true 292 | }, 293 | { 294 | "name":"jdk.management.jfr.FlightRecorderMXBeanImpl", 295 | "allPublicConstructors":true 296 | }, 297 | { 298 | "name":"jdk.management.jfr.RecordingInfo", 299 | "allPublicMethods":true 300 | }, 301 | { 302 | "name":"jdk.management.jfr.SettingDescriptorInfo", 303 | "allPublicMethods":true 304 | }, 305 | { 306 | "name":"org.apache.kafka.clients.consumer.RangeAssignor", 307 | "methods":[{"name":"","parameterTypes":[] }] 308 | }, 309 | { 310 | "name":"org.apache.kafka.common.serialization.StringDeserializer" 311 | }, 312 | { 313 | "name":"org.apache.kafka.common.utils.AppInfoParser$AppInfo", 314 | "allPublicConstructors":true 315 | }, 316 | { 317 | "name":"org.apache.kafka.common.utils.AppInfoParser$AppInfoMBean", 318 | "allPublicMethods":true 319 | }, 320 | { 321 | "name":"picocli.CommandLine$AutoHelpMixin", 322 | "allDeclaredFields":true, 323 | "allDeclaredMethods":true 324 | }, 325 | { 326 | "name":"sun.management.ClassLoadingImpl", 327 | "allPublicConstructors":true 328 | }, 329 | { 330 | "name":"sun.management.CompilationImpl", 331 | "allPublicConstructors":true 332 | }, 333 | { 334 | "name":"sun.management.ManagementFactoryHelper$1", 335 | "allPublicConstructors":true 336 | }, 337 | { 338 | "name":"sun.management.ManagementFactoryHelper$PlatformLoggingImpl", 339 | "allPublicConstructors":true 340 | }, 341 | { 342 | "name":"sun.management.MemoryImpl", 343 | "allPublicConstructors":true 344 | }, 345 | { 346 | "name":"sun.management.MemoryManagerImpl", 347 | "allPublicConstructors":true 348 | }, 349 | { 350 | "name":"sun.management.MemoryPoolImpl", 351 | "allPublicConstructors":true 352 | }, 353 | { 354 | "name":"sun.management.RuntimeImpl", 355 | "allPublicConstructors":true 356 | } 357 | ] 358 | -------------------------------------------------------------------------------- /monitoring-component-console/bin/main/META-INF/native-image/resource-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "resources":[ 3 | {"pattern":"\\Qapplication-meta.properties\\E"}, 4 | {"pattern":"\\Qkafka/kafka-version.properties\\E"}, 5 | {"pattern":"\\Qorg/slf4j/impl/StaticLoggerBinder.class\\E"}, 6 | {"pattern":"\\Qsimplelogger.properties\\E"} 7 | ], 8 | "bundles":[] 9 | } 10 | -------------------------------------------------------------------------------- /monitoring-component-console/bin/main/com/omarsmak/kafka/consumer/lag/monitoring/component/console/ConsoleMonitoringComponent.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.component.console 2 | 3 | import com.github.ajalt.mordant.TermColors 4 | import com.omarsmak.kafka.consumer.lag.monitoring.client.data.Lag 5 | import com.omarsmak.kafka.consumer.lag.monitoring.component.MonitoringComponent 6 | import com.omarsmak.kafka.consumer.lag.monitoring.data.ConsumerGroupLag 7 | import com.omarsmak.kafka.consumer.lag.monitoring.support.castToInt 8 | 9 | class ConsoleMonitoringComponent : MonitoringComponent { 10 | 11 | companion object { 12 | const val LAG_THRESHOLD = "lag.threshold" 13 | const val DEFAULT_LAG_THRESHOLD = 500 14 | } 15 | 16 | private var monitoringLagThreshold = DEFAULT_LAG_THRESHOLD 17 | private val termColors = TermColors() 18 | 19 | override fun configure(configs: Map) { 20 | monitoringLagThreshold = configs.getOrDefault(LAG_THRESHOLD, DEFAULT_LAG_THRESHOLD).castToInt() 21 | } 22 | 23 | override fun start() { 24 | } 25 | 26 | override fun stop() { 27 | } 28 | 29 | override fun beforeProcess() { 30 | print("\u001b[H\u001b[2J") 31 | } 32 | 33 | override fun process(consumerGroup: String, consumerGroupLag: ConsumerGroupLag) { 34 | println("Consumer group: $consumerGroup") 35 | println("==============================================================================") 36 | println() 37 | consumerGroupLag.lag.forEach { 38 | printLagPerTopic(it, monitoringLagThreshold, termColors) 39 | } 40 | } 41 | 42 | override fun afterProcess() { 43 | println() 44 | } 45 | 46 | override fun identifier(): String = "console" 47 | 48 | override fun onError(t: Throwable) { 49 | println(termColors.yellow("Warning:${t.message}")) 50 | } 51 | 52 | private fun printLagPerTopic(metrics: Lag, monitoringLagThreshold: Int, termColors: TermColors) { 53 | println("Topic name: ${metrics.topicName}") 54 | println("Total topic offsets: ${metrics.latestTopicOffsets.values.sum()}") 55 | println("Total consumer offsets: ${metrics.latestConsumerOffsets.values.sum()}") 56 | 57 | when (metrics.totalLag) { 58 | in 0..monitoringLagThreshold -> 59 | println("Total lag: ${termColors.green(metrics.totalLag.toString())}") 60 | in monitoringLagThreshold..monitoringLagThreshold * 2 -> 61 | println("Total lag: ${termColors.yellow(metrics.totalLag.toString())}") 62 | else -> 63 | println("Total lag: ${termColors.red(metrics.totalLag.toString())}") 64 | } 65 | println() 66 | } 67 | } -------------------------------------------------------------------------------- /monitoring-component-console/bin/main/com/omarsmak/kafka/consumer/lag/monitoring/component/console/Main.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.component.console 2 | 3 | import com.github.ajalt.mordant.TermColors 4 | import com.omarsmak.kafka.consumer.lag.monitoring.engine.MonitoringEngine 5 | import com.omarsmak.kafka.consumer.lag.monitoring.support.Utils 6 | import com.omarsmak.kafka.consumer.lag.monitoring.support.asResource 7 | import org.apache.kafka.clients.admin.AdminClientConfig 8 | import picocli.CommandLine 9 | import java.lang.Exception 10 | import java.util.concurrent.Callable 11 | 12 | private const val META_DATA_FILE = "application-meta.properties" 13 | 14 | @CommandLine.Command(mixinStandardHelpOptions = true, versionProvider = VersionProvider::class, 15 | description = ["Prints the kafka consumer lag to the console."]) 16 | class ClientCli : Callable { 17 | 18 | @CommandLine.Option(names = ["-b", "--bootstrap.servers"], description = ["A list of host/port pairs to use for establishing the initial connection to the Kafka cluster"]) 19 | var kafkaBootstrapServers: String = "" 20 | 21 | @CommandLine.Option(names = ["-c", "--consumer.groups"], description = ["A list of Kafka consumer groups or list ending with star (*) to fetch all consumers with matching pattern, e.g: 'test_v*'"]) 22 | var kafkaConsumerGroups: String = "" 23 | 24 | @CommandLine.Option(names = ["-p", "--poll.interval"], description = ["Interval delay in ms to that refreshes the client lag metrics, default to 2000ms"]) 25 | var pollInterval: Int = MonitoringEngine.DEFAULT_POLL_INTERVAL 26 | 27 | 28 | @CommandLine.Option(names = ["-f", "--properties.file"], description = ["Optional. Properties file for Kafka AdminClient configurations, this is the typical Kafka properties file that can be used in the AdminClient. For more info, please take a look at Kafka AdminClient configurations documentation."]) 29 | var kafkaPropertiesFile: String = "" 30 | 31 | override fun call(): Int { 32 | 33 | // Load config 34 | val configs = initializeConfigurations(kafkaConsumerGroups) 35 | 36 | // Run the engine 37 | 38 | val engine = MonitoringEngine.createWithComponentAndConfigs(ConsoleMonitoringComponent(), configs) 39 | 40 | try { 41 | // start our engine 42 | engine.start() 43 | 44 | // add shutdown hook 45 | Runtime.getRuntime().addShutdownHook(Thread { 46 | engine.stop() 47 | }) 48 | } catch (e: Exception) { 49 | println(TermColors().red("Error: $e")) 50 | return 0 51 | } 52 | 53 | return 0 54 | } 55 | 56 | private fun initializeConfigurations(targetConsumerGroups: String) = mutableMapOf( 57 | "${MonitoringEngine.CONFIG_KAFKA_PREFIX}.${AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG}" to kafkaBootstrapServers, 58 | "${MonitoringEngine.CONFIG_LAG_CLIENT_PREFIX}.${MonitoringEngine.POLL_INTERVAL}" to pollInterval, 59 | "${MonitoringEngine.CONFIG_LAG_CLIENT_PREFIX}.${MonitoringEngine.CONSUMER_GROUPS}" to targetConsumerGroups 60 | ).apply { 61 | if (kafkaPropertiesFile.isNotEmpty()) { 62 | putAll(Utils.loadPropertiesFileAsMap(kafkaPropertiesFile)) 63 | } 64 | } 65 | } 66 | 67 | class VersionProvider : CommandLine.IVersionProvider { 68 | private val meta = Utils.loadPropertiesFromInputStream(META_DATA_FILE.asResource()) 69 | 70 | override fun getVersion(): Array = arrayOf(meta["version"] as String) 71 | 72 | 73 | } 74 | 75 | fun main(args: Array) { 76 | 77 | val metadata = Utils.loadPropertiesFromInputStream(META_DATA_FILE.asResource()) 78 | 79 | CommandLine(ClientCli()) 80 | .setCommandName(metadata["name"] as String) 81 | .execute(*args) 82 | } -------------------------------------------------------------------------------- /monitoring-component-console/bin/main/simplelogger.properties: -------------------------------------------------------------------------------- 1 | org.slf4j.simpleLogger.defaultLogLevel=warn -------------------------------------------------------------------------------- /monitoring-component-console/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'application' 2 | apply plugin: 'kotlin-kapt' 3 | 4 | mainClassName = 'com.omarsmak.kafka.consumer.lag.monitoring.component.console.MainKt' 5 | 6 | 7 | ext { 8 | componentName = "console" 9 | dockerExposedPort = 8080 10 | } 11 | 12 | dependencies { 13 | compile project(':monitoring-core') 14 | 15 | compile "com.github.ajalt:mordant:$mordantVersion" 16 | compile "info.picocli:picocli:$picocliVersion" 17 | compile "org.slf4j:slf4j-simple:$slf4jVersion" 18 | 19 | kapt "info.picocli:picocli-codegen:$picocliVersion" 20 | } 21 | 22 | kapt { 23 | arguments { 24 | arg("project", "${project.group}/${project.name}") 25 | } 26 | } 27 | 28 | 29 | task createProperties(dependsOn: processResources) { 30 | doLast { 31 | new File("$buildDir/resources/main/application-meta.properties").withWriter { w -> 32 | Properties p = new Properties() 33 | p['version'] = project.version.toString() 34 | p['name'] = componentFullName 35 | p.store w, null 36 | } 37 | } 38 | } 39 | 40 | classes { 41 | dependsOn createProperties 42 | } -------------------------------------------------------------------------------- /monitoring-component-console/src/main/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/component/console/ConsoleMonitoringComponent.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.component.console 2 | 3 | import com.github.ajalt.mordant.TermColors 4 | import com.omarsmak.kafka.consumer.lag.monitoring.client.data.Lag 5 | import com.omarsmak.kafka.consumer.lag.monitoring.component.MonitoringComponent 6 | import com.omarsmak.kafka.consumer.lag.monitoring.data.ConsumerGroupLag 7 | import com.omarsmak.kafka.consumer.lag.monitoring.support.castToInt 8 | 9 | class ConsoleMonitoringComponent : MonitoringComponent { 10 | 11 | companion object { 12 | const val LAG_THRESHOLD = "lag.threshold" 13 | const val DEFAULT_LAG_THRESHOLD = 500 14 | } 15 | 16 | private var monitoringLagThreshold = DEFAULT_LAG_THRESHOLD 17 | private val termColors = TermColors() 18 | 19 | override fun configure(configs: Map) { 20 | monitoringLagThreshold = configs.getOrDefault(LAG_THRESHOLD, DEFAULT_LAG_THRESHOLD).castToInt() 21 | } 22 | 23 | override fun start() { 24 | } 25 | 26 | override fun stop() { 27 | } 28 | 29 | override fun beforeProcess() { 30 | print("\u001b[H\u001b[2J") 31 | } 32 | 33 | override fun process(consumerGroup: String, consumerGroupLag: ConsumerGroupLag) { 34 | println("Consumer group: $consumerGroup") 35 | println("==============================================================================") 36 | println() 37 | consumerGroupLag.lag.forEach { 38 | printLagPerTopic(it, monitoringLagThreshold, termColors) 39 | } 40 | } 41 | 42 | override fun afterProcess() { 43 | println() 44 | } 45 | 46 | override fun identifier(): String = "console" 47 | 48 | override fun onError(t: Throwable) { 49 | println(termColors.yellow("Warning:${t.message}")) 50 | } 51 | 52 | private fun printLagPerTopic(metrics: Lag, monitoringLagThreshold: Int, termColors: TermColors) { 53 | println("Topic name: ${metrics.topicName}") 54 | println("Total topic offsets: ${metrics.latestTopicOffsets.values.sum()}") 55 | println("Total consumer offsets: ${metrics.latestConsumerOffsets.values.sum()}") 56 | 57 | when (metrics.totalLag) { 58 | in 0..monitoringLagThreshold -> 59 | println("Total lag: ${termColors.green(metrics.totalLag.toString())}") 60 | in monitoringLagThreshold..monitoringLagThreshold * 2 -> 61 | println("Total lag: ${termColors.yellow(metrics.totalLag.toString())}") 62 | else -> 63 | println("Total lag: ${termColors.red(metrics.totalLag.toString())}") 64 | } 65 | println() 66 | } 67 | } -------------------------------------------------------------------------------- /monitoring-component-console/src/main/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/component/console/Main.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.component.console 2 | 3 | import com.github.ajalt.mordant.TermColors 4 | import com.omarsmak.kafka.consumer.lag.monitoring.engine.MonitoringEngine 5 | import com.omarsmak.kafka.consumer.lag.monitoring.support.Utils 6 | import com.omarsmak.kafka.consumer.lag.monitoring.support.asResource 7 | import org.apache.kafka.clients.admin.AdminClientConfig 8 | import picocli.CommandLine 9 | import java.lang.Exception 10 | import java.util.concurrent.Callable 11 | 12 | private const val META_DATA_FILE = "application-meta.properties" 13 | 14 | @CommandLine.Command(mixinStandardHelpOptions = true, versionProvider = VersionProvider::class, 15 | description = ["Prints the kafka consumer lag to the console."]) 16 | class ClientCli : Callable { 17 | 18 | @CommandLine.Option(names = ["-b", "--bootstrap.servers"], description = ["A list of host/port pairs to use for establishing the initial connection to the Kafka cluster"]) 19 | var kafkaBootstrapServers: String = "" 20 | 21 | @CommandLine.Option(names = ["-c", "--consumer.groups"], description = ["A list of Kafka consumer groups or list ending with star (*) to fetch all consumers with matching pattern, e.g: 'test_v*'"]) 22 | var kafkaConsumerGroups: String = "" 23 | 24 | @CommandLine.Option(names = ["-p", "--poll.interval"], description = ["Interval delay in ms to that refreshes the client lag metrics, default to 2000ms"]) 25 | var pollInterval: Int = MonitoringEngine.DEFAULT_POLL_INTERVAL 26 | 27 | 28 | @CommandLine.Option(names = ["-f", "--properties.file"], description = ["Optional. Properties file for Kafka AdminClient configurations, this is the typical Kafka properties file that can be used in the AdminClient. For more info, please take a look at Kafka AdminClient configurations documentation."]) 29 | var kafkaPropertiesFile: String = "" 30 | 31 | override fun call(): Int { 32 | 33 | // Load config 34 | val configs = initializeConfigurations(kafkaConsumerGroups) 35 | 36 | // Run the engine 37 | 38 | val engine = MonitoringEngine.createWithComponentAndConfigs(ConsoleMonitoringComponent(), configs) 39 | 40 | try { 41 | // start our engine 42 | engine.start() 43 | 44 | // add shutdown hook 45 | Runtime.getRuntime().addShutdownHook(Thread { 46 | engine.stop() 47 | }) 48 | } catch (e: Exception) { 49 | println(TermColors().red("Error: $e")) 50 | return 0 51 | } 52 | 53 | return 0 54 | } 55 | 56 | private fun initializeConfigurations(targetConsumerGroups: String) = mutableMapOf( 57 | "${MonitoringEngine.CONFIG_KAFKA_PREFIX}.${AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG}" to kafkaBootstrapServers, 58 | "${MonitoringEngine.CONFIG_LAG_CLIENT_PREFIX}.${MonitoringEngine.POLL_INTERVAL}" to pollInterval, 59 | "${MonitoringEngine.CONFIG_LAG_CLIENT_PREFIX}.${MonitoringEngine.CONSUMER_GROUPS}" to targetConsumerGroups 60 | ).apply { 61 | if (kafkaPropertiesFile.isNotEmpty()) { 62 | putAll(Utils.loadPropertiesFileAsMap(kafkaPropertiesFile)) 63 | } 64 | } 65 | } 66 | 67 | class VersionProvider : CommandLine.IVersionProvider { 68 | private val meta = Utils.loadPropertiesFromInputStream(META_DATA_FILE.asResource()) 69 | 70 | override fun getVersion(): Array = arrayOf(meta["version"] as String) 71 | 72 | 73 | } 74 | 75 | fun main(args: Array) { 76 | 77 | val metadata = Utils.loadPropertiesFromInputStream(META_DATA_FILE.asResource()) 78 | 79 | CommandLine(ClientCli()) 80 | .setCommandName(metadata["name"] as String) 81 | .execute(*args) 82 | } -------------------------------------------------------------------------------- /monitoring-component-console/src/main/resources/META-INF/native-image/jni-config.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name":"com.sun.management.internal.DiagnosticCommandArgumentInfo", 4 | "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","boolean","boolean","int"] }] 5 | }, 6 | { 7 | "name":"com.sun.management.internal.DiagnosticCommandArgumentInfo[]" 8 | }, 9 | { 10 | "name":"com.sun.management.internal.DiagnosticCommandInfo", 11 | "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","java.util.List"] }] 12 | }, 13 | { 14 | "name":"com.sun.management.internal.DiagnosticCommandInfo[]" 15 | }, 16 | { 17 | "name":"java.lang.ClassLoader", 18 | "methods":[ 19 | {"name":"getPlatformClassLoader","parameterTypes":[] }, 20 | {"name":"loadClass","parameterTypes":["java.lang.String"] } 21 | ] 22 | }, 23 | { 24 | "name":"java.lang.ClassNotFoundException" 25 | }, 26 | { 27 | "name":"java.lang.NoSuchMethodError" 28 | }, 29 | { 30 | "name":"java.util.Arrays", 31 | "methods":[{"name":"asList","parameterTypes":["java.lang.Object[]"] }] 32 | }, 33 | { 34 | "name":"sun.management.VMManagementImpl", 35 | "fields":[ 36 | {"name":"compTimeMonitoringSupport"}, 37 | {"name":"currentThreadCpuTimeSupport"}, 38 | {"name":"objectMonitorUsageSupport"}, 39 | {"name":"otherThreadCpuTimeSupport"}, 40 | {"name":"remoteDiagnosticCommandsSupport"}, 41 | {"name":"synchronizerUsageSupport"}, 42 | {"name":"threadAllocatedMemorySupport"}, 43 | {"name":"threadContentionMonitoringSupport"} 44 | ] 45 | } 46 | ] 47 | -------------------------------------------------------------------------------- /monitoring-component-console/src/main/resources/META-INF/native-image/proxy-config.json: -------------------------------------------------------------------------------- 1 | [ 2 | ] 3 | -------------------------------------------------------------------------------- /monitoring-component-console/src/main/resources/META-INF/native-image/reflect-config.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name":"com.omarsmak.kafka.consumer.lag.monitoring.component.console.ClientCli", 4 | "allDeclaredFields":true, 5 | "allDeclaredMethods":true 6 | }, 7 | { 8 | "name":"com.omarsmak.kafka.consumer.lag.monitoring.component.console.VersionProvider", 9 | "allDeclaredFields":true, 10 | "allDeclaredMethods":true, 11 | "methods":[{"name":"","parameterTypes":[] }] 12 | }, 13 | { 14 | "name":"com.sun.management.GarbageCollectorMXBean", 15 | "allPublicMethods":true 16 | }, 17 | { 18 | "name":"com.sun.management.GcInfo", 19 | "allPublicMethods":true 20 | }, 21 | { 22 | "name":"com.sun.management.HotSpotDiagnosticMXBean", 23 | "allPublicMethods":true 24 | }, 25 | { 26 | "name":"com.sun.management.ThreadMXBean", 27 | "allPublicMethods":true 28 | }, 29 | { 30 | "name":"com.sun.management.UnixOperatingSystemMXBean", 31 | "allPublicMethods":true 32 | }, 33 | { 34 | "name":"com.sun.management.VMOption", 35 | "allPublicMethods":true 36 | }, 37 | { 38 | "name":"com.sun.management.internal.GarbageCollectorExtImpl", 39 | "allPublicConstructors":true 40 | }, 41 | { 42 | "name":"com.sun.management.internal.HotSpotDiagnostic", 43 | "allPublicConstructors":true 44 | }, 45 | { 46 | "name":"com.sun.management.internal.HotSpotThreadImpl", 47 | "allPublicConstructors":true 48 | }, 49 | { 50 | "name":"com.sun.management.internal.OperatingSystemImpl", 51 | "allPublicConstructors":true 52 | }, 53 | { 54 | "name":"java.lang.Boolean", 55 | "fields":[{"name":"TYPE"}] 56 | }, 57 | { 58 | "name":"java.lang.Byte", 59 | "fields":[{"name":"TYPE"}] 60 | }, 61 | { 62 | "name":"java.lang.Character", 63 | "fields":[{"name":"TYPE"}] 64 | }, 65 | { 66 | "name":"java.lang.Double", 67 | "fields":[{"name":"TYPE"}] 68 | }, 69 | { 70 | "name":"java.lang.Float", 71 | "fields":[{"name":"TYPE"}] 72 | }, 73 | { 74 | "name":"java.lang.Integer", 75 | "fields":[{"name":"TYPE"}] 76 | }, 77 | { 78 | "name":"java.lang.Long", 79 | "fields":[{"name":"TYPE"}] 80 | }, 81 | { 82 | "name":"java.lang.Object", 83 | "allDeclaredFields":true, 84 | "allDeclaredMethods":true 85 | }, 86 | { 87 | "name":"java.lang.Short", 88 | "fields":[{"name":"TYPE"}] 89 | }, 90 | { 91 | "name":"java.lang.StackTraceElement", 92 | "allPublicMethods":true 93 | }, 94 | { 95 | "name":"java.lang.String" 96 | }, 97 | { 98 | "name":"java.lang.String[]" 99 | }, 100 | { 101 | "name":"java.lang.Void", 102 | "fields":[{"name":"TYPE"}] 103 | }, 104 | { 105 | "name":"java.lang.management.BufferPoolMXBean", 106 | "allPublicMethods":true 107 | }, 108 | { 109 | "name":"java.lang.management.ClassLoadingMXBean", 110 | "allPublicMethods":true 111 | }, 112 | { 113 | "name":"java.lang.management.CompilationMXBean", 114 | "allPublicMethods":true 115 | }, 116 | { 117 | "name":"java.lang.management.LockInfo", 118 | "allPublicMethods":true 119 | }, 120 | { 121 | "name":"java.lang.management.ManagementPermission", 122 | "methods":[{"name":"","parameterTypes":["java.lang.String"] }] 123 | }, 124 | { 125 | "name":"java.lang.management.MemoryMXBean", 126 | "allPublicMethods":true 127 | }, 128 | { 129 | "name":"java.lang.management.MemoryManagerMXBean", 130 | "allPublicMethods":true 131 | }, 132 | { 133 | "name":"java.lang.management.MemoryPoolMXBean", 134 | "allPublicMethods":true 135 | }, 136 | { 137 | "name":"java.lang.management.MemoryUsage", 138 | "allPublicMethods":true 139 | }, 140 | { 141 | "name":"java.lang.management.MonitorInfo", 142 | "allPublicMethods":true 143 | }, 144 | { 145 | "name":"java.lang.management.PlatformLoggingMXBean", 146 | "allPublicMethods":true 147 | }, 148 | { 149 | "name":"java.lang.management.RuntimeMXBean", 150 | "allPublicMethods":true 151 | }, 152 | { 153 | "name":"java.lang.management.ThreadInfo", 154 | "allPublicMethods":true 155 | }, 156 | { 157 | "name":"java.math.BigDecimal" 158 | }, 159 | { 160 | "name":"java.math.BigInteger" 161 | }, 162 | { 163 | "name":"java.nio.file.Path" 164 | }, 165 | { 166 | "name":"java.nio.file.Paths", 167 | "methods":[{"name":"get","parameterTypes":["java.lang.String","java.lang.String[]"] }] 168 | }, 169 | { 170 | "name":"java.sql.Connection" 171 | }, 172 | { 173 | "name":"java.sql.Driver" 174 | }, 175 | { 176 | "name":"java.sql.DriverManager", 177 | "methods":[ 178 | {"name":"getConnection","parameterTypes":["java.lang.String"] }, 179 | {"name":"getDriver","parameterTypes":["java.lang.String"] } 180 | ] 181 | }, 182 | { 183 | "name":"java.sql.Time", 184 | "methods":[{"name":"","parameterTypes":["long"] }] 185 | }, 186 | { 187 | "name":"java.sql.Timestamp", 188 | "methods":[{"name":"valueOf","parameterTypes":["java.lang.String"] }] 189 | }, 190 | { 191 | "name":"java.time.Duration", 192 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 193 | }, 194 | { 195 | "name":"java.time.Instant", 196 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 197 | }, 198 | { 199 | "name":"java.time.LocalDate", 200 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 201 | }, 202 | { 203 | "name":"java.time.LocalDateTime", 204 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 205 | }, 206 | { 207 | "name":"java.time.LocalTime", 208 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 209 | }, 210 | { 211 | "name":"java.time.MonthDay", 212 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 213 | }, 214 | { 215 | "name":"java.time.OffsetDateTime", 216 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 217 | }, 218 | { 219 | "name":"java.time.OffsetTime", 220 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 221 | }, 222 | { 223 | "name":"java.time.Period", 224 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 225 | }, 226 | { 227 | "name":"java.time.Year", 228 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 229 | }, 230 | { 231 | "name":"java.time.YearMonth", 232 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 233 | }, 234 | { 235 | "name":"java.time.ZoneId", 236 | "methods":[{"name":"of","parameterTypes":["java.lang.String"] }] 237 | }, 238 | { 239 | "name":"java.time.ZoneOffset", 240 | "methods":[{"name":"of","parameterTypes":["java.lang.String"] }] 241 | }, 242 | { 243 | "name":"java.time.ZonedDateTime", 244 | "methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }] 245 | }, 246 | { 247 | "name":"java.util.Date" 248 | }, 249 | { 250 | "name":"java.util.PropertyPermission", 251 | "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String"] }] 252 | }, 253 | { 254 | "name":"java.util.logging.LogManager", 255 | "methods":[{"name":"getLoggingMXBean","parameterTypes":[] }] 256 | }, 257 | { 258 | "name":"java.util.logging.LoggingMXBean", 259 | "allPublicMethods":true 260 | }, 261 | { 262 | "name":"javax.management.MBeanOperationInfo", 263 | "allPublicMethods":true 264 | }, 265 | { 266 | "name":"javax.management.MBeanServerBuilder", 267 | "methods":[{"name":"","parameterTypes":[] }] 268 | }, 269 | { 270 | "name":"javax.management.ObjectName" 271 | }, 272 | { 273 | "name":"javax.management.openmbean.CompositeData" 274 | }, 275 | { 276 | "name":"javax.management.openmbean.CompositeData[]" 277 | }, 278 | { 279 | "name":"javax.management.openmbean.TabularData" 280 | }, 281 | { 282 | "name":"jdk.management.jfr.ConfigurationInfo", 283 | "allPublicMethods":true 284 | }, 285 | { 286 | "name":"jdk.management.jfr.EventTypeInfo", 287 | "allPublicMethods":true 288 | }, 289 | { 290 | "name":"jdk.management.jfr.FlightRecorderMXBean", 291 | "allPublicMethods":true 292 | }, 293 | { 294 | "name":"jdk.management.jfr.FlightRecorderMXBeanImpl", 295 | "allPublicConstructors":true 296 | }, 297 | { 298 | "name":"jdk.management.jfr.RecordingInfo", 299 | "allPublicMethods":true 300 | }, 301 | { 302 | "name":"jdk.management.jfr.SettingDescriptorInfo", 303 | "allPublicMethods":true 304 | }, 305 | { 306 | "name":"org.apache.kafka.clients.consumer.RangeAssignor", 307 | "methods":[{"name":"","parameterTypes":[] }] 308 | }, 309 | { 310 | "name":"org.apache.kafka.common.serialization.StringDeserializer" 311 | }, 312 | { 313 | "name":"org.apache.kafka.common.utils.AppInfoParser$AppInfo", 314 | "allPublicConstructors":true 315 | }, 316 | { 317 | "name":"org.apache.kafka.common.utils.AppInfoParser$AppInfoMBean", 318 | "allPublicMethods":true 319 | }, 320 | { 321 | "name":"picocli.CommandLine$AutoHelpMixin", 322 | "allDeclaredFields":true, 323 | "allDeclaredMethods":true 324 | }, 325 | { 326 | "name":"sun.management.ClassLoadingImpl", 327 | "allPublicConstructors":true 328 | }, 329 | { 330 | "name":"sun.management.CompilationImpl", 331 | "allPublicConstructors":true 332 | }, 333 | { 334 | "name":"sun.management.ManagementFactoryHelper$1", 335 | "allPublicConstructors":true 336 | }, 337 | { 338 | "name":"sun.management.ManagementFactoryHelper$PlatformLoggingImpl", 339 | "allPublicConstructors":true 340 | }, 341 | { 342 | "name":"sun.management.MemoryImpl", 343 | "allPublicConstructors":true 344 | }, 345 | { 346 | "name":"sun.management.MemoryManagerImpl", 347 | "allPublicConstructors":true 348 | }, 349 | { 350 | "name":"sun.management.MemoryPoolImpl", 351 | "allPublicConstructors":true 352 | }, 353 | { 354 | "name":"sun.management.RuntimeImpl", 355 | "allPublicConstructors":true 356 | } 357 | ] 358 | -------------------------------------------------------------------------------- /monitoring-component-console/src/main/resources/META-INF/native-image/resource-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "resources":[ 3 | {"pattern":"\\Qapplication-meta.properties\\E"}, 4 | {"pattern":"\\Qkafka/kafka-version.properties\\E"}, 5 | {"pattern":"\\Qorg/slf4j/impl/StaticLoggerBinder.class\\E"}, 6 | {"pattern":"\\Qsimplelogger.properties\\E"} 7 | ], 8 | "bundles":[] 9 | } 10 | -------------------------------------------------------------------------------- /monitoring-component-console/src/main/resources/simplelogger.properties: -------------------------------------------------------------------------------- 1 | org.slf4j.simpleLogger.defaultLogLevel=warn -------------------------------------------------------------------------------- /monitoring-component-prometheus/bin/main/META-INF/native-image/jni-config.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name":"com.sun.management.internal.DiagnosticCommandArgumentInfo", 4 | "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","boolean","boolean","int"] }] 5 | }, 6 | { 7 | "name":"com.sun.management.internal.DiagnosticCommandArgumentInfo[]" 8 | }, 9 | { 10 | "name":"com.sun.management.internal.DiagnosticCommandInfo", 11 | "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","java.util.List"] }] 12 | }, 13 | { 14 | "name":"com.sun.management.internal.DiagnosticCommandInfo[]" 15 | }, 16 | { 17 | "name":"java.lang.ClassLoader", 18 | "methods":[ 19 | {"name":"getPlatformClassLoader","parameterTypes":[] }, 20 | {"name":"loadClass","parameterTypes":["java.lang.String"] } 21 | ] 22 | }, 23 | { 24 | "name":"java.lang.ClassNotFoundException" 25 | }, 26 | { 27 | "name":"java.lang.NoSuchMethodError" 28 | }, 29 | { 30 | "name":"java.util.Arrays", 31 | "methods":[{"name":"asList","parameterTypes":["java.lang.Object[]"] }] 32 | }, 33 | { 34 | "name":"sun.management.VMManagementImpl", 35 | "fields":[ 36 | {"name":"compTimeMonitoringSupport"}, 37 | {"name":"currentThreadCpuTimeSupport"}, 38 | {"name":"objectMonitorUsageSupport"}, 39 | {"name":"otherThreadCpuTimeSupport"}, 40 | {"name":"remoteDiagnosticCommandsSupport"}, 41 | {"name":"synchronizerUsageSupport"}, 42 | {"name":"threadAllocatedMemorySupport"}, 43 | {"name":"threadContentionMonitoringSupport"} 44 | ] 45 | } 46 | ] 47 | -------------------------------------------------------------------------------- /monitoring-component-prometheus/bin/main/META-INF/native-image/proxy-config.json: -------------------------------------------------------------------------------- 1 | [ 2 | ] 3 | -------------------------------------------------------------------------------- /monitoring-component-prometheus/bin/main/META-INF/native-image/resource-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "resources":[ 3 | {"pattern":"\\QMETA-INF/org/apache/logging/log4j/core/config/plugins/Log4j2Plugins.dat\\E"}, 4 | {"pattern":"\\QMETA-INF/services/com.oracle.truffle.api.TruffleLanguage$Provider\\E"}, 5 | {"pattern":"\\QMETA-INF/services/com.oracle.truffle.api.instrumentation.TruffleInstrument$Provider\\E"}, 6 | {"pattern":"\\QMETA-INF/services/org.apache.logging.log4j.core.util.ContextDataProvider\\E"}, 7 | {"pattern":"\\QMETA-INF/services/org.apache.logging.log4j.spi.Provider\\E"}, 8 | {"pattern":"\\QMETA-INF/services/org.apache.logging.log4j.util.PropertySource\\E"}, 9 | {"pattern":"\\Qcom/oracle/truffle/nfi/impl/NFILanguageImpl.class\\E"}, 10 | {"pattern":"\\Qkafka/kafka-version.properties\\E"}, 11 | {"pattern":"\\Qlog4j2.properties\\E"}, 12 | {"pattern":"\\Qorg/slf4j/impl/StaticLoggerBinder.class\\E"}, 13 | {"pattern":"\\Qconfig/log4j2.template\\E"} 14 | ], 15 | "bundles":[] 16 | } 17 | -------------------------------------------------------------------------------- /monitoring-component-prometheus/bin/main/com/omarsmak/kafka/consumer/lag/monitoring/component/prometheus/Main.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.component.prometheus 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.engine.MonitoringEngine 4 | import com.omarsmak.kafka.consumer.lag.monitoring.support.Utils 5 | import mu.KotlinLogging 6 | import java.lang.Exception 7 | 8 | fun main(arg: Array) { 9 | val configs = Utils.getConfigsFromPropertiesFileOrFromEnv(arg) 10 | 11 | // start our logging service 12 | Utils.initializeLog4jLoggingWithConfigs(configs) 13 | 14 | val component = PrometheusMonitoringComponent() 15 | val engine = MonitoringEngine.createWithComponentAndConfigs(component, configs) 16 | 17 | try { 18 | // start our engine 19 | engine.start() 20 | 21 | // add shutdown hook 22 | Runtime.getRuntime().addShutdownHook(Thread { 23 | engine.stop() 24 | }) 25 | } catch (e: Exception) { 26 | KotlinLogging.logger{}.error(e.message, e) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /monitoring-component-prometheus/bin/main/com/omarsmak/kafka/consumer/lag/monitoring/component/prometheus/PrometheusMonitoringComponent.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.component.prometheus 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.client.exceptions.KafkaConsumerLagClientException 4 | import com.omarsmak.kafka.consumer.lag.monitoring.component.MonitoringComponent 5 | import com.omarsmak.kafka.consumer.lag.monitoring.data.ConsumerGroupLag 6 | import com.omarsmak.kafka.consumer.lag.monitoring.support.castToInt 7 | import io.prometheus.client.Gauge 8 | import io.prometheus.client.exporter.HTTPServer 9 | import mu.KotlinLogging 10 | 11 | class PrometheusMonitoringComponent : MonitoringComponent{ 12 | 13 | companion object { 14 | const val HTTP_PORT = "http.port" 15 | const val DEFAULT_HTTP_PORT = 9739 16 | 17 | private val logger = KotlinLogging.logger {} 18 | 19 | private val kafkaConsumerGroupOffsetGauge = Gauge.build() 20 | .name("kafka_consumer_group_offset") 21 | .help("The latest committed offset of a consumer group in a given partition of a topic") 22 | .labelNames("group", "topic", "partition") 23 | .register() 24 | 25 | private val kafkaConsumerLagPerPartitionGauge = Gauge.build() 26 | .name("kafka_consumer_group_partition_lag") 27 | .help("The lag of a consumer group behind the head of a given partition of a topic. Calculated like this: current_topic_offset_per_partition - current_consumer_offset_per_partition") 28 | .labelNames("group", "topic", "partition") 29 | .register() 30 | 31 | private val kafkaTopicLatestOffsetsGauge = Gauge.build() 32 | .name("kafka_topic_latest_offsets") 33 | .help("The latest committed offset of a topic in a given partition") 34 | .labelNames("group", "topic", "partition") 35 | .register() 36 | 37 | private val kafkaConsumerTotalLagGauge = Gauge.build() 38 | .name("kafka_consumer_group_total_lag") 39 | .help("The total lag of a consumer group behind the head of a topic. This gives the total lags over each partition, it provides good visibility but not a precise measurement since is not partition aware") 40 | .labelNames("group", "topic") 41 | .register() 42 | 43 | private val kafkaConsumerMemberLagGauge = Gauge.build() 44 | .name("kafka_consumer_group_member_lag") 45 | .help("The total lag of a consumer group member behind the head of a topic. This gives the total lags over each consumer member within consumer group") 46 | .labelNames("group", "member", "topic") 47 | .register() 48 | 49 | private val kafkaConsumerMemberPartitionLagGauge = Gauge.build() 50 | .name("kafka_consumer_group_member_partition_lag") 51 | .help("The lag of a consumer member within consumer group behind the head of a given partition of a topic") 52 | .labelNames("group", "member", "topic", "partition") 53 | .register() 54 | } 55 | 56 | private var httpPort = DEFAULT_HTTP_PORT 57 | private lateinit var httpServer: HTTPServer 58 | 59 | override fun configure(configs: Map) { 60 | httpPort = configs.getOrDefault(HTTP_PORT, DEFAULT_HTTP_PORT).castToInt() 61 | } 62 | 63 | override fun start() { 64 | // Start a HTTP server to expose metrics 65 | logger.info("Starting HTTP server on $httpPort....") 66 | httpServer = HTTPServer(httpPort) 67 | } 68 | 69 | override fun stop() { 70 | // Stop our HTTP server 71 | logger.info("Stopping HTTP server on $httpPort....") 72 | httpServer.stop() 73 | } 74 | 75 | override fun beforeProcess() { 76 | // reset metrics before we process a consumer on every polling 77 | kafkaConsumerGroupOffsetGauge.clear() 78 | kafkaConsumerLagPerPartitionGauge.clear() 79 | kafkaTopicLatestOffsetsGauge.clear() 80 | kafkaConsumerTotalLagGauge.clear() 81 | kafkaConsumerMemberLagGauge.clear() 82 | } 83 | 84 | override fun process(consumerGroup: String, consumerGroupLag: ConsumerGroupLag) { 85 | try { 86 | val lag = consumerGroupLag.lag 87 | val memberLag = consumerGroupLag.memberLag 88 | 89 | // Push metrics for each topic 90 | lag.forEach { topic -> 91 | // Push kafka_consumer_group_total_lag 92 | kafkaConsumerTotalLagGauge.labels(consumerGroup, topic.topicName).set(topic.totalLag.toDouble()) 93 | 94 | // Push kafka_consumer_group_offset metrics for each partition 95 | topic.latestConsumerOffsets.forEach { (t, u) -> 96 | kafkaConsumerGroupOffsetGauge 97 | .pushKafkaMetricsPerPartition(consumerGroup, topic.topicName, t, u.toDouble()) 98 | } 99 | 100 | // Push kafka_topic_latest_offsets metrics for each partition 101 | topic.latestTopicOffsets.forEach { (t, u) -> 102 | kafkaTopicLatestOffsetsGauge 103 | .pushKafkaMetricsPerPartition(consumerGroup, topic.topicName, t, u.toDouble()) 104 | } 105 | 106 | // Push kafka_consumer_group_partition_lag metrics for each partition 107 | topic.lagPerPartition.forEach { (t, u) -> 108 | kafkaConsumerLagPerPartitionGauge 109 | .pushKafkaMetricsPerPartition(consumerGroup, topic.topicName, t, u.toDouble()) 110 | } 111 | } 112 | 113 | memberLag.forEach { (member, lags) -> 114 | lags.forEach { 115 | kafkaConsumerMemberLagGauge.labels(consumerGroup, member, it.topicName).set(it.totalLag.toDouble()) 116 | 117 | it.lagPerPartition.forEach { (t, u) -> 118 | kafkaConsumerMemberPartitionLagGauge.labels(consumerGroup, member, it.topicName, t.toString()).set(u.toDouble()) 119 | } 120 | } 121 | } 122 | } catch (e: KafkaConsumerLagClientException) { 123 | logger.error(e.message, e.cause) 124 | } 125 | } 126 | 127 | override fun afterProcess() { 128 | } 129 | 130 | override fun identifier(): String = "prometheus" 131 | 132 | override fun onError(t: Throwable) { 133 | logger.error(t.message, t) 134 | } 135 | 136 | private fun Gauge.pushKafkaMetricsPerPartition(consumer: String, topicName: String, partition: Int, value: Double) { 137 | this.labels( 138 | consumer, 139 | topicName, 140 | partition.toString() 141 | ).set(value) 142 | } 143 | } -------------------------------------------------------------------------------- /monitoring-component-prometheus/bin/main/configs.properties: -------------------------------------------------------------------------------- 1 | kafka.bootstrap.servers=localhost:9092 2 | kafka.retry.backoff.ms = 200 3 | monitoring.lag.consumer.groups=test* 4 | monitoring.lag.prometheus.http.port=9772 5 | monitoring.lag.logging.rootLogger.appenderRef.stdout.ref=LogToConsole 6 | monitoring.lag.logging.rootLogger.level=info -------------------------------------------------------------------------------- /monitoring-component-prometheus/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'application' 2 | 3 | mainClassName = 'com.omarsmak.kafka.consumer.lag.monitoring.component.prometheus.MainKt' 4 | 5 | ext { 6 | componentName = "prometheus" 7 | dockerExposedPort = 9739 8 | } 9 | 10 | dependencies { 11 | compile project(':monitoring-core') 12 | 13 | compile "io.prometheus:simpleclient:$prometheusVersion" 14 | compile "io.prometheus:simpleclient_hotspot:$prometheusVersion" 15 | compile "io.prometheus:simpleclient_httpserver:$prometheusVersion" 16 | compile "org.apache.logging.log4j:log4j-slf4j-impl:$log4jSlf4jImplVersion" 17 | compile "com.fasterxml.jackson.core:jackson-databind:$jacksonDatabindVersion" 18 | } -------------------------------------------------------------------------------- /monitoring-component-prometheus/src/main/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/component/prometheus/Main.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.component.prometheus 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.engine.MonitoringEngine 4 | import com.omarsmak.kafka.consumer.lag.monitoring.support.Utils 5 | import mu.KotlinLogging 6 | import java.lang.Exception 7 | 8 | fun main(arg: Array) { 9 | val configs = Utils.getConfigsFromPropertiesFileOrFromEnv(arg) 10 | 11 | // start our logging service 12 | Utils.initializeLog4jLoggingWithConfigs(configs) 13 | 14 | val component = PrometheusMonitoringComponent() 15 | val engine = MonitoringEngine.createWithComponentAndConfigs(component, configs) 16 | 17 | try { 18 | // start our engine 19 | engine.start() 20 | 21 | // add shutdown hook 22 | Runtime.getRuntime().addShutdownHook(Thread { 23 | engine.stop() 24 | }) 25 | } catch (e: Exception) { 26 | KotlinLogging.logger{}.error(e.message, e) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /monitoring-component-prometheus/src/main/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/component/prometheus/PrometheusMonitoringComponent.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.component.prometheus 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.client.exceptions.KafkaConsumerLagClientException 4 | import com.omarsmak.kafka.consumer.lag.monitoring.component.MonitoringComponent 5 | import com.omarsmak.kafka.consumer.lag.monitoring.data.ConsumerGroupLag 6 | import com.omarsmak.kafka.consumer.lag.monitoring.support.castToInt 7 | import io.prometheus.client.Gauge 8 | import io.prometheus.client.exporter.HTTPServer 9 | import mu.KotlinLogging 10 | 11 | class PrometheusMonitoringComponent : MonitoringComponent{ 12 | 13 | companion object { 14 | const val HTTP_PORT = "http.port" 15 | const val DEFAULT_HTTP_PORT = 9739 16 | 17 | private val logger = KotlinLogging.logger {} 18 | 19 | private val kafkaConsumerGroupOffsetGauge = Gauge.build() 20 | .name("kafka_consumer_group_offset") 21 | .help("The latest committed offset of a consumer group in a given partition of a topic") 22 | .labelNames("group", "topic", "partition") 23 | .register() 24 | 25 | private val kafkaConsumerLagPerPartitionGauge = Gauge.build() 26 | .name("kafka_consumer_group_partition_lag") 27 | .help("The lag of a consumer group behind the head of a given partition of a topic. Calculated like this: current_topic_offset_per_partition - current_consumer_offset_per_partition") 28 | .labelNames("group", "topic", "partition") 29 | .register() 30 | 31 | private val kafkaTopicLatestOffsetsGauge = Gauge.build() 32 | .name("kafka_topic_latest_offsets") 33 | .help("The latest committed offset of a topic in a given partition") 34 | .labelNames("group", "topic", "partition") 35 | .register() 36 | 37 | private val kafkaConsumerTotalLagGauge = Gauge.build() 38 | .name("kafka_consumer_group_total_lag") 39 | .help("The total lag of a consumer group behind the head of a topic. This gives the total lags over each partition, it provides good visibility but not a precise measurement since is not partition aware") 40 | .labelNames("group", "topic") 41 | .register() 42 | 43 | private val kafkaConsumerMemberLagGauge = Gauge.build() 44 | .name("kafka_consumer_group_member_lag") 45 | .help("The total lag of a consumer group member behind the head of a topic. This gives the total lags over each consumer member within consumer group") 46 | .labelNames("group", "member", "topic") 47 | .register() 48 | 49 | private val kafkaConsumerMemberPartitionLagGauge = Gauge.build() 50 | .name("kafka_consumer_group_member_partition_lag") 51 | .help("The lag of a consumer member within consumer group behind the head of a given partition of a topic") 52 | .labelNames("group", "member", "topic", "partition") 53 | .register() 54 | } 55 | 56 | private var httpPort = DEFAULT_HTTP_PORT 57 | private lateinit var httpServer: HTTPServer 58 | 59 | override fun configure(configs: Map) { 60 | httpPort = configs.getOrDefault(HTTP_PORT, DEFAULT_HTTP_PORT).castToInt() 61 | } 62 | 63 | override fun start() { 64 | // Start a HTTP server to expose metrics 65 | logger.info("Starting HTTP server on $httpPort....") 66 | httpServer = HTTPServer(httpPort) 67 | } 68 | 69 | override fun stop() { 70 | // Stop our HTTP server 71 | logger.info("Stopping HTTP server on $httpPort....") 72 | httpServer.stop() 73 | } 74 | 75 | override fun beforeProcess() { 76 | // reset metrics before we process a consumer on every polling 77 | kafkaConsumerGroupOffsetGauge.clear() 78 | kafkaConsumerLagPerPartitionGauge.clear() 79 | kafkaTopicLatestOffsetsGauge.clear() 80 | kafkaConsumerTotalLagGauge.clear() 81 | kafkaConsumerMemberLagGauge.clear() 82 | } 83 | 84 | override fun process(consumerGroup: String, consumerGroupLag: ConsumerGroupLag) { 85 | try { 86 | val lag = consumerGroupLag.lag 87 | val memberLag = consumerGroupLag.memberLag 88 | 89 | // Push metrics for each topic 90 | lag.forEach { topic -> 91 | // Push kafka_consumer_group_total_lag 92 | kafkaConsumerTotalLagGauge.labels(consumerGroup, topic.topicName).set(topic.totalLag.toDouble()) 93 | 94 | // Push kafka_consumer_group_offset metrics for each partition 95 | topic.latestConsumerOffsets.forEach { (t, u) -> 96 | kafkaConsumerGroupOffsetGauge 97 | .pushKafkaMetricsPerPartition(consumerGroup, topic.topicName, t, u.toDouble()) 98 | } 99 | 100 | // Push kafka_topic_latest_offsets metrics for each partition 101 | topic.latestTopicOffsets.forEach { (t, u) -> 102 | kafkaTopicLatestOffsetsGauge 103 | .pushKafkaMetricsPerPartition(consumerGroup, topic.topicName, t, u.toDouble()) 104 | } 105 | 106 | // Push kafka_consumer_group_partition_lag metrics for each partition 107 | topic.lagPerPartition.forEach { (t, u) -> 108 | kafkaConsumerLagPerPartitionGauge 109 | .pushKafkaMetricsPerPartition(consumerGroup, topic.topicName, t, u.toDouble()) 110 | } 111 | } 112 | 113 | memberLag.forEach { (member, lags) -> 114 | lags.forEach { 115 | kafkaConsumerMemberLagGauge.labels(consumerGroup, member, it.topicName).set(it.totalLag.toDouble()) 116 | 117 | it.lagPerPartition.forEach { (t, u) -> 118 | kafkaConsumerMemberPartitionLagGauge.labels(consumerGroup, member, it.topicName, t.toString()).set(u.toDouble()) 119 | } 120 | } 121 | } 122 | } catch (e: KafkaConsumerLagClientException) { 123 | logger.error(e.message, e.cause) 124 | } 125 | } 126 | 127 | override fun afterProcess() { 128 | } 129 | 130 | override fun identifier(): String = "prometheus" 131 | 132 | override fun onError(t: Throwable) { 133 | logger.error(t.message, t) 134 | } 135 | 136 | private fun Gauge.pushKafkaMetricsPerPartition(consumer: String, topicName: String, partition: Int, value: Double) { 137 | this.labels( 138 | consumer, 139 | topicName, 140 | partition.toString() 141 | ).set(value) 142 | } 143 | } -------------------------------------------------------------------------------- /monitoring-component-prometheus/src/main/resources/META-INF/native-image/jni-config.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name":"com.sun.management.internal.DiagnosticCommandArgumentInfo", 4 | "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","boolean","boolean","int"] }] 5 | }, 6 | { 7 | "name":"com.sun.management.internal.DiagnosticCommandArgumentInfo[]" 8 | }, 9 | { 10 | "name":"com.sun.management.internal.DiagnosticCommandInfo", 11 | "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","java.util.List"] }] 12 | }, 13 | { 14 | "name":"com.sun.management.internal.DiagnosticCommandInfo[]" 15 | }, 16 | { 17 | "name":"java.lang.ClassLoader", 18 | "methods":[ 19 | {"name":"getPlatformClassLoader","parameterTypes":[] }, 20 | {"name":"loadClass","parameterTypes":["java.lang.String"] } 21 | ] 22 | }, 23 | { 24 | "name":"java.lang.ClassNotFoundException" 25 | }, 26 | { 27 | "name":"java.lang.NoSuchMethodError" 28 | }, 29 | { 30 | "name":"java.util.Arrays", 31 | "methods":[{"name":"asList","parameterTypes":["java.lang.Object[]"] }] 32 | }, 33 | { 34 | "name":"sun.management.VMManagementImpl", 35 | "fields":[ 36 | {"name":"compTimeMonitoringSupport"}, 37 | {"name":"currentThreadCpuTimeSupport"}, 38 | {"name":"objectMonitorUsageSupport"}, 39 | {"name":"otherThreadCpuTimeSupport"}, 40 | {"name":"remoteDiagnosticCommandsSupport"}, 41 | {"name":"synchronizerUsageSupport"}, 42 | {"name":"threadAllocatedMemorySupport"}, 43 | {"name":"threadContentionMonitoringSupport"} 44 | ] 45 | } 46 | ] 47 | -------------------------------------------------------------------------------- /monitoring-component-prometheus/src/main/resources/META-INF/native-image/proxy-config.json: -------------------------------------------------------------------------------- 1 | [ 2 | ] 3 | -------------------------------------------------------------------------------- /monitoring-component-prometheus/src/main/resources/META-INF/native-image/resource-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "resources":[ 3 | {"pattern":"\\QMETA-INF/org/apache/logging/log4j/core/config/plugins/Log4j2Plugins.dat\\E"}, 4 | {"pattern":"\\QMETA-INF/services/com.oracle.truffle.api.TruffleLanguage$Provider\\E"}, 5 | {"pattern":"\\QMETA-INF/services/com.oracle.truffle.api.instrumentation.TruffleInstrument$Provider\\E"}, 6 | {"pattern":"\\QMETA-INF/services/org.apache.logging.log4j.core.util.ContextDataProvider\\E"}, 7 | {"pattern":"\\QMETA-INF/services/org.apache.logging.log4j.spi.Provider\\E"}, 8 | {"pattern":"\\QMETA-INF/services/org.apache.logging.log4j.util.PropertySource\\E"}, 9 | {"pattern":"\\Qcom/oracle/truffle/nfi/impl/NFILanguageImpl.class\\E"}, 10 | {"pattern":"\\Qkafka/kafka-version.properties\\E"}, 11 | {"pattern":"\\Qlog4j2.properties\\E"}, 12 | {"pattern":"\\Qorg/slf4j/impl/StaticLoggerBinder.class\\E"}, 13 | {"pattern":"\\Qconfig/log4j2.template\\E"} 14 | ], 15 | "bundles":[] 16 | } 17 | -------------------------------------------------------------------------------- /monitoring-component-prometheus/src/main/resources/configs.properties: -------------------------------------------------------------------------------- 1 | kafka.bootstrap.servers=localhost:9092 2 | kafka.retry.backoff.ms = 200 3 | monitoring.lag.consumer.groups=test* 4 | monitoring.lag.prometheus.http.port=9772 5 | monitoring.lag.logging.rootLogger.appenderRef.stdout.ref=LogToConsole 6 | monitoring.lag.logging.rootLogger.level=info -------------------------------------------------------------------------------- /monitoring-core/bin/main/com/omarsmak/kafka/consumer/lag/monitoring/component/MonitoringComponent.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.component 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.data.ConsumerGroupLag 4 | 5 | interface MonitoringComponent { 6 | /** 7 | * Configure the monitoring component with supplied configs, the configs are only specific for the component with config prefix 8 | * e.g: monitoring.lag.datadog.config-1 = test, you will get config-1=test in the Map 9 | * 10 | * @param configs provided configs for component from [MonitoringEngine] 11 | */ 12 | fun configure(configs: Map) 13 | 14 | /** 15 | * Start our monitoring component process, here we can start anything related to our component, e.g: a HTTP server or 16 | * a client 17 | */ 18 | fun start() 19 | 20 | /** 21 | * Stop our monitoring component process, here you can stop anything that you started with start() 22 | */ 23 | fun stop() 24 | 25 | /** 26 | * beforeProcess hook will be called before processing a lag for any consumer 27 | */ 28 | fun beforeProcess() 29 | 30 | /** 31 | * process hook will be called upon processing a lag per consumer by providing the following parameters 32 | * 33 | * @param consumerGroup the processed consumer group name 34 | * @param consumerGroupLag the consumer lag per topic of the consumer group 35 | */ 36 | fun process(consumerGroup : String, consumerGroupLag: ConsumerGroupLag) 37 | 38 | /** 39 | * afterProcess hook will be called after processing all lags for all consumers 40 | */ 41 | fun afterProcess() 42 | 43 | /** 44 | * Identifier for the component that can be used to parse the configs automatically by the context 45 | */ 46 | fun identifier(): String 47 | 48 | /** 49 | * Error hook to be called on error while the client is polling the data 50 | */ 51 | fun onError(t: Throwable) 52 | } -------------------------------------------------------------------------------- /monitoring-core/bin/main/com/omarsmak/kafka/consumer/lag/monitoring/data/ConsumerGroupLag.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.data 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.client.data.Lag 4 | 5 | data class ConsumerGroupLag( 6 | /** 7 | * the name of the consumer group where the lag belongs to 8 | */ 9 | val consumerGroup: String, 10 | 11 | /** 12 | * the consumer lag per topic of the consumer group, hence the list of lags here correspond to list of topics 13 | * whereby one lag represents one topic that group assigned to 14 | */ 15 | val lag: List, 16 | 17 | /** 18 | * the member lag per member per topic of the consumer group, here the key is the consumer group member and the list of lags 19 | * per topic 20 | */ 21 | val memberLag: Map> 22 | ) -------------------------------------------------------------------------------- /monitoring-core/bin/main/com/omarsmak/kafka/consumer/lag/monitoring/engine/MonitoringEngine.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.engine 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.client.KafkaConsumerLagClient 4 | import com.omarsmak.kafka.consumer.lag.monitoring.client.KafkaConsumerLagClientFactory 5 | import com.omarsmak.kafka.consumer.lag.monitoring.component.MonitoringComponent 6 | import com.omarsmak.kafka.consumer.lag.monitoring.data.ConsumerGroupLag 7 | import com.omarsmak.kafka.consumer.lag.monitoring.support.Utils 8 | import com.omarsmak.kafka.consumer.lag.monitoring.support.castToLong 9 | import mu.KotlinLogging 10 | import java.lang.Exception 11 | import java.lang.IllegalArgumentException 12 | import java.lang.NumberFormatException 13 | import java.util.Timer 14 | import kotlin.concurrent.scheduleAtFixedRate 15 | 16 | class MonitoringEngine private constructor(monitoringComponent: MonitoringComponent, configs: Map) { 17 | 18 | companion object { 19 | private val logger = KotlinLogging.logger {} 20 | private val timer = Timer() 21 | 22 | const val CONFIG_LAG_CLIENT_PREFIX = "monitoring.lag" 23 | const val CONFIG_KAFKA_PREFIX = "kafka" 24 | 25 | // default options 26 | const val POLL_INTERVAL = "poll.interval" 27 | const val DEFAULT_POLL_INTERVAL = 2000 28 | 29 | const val CONSUMER_GROUPS = "consumer.groups" 30 | 31 | fun createWithComponentAndConfigs(monitoringComponent: MonitoringComponent, configs: Map): MonitoringEngine = 32 | MonitoringEngine(monitoringComponent, configs) 33 | } 34 | 35 | val kafkaConfigs: Map 36 | val componentConfigs: Map 37 | 38 | private var trackedConsumerGroups: Set = emptySet() 39 | 40 | private lateinit var monitoringComponent: MonitoringComponent 41 | private lateinit var kafkaConsumerLagClient: KafkaConsumerLagClient 42 | 43 | init { 44 | kafkaConfigs = Utils.getConfigsWithPrefixCaseInSensitive(configs, CONFIG_KAFKA_PREFIX) 45 | componentConfigs = initializeComponentDefaultConfigs().plus(Utils.getConfigsWithPrefixCaseInSensitive(configs, CONFIG_LAG_CLIENT_PREFIX)) 46 | 47 | var kafkaLoggableConfigs = kafkaConfigs.mapNotNull{ (key, value) -> 48 | when (key) { 49 | "ssl.truststore.password" -> key to "[hidden]" 50 | "ssl.key.password" -> key to "[hidden]" 51 | "ssl.keystore.certificate.chain" -> key to "[hidden]" 52 | "sasl.jaas.config" -> key to value.toString().replace("password=\"(.*)\"".toRegex(), "password=\"[hidden]\"") 53 | else -> key to value 54 | } 55 | }.toMap() 56 | 57 | logger.info("Component Configs: $componentConfigs") 58 | logger.info("Kafka Configs: $kafkaLoggableConfigs") 59 | 60 | registerComponent(monitoringComponent) 61 | } 62 | 63 | fun start() { 64 | logger.info("Starting client...") 65 | // validate our configs 66 | validateComponentConfigs(CONSUMER_GROUPS, String, true) 67 | 68 | // start our client 69 | kafkaConsumerLagClient = KafkaConsumerLagClientFactory.create(kafkaConfigs) 70 | 71 | // start our component 72 | monitoringComponent.start() 73 | 74 | // start our context and execute our component 75 | val monitoringPollInterval = getConfigAsLong(POLL_INTERVAL) 76 | 77 | logger.info("Updating metrics every ${monitoringPollInterval}ms...") 78 | 79 | timer.scheduleAtFixedRate(0, monitoringPollInterval) { 80 | try { 81 | // get our full target consumer groups, however we do have to check here to make sure we catch any new consumer group 82 | val consumerGroups = Utils.getTargetConsumerGroups(kafkaConsumerLagClient, initializeConsumerGroups()) 83 | val diffConsumerGroups = Utils.updateAndTrackConsumerGroups(trackedConsumerGroups, consumerGroups, false) 84 | trackedConsumerGroups = diffConsumerGroups.updatedConsumerGroups 85 | 86 | logger.debug("Monitoring consumer groups: '$trackedConsumerGroups'") 87 | 88 | // we log these info 89 | if (diffConsumerGroups.newGroups.isNotEmpty()) { 90 | logger.info("New consumer groups joined the client: '${diffConsumerGroups.newGroups}'") 91 | } 92 | 93 | if (diffConsumerGroups.removedGroups.isNotEmpty()) { 94 | logger.info("Some consumer groups left the client: '${diffConsumerGroups.removedGroups}'") 95 | } 96 | 97 | // before we process the lag, call our component hook 98 | monitoringComponent.beforeProcess() 99 | 100 | // process our lag per consumer group 101 | trackedConsumerGroups.forEach { 102 | logger.debug("Polling lags for consumer '$it'...") 103 | 104 | val lag = kafkaConsumerLagClient.getConsumerLag(it) 105 | val memberLag = kafkaConsumerLagClient.getConsumerMemberLag(it) 106 | 107 | logger.debug("Consumer: $it, Lag: $lag, Member Lag: $memberLag") 108 | 109 | // process our lag per consumer 110 | monitoringComponent.process(it, ConsumerGroupLag(it, lag, memberLag)) 111 | } 112 | 113 | // after we are done, we call our component hook 114 | monitoringComponent.afterProcess() 115 | } catch (ex: Exception) { 116 | // call onError hook 117 | monitoringComponent.onError(ex) 118 | } 119 | } 120 | } 121 | 122 | fun stop() { 123 | logger.info("Stopping client...") 124 | 125 | // stop our timer 126 | timer.cancel() 127 | timer.purge() 128 | 129 | // stop our client 130 | kafkaConsumerLagClient.close() 131 | 132 | // stop our component 133 | monitoringComponent.stop() 134 | } 135 | 136 | private fun registerComponent(monitoringComponent: MonitoringComponent) { 137 | this.monitoringComponent = monitoringComponent 138 | 139 | logger.debug("Registering component: ${monitoringComponent.identifier()}") 140 | 141 | // initialize our component 142 | this.monitoringComponent.configure(initializeSpecificComponentConfigs()) 143 | } 144 | 145 | private fun initializeComponentDefaultConfigs() = mapOf( 146 | POLL_INTERVAL to DEFAULT_POLL_INTERVAL 147 | ) 148 | 149 | private fun validateComponentConfigs(key: String, type: T, required: Boolean) { 150 | val value: Any? = componentConfigs[key] 151 | 152 | // check if exists 153 | if (required && value == null) throw IllegalArgumentException("Missing required configuration '$key' which has no default value.") 154 | 155 | // check type 156 | if (type is String) { 157 | val valueAsString: String = value as String 158 | if (required && valueAsString.isEmpty()) throw IllegalArgumentException("Missing required configuration '$key' which has no default value.") 159 | } 160 | } 161 | 162 | private fun getConfigAsLong(key: String) = try { 163 | componentConfigs.getValue(key).castToLong() 164 | } catch (e: NumberFormatException) { 165 | throw IllegalArgumentException("The value '" + componentConfigs[key] + "' of key '$key' cannot be converted to long") 166 | } 167 | 168 | private fun initializeConsumerGroups(): List { 169 | val consumerGroups = componentConfigs[CONSUMER_GROUPS] as String? 170 | 171 | if (consumerGroups.isNullOrEmpty()) { 172 | throw IllegalArgumentException("Missing required configuration '$CONSUMER_GROUPS' which has no default value.") 173 | } 174 | 175 | return consumerGroups.split(",") 176 | } 177 | 178 | 179 | private fun initializeSpecificComponentConfigs(): Map = 180 | componentConfigs.filter { it.key.startsWith(monitoringComponent.identifier()) } 181 | .mapKeys { it.key.removePrefix(monitoringComponent.identifier() + ".") } 182 | } 183 | -------------------------------------------------------------------------------- /monitoring-core/bin/main/com/omarsmak/kafka/consumer/lag/monitoring/support/Utils.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.support 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.client.KafkaConsumerLagClient 4 | import com.omarsmak.kafka.consumer.lag.monitoring.engine.MonitoringEngine 5 | import mu.KotlinLogging 6 | import org.apache.logging.log4j.core.config.Configurator 7 | import org.apache.logging.log4j.core.config.properties.PropertiesConfigurationBuilder 8 | import java.io.* 9 | import java.util.Properties 10 | 11 | object Utils { 12 | 13 | const val DEFAULT_LOGGING_FILE = "config/log4j2.template" 14 | const val DEFAULT_LOGGING_PREFIX = "${MonitoringEngine.CONFIG_LAG_CLIENT_PREFIX}.logging" 15 | 16 | fun getTargetConsumerGroups(client: KafkaConsumerLagClient, configConsumerGroups: List): Set { 17 | // Get consumer groups from the kafka broker 18 | val consumerGroups = client.getConsumerGroupsList() 19 | 20 | // Fetch consumers based no a regex 21 | val matchedConsumersGroups = configConsumerGroups 22 | .filter { it.contains("*") } 23 | .map { x -> 24 | consumerGroups.filter { it.startsWith(x.replace("*", "")) } 25 | } 26 | .flatten() 27 | 28 | return configConsumerGroups 29 | .filterNot { it.contains("*") } 30 | .union(matchedConsumersGroups) 31 | } 32 | 33 | fun loadPropertiesFile(filePath: String): Properties { 34 | val inputStream: FileInputStream? = File(filePath).inputStream() 35 | val prop = Properties() 36 | 37 | if (inputStream == null) { 38 | throw FileNotFoundException("File '$filePath' not found!") 39 | } 40 | 41 | prop.load(inputStream) 42 | 43 | return prop 44 | } 45 | 46 | fun loadPropertiesFileAsMap(filePath: String): Map { 47 | val props = loadPropertiesFile(filePath) 48 | 49 | return props.toMap() 50 | .mapKeys { it.key as String } 51 | } 52 | 53 | fun loadPropertiesFromInputStream(inputStream: InputStream): Properties { 54 | val prop = Properties() 55 | 56 | prop.load(inputStream) 57 | 58 | return prop 59 | } 60 | 61 | fun getConfigsFromPropertiesFileOrFromEnv(arg: Array): Map { 62 | // first we try to load the from properties file using provided args 63 | // if we fail to find something, we use the env variables as fall back 64 | return if (arg.isNotEmpty()) loadPropertiesFileAsMap(arg[0]) else System.getenv() 65 | } 66 | 67 | fun getConfigsWithPrefixCaseInSensitive(configs: Map, prefix: String): Map { 68 | val parsedConfigs = configs.mapKeys { it.key.toLowerCase().replace("_", ".") } 69 | 70 | return getConfigsWithPrefix(parsedConfigs, prefix) 71 | } 72 | 73 | fun getConfigsWithPrefixCaseSensitive(configs: Map, prefix: String): Map { 74 | val parsedConfigs = configs.mapKeys { it.key.replace("_", ".") } 75 | 76 | return getConfigsWithPrefix(parsedConfigs, prefix) 77 | } 78 | 79 | private fun getConfigsWithPrefix(configs: Map, prefix: String): Map = configs 80 | .filter { it.key.startsWith(prefix, true) && it.value != null } 81 | .mapKeys { it.key.replace("$prefix.", "", true) } 82 | .mapValues { it.value as Any } 83 | 84 | /** 85 | * The will initialize log4j configs, however if you wish to use this, you will need to make sure that you have 86 | * log4j-slf4j-impl in the class path 87 | */ 88 | fun initializeLog4jLoggingWithConfigs(configs: Map, prefix: String = DEFAULT_LOGGING_PREFIX) { 89 | val userConfigs = getConfigsWithPrefixCaseSensitive(configs, prefix) 90 | val defaultLoggingConfig = loadPropertiesFromInputStream(DEFAULT_LOGGING_FILE.asResource()) 91 | 92 | defaultLoggingConfig.putAll(userConfigs) 93 | 94 | val defaultLoggingConfigAsMap = defaultLoggingConfig.toMap() 95 | 96 | Configurator.initialize(PropertiesConfigurationBuilder().setRootProperties(defaultLoggingConfig).build()) 97 | 98 | KotlinLogging.logger {}.info("Logging Configs: $defaultLoggingConfigAsMap") 99 | } 100 | 101 | fun updateAndTrackConsumerGroups(trackedConsumerGroups: Set, sourceConsumerGroups: Set, keepGroups: Boolean = true): DiffConsumerGroups { 102 | // first check if we have new consumer groups 103 | val newGroups = sourceConsumerGroups.minus(trackedConsumerGroups) 104 | 105 | // second check if we have removed groups 106 | val removedGroups = trackedConsumerGroups.minus(sourceConsumerGroups) 107 | 108 | // update our trackedConsumerGroups 109 | if (keepGroups) { 110 | return DiffConsumerGroups(newGroups, removedGroups, trackedConsumerGroups 111 | .plus(newGroups)) 112 | } 113 | 114 | // update our trackedConsumerGroups 115 | return DiffConsumerGroups(newGroups, removedGroups, trackedConsumerGroups 116 | .plus(newGroups).minus(removedGroups)) 117 | } 118 | 119 | data class DiffConsumerGroups( 120 | val newGroups: Set, 121 | val removedGroups: Set, 122 | val updatedConsumerGroups: Set 123 | ) 124 | } 125 | -------------------------------------------------------------------------------- /monitoring-core/bin/main/com/omarsmak/kafka/consumer/lag/monitoring/support/extensions.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.support 2 | 3 | import java.io.FileNotFoundException 4 | import java.io.InputStream 5 | import java.net.URL 6 | 7 | 8 | fun Any.castToInt(): Int { 9 | if (this is String) return this.toInt() 10 | 11 | return (this as Number).toInt() 12 | } 13 | 14 | fun Any.castToLong(): Long { 15 | if (this is String) return this.toLong() 16 | 17 | return (this as Number).toLong() 18 | } 19 | 20 | fun String.asResource(): InputStream = 21 | object {}.javaClass.getResourceAsStream("/$this") ?: throw FileNotFoundException("File '$this' not found..") 22 | -------------------------------------------------------------------------------- /monitoring-core/bin/main/config/log4j2.template: -------------------------------------------------------------------------------- 1 | # Log to console 2 | appender.console.type = Console 3 | appender.console.name = LogToConsole 4 | appender.console.layout.type = PatternLayout 5 | appender.console.layout.pattern = [%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n 6 | 7 | # Log to console as JSON 8 | appender.json.type = Console 9 | appender.json.name = LogInJSON 10 | appender.json.layout.type = JsonLayout 11 | appender.json.layout.complete = true 12 | appender.json.layout.compact = false 13 | 14 | rootLogger.level = info 15 | rootLogger.appenderRef.stdout.ref = LogInJSON -------------------------------------------------------------------------------- /monitoring-core/bin/test/com/omarsmak/kafka/consumer/lag/monitoring/engine/MonitoringEngineTest.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.engine 2 | 3 | import org.junit.jupiter.api.Assertions.* 4 | import org.junit.jupiter.api.Test 5 | 6 | internal class MonitoringEngineTest { 7 | 8 | @Test 9 | fun testConfigs() { 10 | val monitoringComponent = TestMonitoringComponent() 11 | val configs = mapOf( 12 | "kafka.bootstrap.server" to "localhost:9090" , 13 | "kafka.poll.interval" to "900", 14 | "KAFKA_FETCH_RATE" to "100", 15 | "monitoring.lag.consumer.groups" to "test1,test2", 16 | "monitoring.lag.datadog.poll.interval" to "300", 17 | "monitoring.lag.poll.interval" to null, 18 | "monitoring.lag." + monitoringComponent.identifier() +".config" to "test", 19 | "nothing" to "nothing" 20 | ) 21 | 22 | val context = MonitoringEngine.createWithComponentAndConfigs(TestMonitoringComponent(), configs) 23 | 24 | // assert only kafka configs 25 | assertEquals(3, context.kafkaConfigs.size) 26 | assertEquals("localhost:9090", context.kafkaConfigs["bootstrap.server"]) 27 | assertEquals("900", context.kafkaConfigs["poll.interval"]) 28 | assertEquals("100", context.kafkaConfigs["fetch.rate"]) 29 | 30 | // assert only lag configs 31 | assertEquals(4, context.componentConfigs.size) 32 | assertEquals("test1,test2", context.componentConfigs["consumer.groups"]) 33 | assertEquals("300", context.componentConfigs["datadog.poll.interval"]) 34 | assertEquals(2000, context.componentConfigs["poll.interval"]) 35 | 36 | // assert component config 37 | assertEquals("test", context.componentConfigs["test.component.config"]) 38 | } 39 | } -------------------------------------------------------------------------------- /monitoring-core/bin/test/com/omarsmak/kafka/consumer/lag/monitoring/engine/TestMonitoringComponent.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.engine 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.component.MonitoringComponent 4 | import com.omarsmak.kafka.consumer.lag.monitoring.data.ConsumerGroupLag 5 | 6 | class TestMonitoringComponent: MonitoringComponent { 7 | override fun configure(configs: Map) { 8 | println(configs) 9 | } 10 | 11 | override fun start() { 12 | } 13 | 14 | override fun stop() { 15 | } 16 | 17 | override fun beforeProcess() { 18 | } 19 | 20 | override fun process(consumerGroup: String, consumerGroupLag: ConsumerGroupLag) { 21 | println("$consumerGroup $consumerGroupLag") 22 | } 23 | 24 | override fun afterProcess() { 25 | } 26 | 27 | override fun identifier(): String = "test.component" 28 | 29 | override fun onError(t: Throwable) { 30 | println(t) 31 | } 32 | } -------------------------------------------------------------------------------- /monitoring-core/bin/test/com/omarsmak/kafka/consumer/lag/monitoring/support/ExtensionsTest.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.support 2 | 3 | import org.junit.jupiter.api.Test 4 | import kotlin.test.assertEquals 5 | 6 | 7 | class ExtensionsTest { 8 | 9 | @Test 10 | fun toIntTest() { 11 | val test: Any = "12345" 12 | val test2: Any = 12345L 13 | 14 | assertEquals(12345, test.castToInt()) 15 | assertEquals(12345, test2.castToInt()) 16 | } 17 | 18 | @Test 19 | fun toLongTest() { 20 | val test: Any = "12345" 21 | val test2: Any = 12345 22 | 23 | assertEquals(12345L, test.castToLong()) 24 | assertEquals(12345L, test2.castToLong()) 25 | } 26 | } -------------------------------------------------------------------------------- /monitoring-core/bin/test/com/omarsmak/kafka/consumer/lag/monitoring/support/UtilsTest.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.support 2 | 3 | import org.junit.jupiter.api.Assertions.assertEquals 4 | import org.junit.jupiter.api.Test 5 | import kotlin.test.assertTrue 6 | 7 | internal class UtilsTest { 8 | 9 | @Test 10 | fun testGetConfigsWithPrefixCaseSensitive() { 11 | val configs = mapOf( 12 | "PREFIX_TEST_test_configTest_1" to "test-config-1", 13 | "prefix.test.test.configTest.2" to "test-config-2" 14 | ) 15 | 16 | val parsedConfigs = Utils.getConfigsWithPrefixCaseSensitive(configs, "prefix.test") 17 | 18 | assertEquals("test-config-1", parsedConfigs["test.configTest.1"]) 19 | assertEquals("test-config-2", parsedConfigs["test.configTest.2"]) 20 | } 21 | 22 | @Test 23 | fun testGetConfigsWithPrefixCaseInSensitive() { 24 | val configs = mapOf( 25 | "PREFIX_TEST_test_configTest_1" to "test-config-1", 26 | "prefix.test.test.configTest.2" to "test-config-2" 27 | ) 28 | 29 | val parsedConfigs = Utils.getConfigsWithPrefixCaseInSensitive(configs, "prefix.test") 30 | 31 | assertEquals("test-config-1", parsedConfigs["test.configtest.1"]) 32 | assertEquals("test-config-2", parsedConfigs["test.configtest.2"]) 33 | } 34 | 35 | @Test 36 | fun testUpdateAndTrackConsumerGroups() { 37 | // first if both the same 38 | assertEquals(setOf("test"), Utils.updateAndTrackConsumerGroups(setOf("test"), setOf("test")).updatedConsumerGroups) 39 | 40 | // removed groups 41 | assertEquals(setOf("test"), Utils.updateAndTrackConsumerGroups(setOf("test-1", "test-2"), setOf("test"), false).updatedConsumerGroups) 42 | 43 | // added groups 44 | assertEquals(setOf("test-1", "test-2"), Utils.updateAndTrackConsumerGroups(setOf("test-1"), setOf("test-2", "test-1"), false).updatedConsumerGroups) 45 | 46 | // no groups 47 | assertTrue(Utils.updateAndTrackConsumerGroups(setOf("test-1"), emptySet(), false).updatedConsumerGroups.isEmpty()) 48 | 49 | // new groups 50 | assertEquals(setOf("test"), Utils.updateAndTrackConsumerGroups(emptySet(), setOf("test")).updatedConsumerGroups) 51 | 52 | 53 | // with kept groups 54 | // removed groups 55 | assertEquals(setOf("test-1", "test-2","test"), Utils.updateAndTrackConsumerGroups(setOf("test-1", "test-2"), setOf("test")).updatedConsumerGroups) 56 | 57 | // added groups 58 | assertEquals(setOf("test-1", "test-2"), Utils.updateAndTrackConsumerGroups(setOf("test-1"), setOf("test-2", "test-1")).updatedConsumerGroups) 59 | 60 | // no groups 61 | assertEquals(setOf("test-1"), Utils.updateAndTrackConsumerGroups(setOf("test-1"), emptySet()).updatedConsumerGroups) 62 | } 63 | } -------------------------------------------------------------------------------- /monitoring-core/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | compile project(':monitoring-client') 3 | 4 | compile "io.github.microutils:kotlin-logging:$kotlinLoggingVersion" 5 | compileOnly "org.apache.logging.log4j:log4j-core:$log4jSlf4jImplVersion" 6 | 7 | testImplementation "org.junit.jupiter:junit-jupiter-api:${junitVersion}" 8 | testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${junitVersion}" 9 | testCompile "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" 10 | } -------------------------------------------------------------------------------- /monitoring-core/src/main/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/component/MonitoringComponent.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.component 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.data.ConsumerGroupLag 4 | 5 | interface MonitoringComponent { 6 | /** 7 | * Configure the monitoring component with supplied configs, the configs are only specific for the component with config prefix 8 | * e.g: monitoring.lag.datadog.config-1 = test, you will get config-1=test in the Map 9 | * 10 | * @param configs provided configs for component from [MonitoringEngine] 11 | */ 12 | fun configure(configs: Map) 13 | 14 | /** 15 | * Start our monitoring component process, here we can start anything related to our component, e.g: a HTTP server or 16 | * a client 17 | */ 18 | fun start() 19 | 20 | /** 21 | * Stop our monitoring component process, here you can stop anything that you started with start() 22 | */ 23 | fun stop() 24 | 25 | /** 26 | * beforeProcess hook will be called before processing a lag for any consumer 27 | */ 28 | fun beforeProcess() 29 | 30 | /** 31 | * process hook will be called upon processing a lag per consumer by providing the following parameters 32 | * 33 | * @param consumerGroup the processed consumer group name 34 | * @param consumerGroupLag the consumer lag per topic of the consumer group 35 | */ 36 | fun process(consumerGroup : String, consumerGroupLag: ConsumerGroupLag) 37 | 38 | /** 39 | * afterProcess hook will be called after processing all lags for all consumers 40 | */ 41 | fun afterProcess() 42 | 43 | /** 44 | * Identifier for the component that can be used to parse the configs automatically by the context 45 | */ 46 | fun identifier(): String 47 | 48 | /** 49 | * Error hook to be called on error while the client is polling the data 50 | */ 51 | fun onError(t: Throwable) 52 | } -------------------------------------------------------------------------------- /monitoring-core/src/main/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/data/ConsumerGroupLag.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.data 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.client.data.Lag 4 | 5 | data class ConsumerGroupLag( 6 | /** 7 | * the name of the consumer group where the lag belongs to 8 | */ 9 | val consumerGroup: String, 10 | 11 | /** 12 | * the consumer lag per topic of the consumer group, hence the list of lags here correspond to list of topics 13 | * whereby one lag represents one topic that group assigned to 14 | */ 15 | val lag: List, 16 | 17 | /** 18 | * the member lag per member per topic of the consumer group, here the key is the consumer group member and the list of lags 19 | * per topic 20 | */ 21 | val memberLag: Map> 22 | ) -------------------------------------------------------------------------------- /monitoring-core/src/main/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/engine/MonitoringEngine.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.engine 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.client.KafkaConsumerLagClient 4 | import com.omarsmak.kafka.consumer.lag.monitoring.client.KafkaConsumerLagClientFactory 5 | import com.omarsmak.kafka.consumer.lag.monitoring.component.MonitoringComponent 6 | import com.omarsmak.kafka.consumer.lag.monitoring.data.ConsumerGroupLag 7 | import com.omarsmak.kafka.consumer.lag.monitoring.support.Utils 8 | import com.omarsmak.kafka.consumer.lag.monitoring.support.castToLong 9 | import mu.KotlinLogging 10 | import java.lang.Exception 11 | import java.lang.IllegalArgumentException 12 | import java.lang.NumberFormatException 13 | import java.util.Timer 14 | import kotlin.concurrent.scheduleAtFixedRate 15 | 16 | class MonitoringEngine private constructor(monitoringComponent: MonitoringComponent, configs: Map) { 17 | 18 | companion object { 19 | private val logger = KotlinLogging.logger {} 20 | private val timer = Timer() 21 | 22 | const val CONFIG_LAG_CLIENT_PREFIX = "monitoring.lag" 23 | const val CONFIG_KAFKA_PREFIX = "kafka" 24 | 25 | // default options 26 | const val POLL_INTERVAL = "poll.interval" 27 | const val DEFAULT_POLL_INTERVAL = 2000 28 | 29 | const val CONSUMER_GROUPS = "consumer.groups" 30 | const val EXCLUDED_CONSUMER_GROUPS = "consumer.groups.exclude" 31 | 32 | fun createWithComponentAndConfigs(monitoringComponent: MonitoringComponent, configs: Map): MonitoringEngine = 33 | MonitoringEngine(monitoringComponent, configs) 34 | } 35 | 36 | val kafkaConfigs: Map 37 | val componentConfigs: Map 38 | 39 | private var trackedConsumerGroups: Set = emptySet() 40 | 41 | private lateinit var monitoringComponent: MonitoringComponent 42 | private lateinit var kafkaConsumerLagClient: KafkaConsumerLagClient 43 | 44 | init { 45 | kafkaConfigs = Utils.getConfigsWithPrefixCaseInSensitive(configs, CONFIG_KAFKA_PREFIX) 46 | componentConfigs = initializeComponentDefaultConfigs().plus(Utils.getConfigsWithPrefixCaseInSensitive(configs, CONFIG_LAG_CLIENT_PREFIX)) 47 | 48 | var kafkaLoggableConfigs = kafkaConfigs.mapNotNull{ (key, value) -> 49 | when (key) { 50 | "ssl.truststore.password" -> key to "[hidden]" 51 | "ssl.key.password" -> key to "[hidden]" 52 | "ssl.keystore.certificate.chain" -> key to "[hidden]" 53 | "sasl.jaas.config" -> key to value.toString().replace("password=\"(.*)\"".toRegex(), "password=\"[hidden]\"") 54 | else -> key to value 55 | } 56 | }.toMap() 57 | 58 | logger.info("Component Configs: $componentConfigs") 59 | logger.info("Kafka Configs: $kafkaLoggableConfigs") 60 | 61 | registerComponent(monitoringComponent) 62 | } 63 | 64 | fun start() { 65 | logger.info("Starting client...") 66 | // validate our configs 67 | validateComponentConfigs(CONSUMER_GROUPS, String, true) 68 | 69 | // start our client 70 | kafkaConsumerLagClient = KafkaConsumerLagClientFactory.create(kafkaConfigs) 71 | 72 | // start our component 73 | monitoringComponent.start() 74 | 75 | // start our context and execute our component 76 | val monitoringPollInterval = getConfigAsLong(POLL_INTERVAL) 77 | 78 | logger.info("Updating metrics every ${monitoringPollInterval}ms...") 79 | 80 | timer.scheduleAtFixedRate(0, monitoringPollInterval) { 81 | try { 82 | // get our full target consumer groups, however we do have to check here to make sure we catch any new consumer group 83 | val consumerGroups = Utils.getTargetConsumerGroups(kafkaConsumerLagClient, initializeConsumerGroups(), initializeExcludedConsumerGroups()) 84 | val diffConsumerGroups = Utils.updateAndTrackConsumerGroups(trackedConsumerGroups, consumerGroups, false) 85 | trackedConsumerGroups = diffConsumerGroups.updatedConsumerGroups 86 | 87 | logger.debug("Monitoring consumer groups: '$trackedConsumerGroups'") 88 | 89 | // we log these info 90 | if (diffConsumerGroups.newGroups.isNotEmpty()) { 91 | logger.info("New consumer groups joined the client: '${diffConsumerGroups.newGroups}'") 92 | } 93 | 94 | if (diffConsumerGroups.removedGroups.isNotEmpty()) { 95 | logger.info("Some consumer groups left the client: '${diffConsumerGroups.removedGroups}'") 96 | } 97 | 98 | // before we process the lag, call our component hook 99 | monitoringComponent.beforeProcess() 100 | 101 | // process our lag per consumer group 102 | trackedConsumerGroups.forEach { 103 | logger.debug("Polling lags for consumer '$it'...") 104 | 105 | val lag = kafkaConsumerLagClient.getConsumerLag(it) 106 | val memberLag = kafkaConsumerLagClient.getConsumerMemberLag(it) 107 | 108 | logger.debug("Consumer: $it, Lag: $lag, Member Lag: $memberLag") 109 | 110 | // process our lag per consumer 111 | monitoringComponent.process(it, ConsumerGroupLag(it, lag, memberLag)) 112 | } 113 | 114 | // after we are done, we call our component hook 115 | monitoringComponent.afterProcess() 116 | } catch (ex: Exception) { 117 | // call onError hook 118 | monitoringComponent.onError(ex) 119 | } 120 | } 121 | } 122 | 123 | fun stop() { 124 | logger.info("Stopping client...") 125 | 126 | // stop our timer 127 | timer.cancel() 128 | timer.purge() 129 | 130 | // stop our client 131 | kafkaConsumerLagClient.close() 132 | 133 | // stop our component 134 | monitoringComponent.stop() 135 | } 136 | 137 | private fun registerComponent(monitoringComponent: MonitoringComponent) { 138 | this.monitoringComponent = monitoringComponent 139 | 140 | logger.debug("Registering component: ${monitoringComponent.identifier()}") 141 | 142 | // initialize our component 143 | this.monitoringComponent.configure(initializeSpecificComponentConfigs()) 144 | } 145 | 146 | private fun initializeComponentDefaultConfigs() = mapOf( 147 | POLL_INTERVAL to DEFAULT_POLL_INTERVAL 148 | ) 149 | 150 | private fun validateComponentConfigs(key: String, type: T, required: Boolean) { 151 | val value: Any? = componentConfigs[key] 152 | 153 | // check if exists 154 | if (required && value == null) throw IllegalArgumentException("Missing required configuration '$key' which has no default value.") 155 | 156 | // check type 157 | if (type is String) { 158 | val valueAsString: String = value as String 159 | if (required && valueAsString.isEmpty()) throw IllegalArgumentException("Missing required configuration '$key' which has no default value.") 160 | } 161 | } 162 | 163 | private fun getConfigAsLong(key: String) = try { 164 | componentConfigs.getValue(key).castToLong() 165 | } catch (e: NumberFormatException) { 166 | throw IllegalArgumentException("The value '" + componentConfigs[key] + "' of key '$key' cannot be converted to long") 167 | } 168 | 169 | private fun initializeConsumerGroups(): List { 170 | val consumerGroups = componentConfigs[CONSUMER_GROUPS] as String? 171 | 172 | if (consumerGroups.isNullOrEmpty()) { 173 | throw IllegalArgumentException("Missing required configuration '$CONSUMER_GROUPS' which has no default value.") 174 | } 175 | 176 | return consumerGroups.split(",") 177 | } 178 | 179 | private fun initializeExcludedConsumerGroups(): List { 180 | val excludedConsumerGroups = componentConfigs[EXCLUDED_CONSUMER_GROUPS] as String? 181 | 182 | if (excludedConsumerGroups.isNullOrEmpty()) { 183 | return listOf(); 184 | } 185 | 186 | return excludedConsumerGroups.split(",") 187 | } 188 | 189 | 190 | private fun initializeSpecificComponentConfigs(): Map = 191 | componentConfigs.filter { it.key.startsWith(monitoringComponent.identifier()) } 192 | .mapKeys { it.key.removePrefix(monitoringComponent.identifier() + ".") } 193 | } 194 | -------------------------------------------------------------------------------- /monitoring-core/src/main/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/support/Utils.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.support 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.client.KafkaConsumerLagClient 4 | import com.omarsmak.kafka.consumer.lag.monitoring.engine.MonitoringEngine 5 | import mu.KotlinLogging 6 | import org.apache.logging.log4j.core.config.Configurator 7 | import org.apache.logging.log4j.core.config.properties.PropertiesConfigurationBuilder 8 | import java.io.* 9 | import java.util.Properties 10 | 11 | object Utils { 12 | 13 | const val DEFAULT_LOGGING_FILE = "config/log4j2.template" 14 | const val DEFAULT_LOGGING_PREFIX = "${MonitoringEngine.CONFIG_LAG_CLIENT_PREFIX}.logging" 15 | 16 | fun getTargetConsumerGroups(client: KafkaConsumerLagClient, configConsumerGroups: List, configExcludedConsumerGroups: List): Set { 17 | // Get consumer groups from the kafka broker 18 | val consumerGroups = client.getConsumerGroupsList() 19 | 20 | // Fetch consumers based no a regex 21 | val matchedConsumersGroups = configConsumerGroups 22 | .filter { it.contains("*") } 23 | .map { x -> 24 | consumerGroups.filter { 25 | if (x.startsWith("*") && x.endsWith("*")) 26 | it.contains(x.replace("*", "")) 27 | else if (x.endsWith("*")) 28 | it.startsWith(x.replace("*", "")) 29 | else if (x.startsWith("*")) 30 | it.endsWith(x.replace("*", "")) 31 | else 32 | false 33 | } 34 | } 35 | .flatten() 36 | .union( 37 | configConsumerGroups 38 | .filterNot { it.contains("*") } 39 | ) 40 | 41 | val exludedConsumersGroups = configExcludedConsumerGroups 42 | .filter { it.contains("*") } 43 | .map { x -> 44 | consumerGroups.filter { 45 | if (x.startsWith("*") && x.endsWith("*")) 46 | it.contains(x.replace("*", "")) 47 | else if (x.endsWith("*")) 48 | it.startsWith(x.replace("*", "")) 49 | else if (x.startsWith("*")) 50 | it.endsWith(x.replace("*", "")) 51 | else 52 | false 53 | } 54 | } 55 | .flatten() 56 | .union( 57 | configExcludedConsumerGroups 58 | .filterNot { it.contains("*") } 59 | ) 60 | 61 | return matchedConsumersGroups.minus(exludedConsumersGroups) 62 | } 63 | 64 | fun loadPropertiesFile(filePath: String): Properties { 65 | val inputStream: FileInputStream? = File(filePath).inputStream() 66 | val prop = Properties() 67 | 68 | if (inputStream == null) { 69 | throw FileNotFoundException("File '$filePath' not found!") 70 | } 71 | 72 | prop.load(inputStream) 73 | 74 | return prop 75 | } 76 | 77 | fun loadPropertiesFileAsMap(filePath: String): Map { 78 | val props = loadPropertiesFile(filePath) 79 | 80 | return props.toMap() 81 | .mapKeys { it.key as String } 82 | } 83 | 84 | fun loadPropertiesFromInputStream(inputStream: InputStream): Properties { 85 | val prop = Properties() 86 | 87 | prop.load(inputStream) 88 | 89 | return prop 90 | } 91 | 92 | fun getConfigsFromPropertiesFileOrFromEnv(arg: Array): Map { 93 | // first we try to load the from properties file using provided args 94 | // if we fail to find something, we use the env variables as fall back 95 | return if (arg.isNotEmpty()) loadPropertiesFileAsMap(arg[0]) else System.getenv() 96 | } 97 | 98 | fun getConfigsWithPrefixCaseInSensitive(configs: Map, prefix: String): Map { 99 | val parsedConfigs = configs.mapKeys { it.key.toLowerCase().replace("_", ".") } 100 | 101 | return getConfigsWithPrefix(parsedConfigs, prefix) 102 | } 103 | 104 | fun getConfigsWithPrefixCaseSensitive(configs: Map, prefix: String): Map { 105 | val parsedConfigs = configs.mapKeys { it.key.replace("_", ".") } 106 | 107 | return getConfigsWithPrefix(parsedConfigs, prefix) 108 | } 109 | 110 | private fun getConfigsWithPrefix(configs: Map, prefix: String): Map = configs 111 | .filter { it.key.startsWith(prefix, true) && it.value != null } 112 | .mapKeys { it.key.replace("$prefix.", "", true) } 113 | .mapValues { it.value as Any } 114 | 115 | /** 116 | * The will initialize log4j configs, however if you wish to use this, you will need to make sure that you have 117 | * log4j-slf4j-impl in the class path 118 | */ 119 | fun initializeLog4jLoggingWithConfigs(configs: Map, prefix: String = DEFAULT_LOGGING_PREFIX) { 120 | val userConfigs = getConfigsWithPrefixCaseSensitive(configs, prefix) 121 | val defaultLoggingConfig = loadPropertiesFromInputStream(DEFAULT_LOGGING_FILE.asResource()) 122 | 123 | defaultLoggingConfig.putAll(userConfigs) 124 | 125 | val defaultLoggingConfigAsMap = defaultLoggingConfig.toMap() 126 | 127 | Configurator.initialize(PropertiesConfigurationBuilder().setRootProperties(defaultLoggingConfig).build()) 128 | 129 | KotlinLogging.logger {}.info("Logging Configs: $defaultLoggingConfigAsMap") 130 | } 131 | 132 | fun updateAndTrackConsumerGroups(trackedConsumerGroups: Set, sourceConsumerGroups: Set, keepGroups: Boolean = true): DiffConsumerGroups { 133 | // first check if we have new consumer groups 134 | val newGroups = sourceConsumerGroups.minus(trackedConsumerGroups) 135 | 136 | // second check if we have removed groups 137 | val removedGroups = trackedConsumerGroups.minus(sourceConsumerGroups) 138 | 139 | // update our trackedConsumerGroups 140 | if (keepGroups) { 141 | return DiffConsumerGroups(newGroups, removedGroups, trackedConsumerGroups 142 | .plus(newGroups)) 143 | } 144 | 145 | // update our trackedConsumerGroups 146 | return DiffConsumerGroups(newGroups, removedGroups, trackedConsumerGroups 147 | .plus(newGroups).minus(removedGroups)) 148 | } 149 | 150 | data class DiffConsumerGroups( 151 | val newGroups: Set, 152 | val removedGroups: Set, 153 | val updatedConsumerGroups: Set 154 | ) 155 | } 156 | -------------------------------------------------------------------------------- /monitoring-core/src/main/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/support/extensions.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.support 2 | 3 | import java.io.FileNotFoundException 4 | import java.io.InputStream 5 | import java.net.URL 6 | 7 | 8 | fun Any.castToInt(): Int { 9 | if (this is String) return this.toInt() 10 | 11 | return (this as Number).toInt() 12 | } 13 | 14 | fun Any.castToLong(): Long { 15 | if (this is String) return this.toLong() 16 | 17 | return (this as Number).toLong() 18 | } 19 | 20 | fun String.asResource(): InputStream = 21 | object {}.javaClass.getResourceAsStream("/$this") ?: throw FileNotFoundException("File '$this' not found..") 22 | -------------------------------------------------------------------------------- /monitoring-core/src/main/resources/config/log4j2.template: -------------------------------------------------------------------------------- 1 | # Log to console 2 | appender.console.type = Console 3 | appender.console.name = LogToConsole 4 | appender.console.layout.type = PatternLayout 5 | appender.console.layout.pattern = [%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n 6 | 7 | # Log to console as JSON 8 | appender.json.type = Console 9 | appender.json.name = LogInJSON 10 | appender.json.layout.type = JsonLayout 11 | appender.json.layout.complete = true 12 | appender.json.layout.compact = false 13 | 14 | rootLogger.level = info 15 | rootLogger.appenderRef.stdout.ref = LogInJSON -------------------------------------------------------------------------------- /monitoring-core/src/test/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/engine/MonitoringEngineTest.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.engine 2 | 3 | import org.junit.jupiter.api.Assertions.* 4 | import org.junit.jupiter.api.Test 5 | 6 | internal class MonitoringEngineTest { 7 | 8 | @Test 9 | fun testConfigs() { 10 | val monitoringComponent = TestMonitoringComponent() 11 | val configs = mapOf( 12 | "kafka.bootstrap.server" to "localhost:9090" , 13 | "kafka.poll.interval" to "900", 14 | "KAFKA_FETCH_RATE" to "100", 15 | "monitoring.lag.consumer.groups" to "test1,test2", 16 | "monitoring.lag.datadog.poll.interval" to "300", 17 | "monitoring.lag.poll.interval" to null, 18 | "monitoring.lag." + monitoringComponent.identifier() +".config" to "test", 19 | "nothing" to "nothing" 20 | ) 21 | 22 | val context = MonitoringEngine.createWithComponentAndConfigs(TestMonitoringComponent(), configs) 23 | 24 | // assert only kafka configs 25 | assertEquals(3, context.kafkaConfigs.size) 26 | assertEquals("localhost:9090", context.kafkaConfigs["bootstrap.server"]) 27 | assertEquals("900", context.kafkaConfigs["poll.interval"]) 28 | assertEquals("100", context.kafkaConfigs["fetch.rate"]) 29 | 30 | // assert only lag configs 31 | assertEquals(4, context.componentConfigs.size) 32 | assertEquals("test1,test2", context.componentConfigs["consumer.groups"]) 33 | assertEquals("300", context.componentConfigs["datadog.poll.interval"]) 34 | assertEquals(2000, context.componentConfigs["poll.interval"]) 35 | 36 | // assert component config 37 | assertEquals("test", context.componentConfigs["test.component.config"]) 38 | } 39 | } -------------------------------------------------------------------------------- /monitoring-core/src/test/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/engine/TestMonitoringComponent.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.engine 2 | 3 | import com.omarsmak.kafka.consumer.lag.monitoring.component.MonitoringComponent 4 | import com.omarsmak.kafka.consumer.lag.monitoring.data.ConsumerGroupLag 5 | 6 | class TestMonitoringComponent: MonitoringComponent { 7 | override fun configure(configs: Map) { 8 | println(configs) 9 | } 10 | 11 | override fun start() { 12 | } 13 | 14 | override fun stop() { 15 | } 16 | 17 | override fun beforeProcess() { 18 | } 19 | 20 | override fun process(consumerGroup: String, consumerGroupLag: ConsumerGroupLag) { 21 | println("$consumerGroup $consumerGroupLag") 22 | } 23 | 24 | override fun afterProcess() { 25 | } 26 | 27 | override fun identifier(): String = "test.component" 28 | 29 | override fun onError(t: Throwable) { 30 | println(t) 31 | } 32 | } -------------------------------------------------------------------------------- /monitoring-core/src/test/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/support/ExtensionsTest.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.support 2 | 3 | import org.junit.jupiter.api.Test 4 | import kotlin.test.assertEquals 5 | 6 | 7 | class ExtensionsTest { 8 | 9 | @Test 10 | fun toIntTest() { 11 | val test: Any = "12345" 12 | val test2: Any = 12345L 13 | 14 | assertEquals(12345, test.castToInt()) 15 | assertEquals(12345, test2.castToInt()) 16 | } 17 | 18 | @Test 19 | fun toLongTest() { 20 | val test: Any = "12345" 21 | val test2: Any = 12345 22 | 23 | assertEquals(12345L, test.castToLong()) 24 | assertEquals(12345L, test2.castToLong()) 25 | } 26 | } -------------------------------------------------------------------------------- /monitoring-core/src/test/kotlin/com/omarsmak/kafka/consumer/lag/monitoring/support/UtilsTest.kt: -------------------------------------------------------------------------------- 1 | package com.omarsmak.kafka.consumer.lag.monitoring.support 2 | 3 | import org.junit.jupiter.api.Assertions.assertEquals 4 | import org.junit.jupiter.api.Test 5 | import kotlin.test.assertTrue 6 | 7 | internal class UtilsTest { 8 | 9 | @Test 10 | fun testGetConfigsWithPrefixCaseSensitive() { 11 | val configs = mapOf( 12 | "PREFIX_TEST_test_configTest_1" to "test-config-1", 13 | "prefix.test.test.configTest.2" to "test-config-2" 14 | ) 15 | 16 | val parsedConfigs = Utils.getConfigsWithPrefixCaseSensitive(configs, "prefix.test") 17 | 18 | assertEquals("test-config-1", parsedConfigs["test.configTest.1"]) 19 | assertEquals("test-config-2", parsedConfigs["test.configTest.2"]) 20 | } 21 | 22 | @Test 23 | fun testGetConfigsWithPrefixCaseInSensitive() { 24 | val configs = mapOf( 25 | "PREFIX_TEST_test_configTest_1" to "test-config-1", 26 | "prefix.test.test.configTest.2" to "test-config-2" 27 | ) 28 | 29 | val parsedConfigs = Utils.getConfigsWithPrefixCaseInSensitive(configs, "prefix.test") 30 | 31 | assertEquals("test-config-1", parsedConfigs["test.configtest.1"]) 32 | assertEquals("test-config-2", parsedConfigs["test.configtest.2"]) 33 | } 34 | 35 | @Test 36 | fun testUpdateAndTrackConsumerGroups() { 37 | // first if both the same 38 | assertEquals(setOf("test"), Utils.updateAndTrackConsumerGroups(setOf("test"), setOf("test")).updatedConsumerGroups) 39 | 40 | // removed groups 41 | assertEquals(setOf("test"), Utils.updateAndTrackConsumerGroups(setOf("test-1", "test-2"), setOf("test"), false).updatedConsumerGroups) 42 | 43 | // added groups 44 | assertEquals(setOf("test-1", "test-2"), Utils.updateAndTrackConsumerGroups(setOf("test-1"), setOf("test-2", "test-1"), false).updatedConsumerGroups) 45 | 46 | // no groups 47 | assertTrue(Utils.updateAndTrackConsumerGroups(setOf("test-1"), emptySet(), false).updatedConsumerGroups.isEmpty()) 48 | 49 | // new groups 50 | assertEquals(setOf("test"), Utils.updateAndTrackConsumerGroups(emptySet(), setOf("test")).updatedConsumerGroups) 51 | 52 | 53 | // with kept groups 54 | // removed groups 55 | assertEquals(setOf("test-1", "test-2","test"), Utils.updateAndTrackConsumerGroups(setOf("test-1", "test-2"), setOf("test")).updatedConsumerGroups) 56 | 57 | // added groups 58 | assertEquals(setOf("test-1", "test-2"), Utils.updateAndTrackConsumerGroups(setOf("test-1"), setOf("test-2", "test-1")).updatedConsumerGroups) 59 | 60 | // no groups 61 | assertEquals(setOf("test-1"), Utils.updateAndTrackConsumerGroups(setOf("test-1"), emptySet()).updatedConsumerGroups) 62 | } 63 | } -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = "kafka-consumer-lag-monitoring" 2 | 3 | include 'monitoring-client' 4 | include 'monitoring-core' 5 | include 'monitoring-component-prometheus' 6 | include 'monitoring-component-console' --------------------------------------------------------------------------------