├── .github ├── dependabot.yml └── workflows │ └── gradle.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── build.gradle.kts ├── docker-compose.yml ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── grafana └── provisioning │ ├── dashboards │ ├── Flink.json │ └── dashboards.yml │ └── datasources │ └── prometheus.yml ├── prometheus ├── flink.rules.yml └── prometheus.yml └── src ├── integrationTest └── java │ └── com │ └── github │ └── mbode │ └── flink_prometheus_example │ ├── FlinkIT.java │ ├── GrafanaIT.java │ └── PrometheusIT.java ├── main └── java │ └── com │ └── github │ └── mbode │ └── flink_prometheus_example │ ├── FlinkMetricsExposingMapFunction.java │ ├── PrometheusExampleJob.java │ └── RandomSourceFunction.java └── test └── java └── com └── github └── mbode └── flink_prometheus_example ├── CollectSink.java ├── FlinkMetricsExposingMapFunctionTest.java ├── PrometheusExampleJobTest.java └── RandomSourceFunctionTest.java /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gradle" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | - package-ecosystem: "docker" 8 | directory: "/" 9 | schedule: 10 | interval: "daily" 11 | - package-ecosystem: "github-actions" 12 | directory: "/" 13 | schedule: 14 | interval: "daily" 15 | -------------------------------------------------------------------------------- /.github/workflows/gradle.yml: -------------------------------------------------------------------------------- 1 | name: Gradle 2 | 3 | on: [push, workflow_dispatch] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v4 12 | - name: Set up JDK 11 13 | uses: actions/setup-java@v4.7.1 14 | with: 15 | distribution: zulu 16 | java-version: 11 17 | - name: Set up Gradle 18 | uses: gradle/actions/setup-gradle@v4 19 | - name: Check with Gradle 20 | run: ./gradlew check 21 | - name: Integration Test with Gradle 22 | run: ./gradlew integrationTest 23 | - uses: codecov/codecov-action@v3 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .gradle 3 | /build/ 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM gradle:8.14.1-jdk11 as builder 2 | COPY . . 3 | RUN gradle shadowJar 4 | 5 | FROM flink:1.20.1-java11 6 | RUN echo "metrics.reporters: prom" >> "$FLINK_HOME/conf/config.yaml"; \ 7 | echo "metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory" >> "$FLINK_HOME/conf/config.yaml" 8 | COPY --from=builder /home/gradle/build/libs/*.jar $FLINK_HOME/usrlib/ 9 | RUN mkdir /state && chown flink:flink /state # workaround for https://github.com/docker/compose/issues/3270 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Actions Status](https://github.com/mbode/flink-prometheus-example/workflows/Gradle/badge.svg)](https://github.com/mbode/flink-prometheus-example/actions) 2 | [![codecov](https://codecov.io/gh/mbode/flink-prometheus-example/branch/master/graph/badge.svg)](https://codecov.io/gh/mbode/flink-prometheus-example) 3 | [![Flink v1.20.1](https://img.shields.io/badge/flink-v1.20.1-blue.svg)](https://github.com/apache/flink/releases/tag/release-1.20.1) 4 | [![Prometheus v2.37.1](https://img.shields.io/badge/prometheus-v2.37.1-blue.svg)](https://github.com/prometheus/prometheus/releases/tag/v2.37.1) 5 | 6 | This repository contains the live demo to my talk _Monitoring Flink with Prometheus_, which I have given at: 7 | * [Flink Forward Berlin 2018](https://berlin-2018.flink-forward.org/conference-program/#monitoring-flink-with-prometheus), _2018-09-04_ (:video_camera: [Video](https://www.youtube.com/watch?v=vesj-ghLimA) :page_facing_up: [Slides](https://www.slideshare.net/MaximilianBode1/monitoring-flink-with-prometheus)) 8 | * [Spark & Hadoop User Group Munich](https://www.meetup.com/de-DE/Hadoop-User-Group-Munich/events/252393503/), _2018-09-26_ 9 | 10 | The blog post [Flink and Prometheus: Cloud-native monitoring of streaming applications](https://flink.apache.org/features/2019/03/11/prometheus-monitoring.html) explains how to run the demo yourself. 11 | 12 | ## Getting Started 13 | 14 | ### Startup 15 | ``` 16 | ./gradlew composeUp 17 | ``` 18 | 19 | ### Web UIs 20 | - [Flink JobManager](http://localhost:8081/#/overview) 21 | - [Prometheus](http://localhost:9090/graph) 22 | - [Grafana](http://localhost:3000) (credentials _admin:flink_) 23 | - Prometheus endpoints 24 | - [Job Manager](http://localhost:9249/metrics) 25 | - [Task Manager 1](http://localhost:9250/metrics) 26 | - [Task Manager 2](http://localhost:9251/metrics) 27 | 28 | ## Built With 29 | 30 | - [Apache Flink](https://flink.apache.org) 31 | - [Prometheus](https://prometheus.io) 32 | - [Grafana](https://grafana.com) 33 | - [docker-compose](https://docs.docker.com/compose/) – provisioning of the test environment 34 | - [Gradle](https://gradle.org) with [kotlin-dsl](https://github.com/gradle/kotlin-dsl) 35 | - [gradle-testsets-plugin](https://github.com/unbroken-dome/gradle-testsets-plugin) 36 | - [shadow](https://github.com/johnrengelman/shadow) 37 | - [spotless](https://github.com/diffplug/spotless/tree/master/plugin-gradle) 38 | - [spotbugs](https://github.com/spotbugs/spotbugs-gradle-plugin) 39 | - [gradle-docker-compose-plugin](https://github.com/avast/gradle-docker-compose-plugin) 40 | - [gradle-versions-plugin](https://github.com/ben-manes/gradle-versions-plugin) 41 | 42 | ## Development 43 | typical tasks: 44 | - verify: `./gradlew check` 45 | - integration tests: `./gradlew integrationTest` 46 | - list outdated dependenices: `./gradlew dependencyUpdates` 47 | - update gradle: `./gradlew wrapper --gradle-version=` (twice) 48 | -------------------------------------------------------------------------------- /build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | java 3 | jacoco 4 | 5 | id("jvm-test-suite") 6 | id("com.github.johnrengelman.shadow") version "8.1.1" 7 | id("com.diffplug.spotless") version "7.0.4" 8 | id("com.github.spotbugs") version "6.1.13" 9 | id("com.avast.gradle.docker-compose") version "0.17.12" 10 | id("com.github.ben-manes.versions") version "0.52.0" 11 | } 12 | 13 | java { 14 | sourceCompatibility = JavaVersion.VERSION_11 15 | targetCompatibility = JavaVersion.VERSION_11 16 | } 17 | 18 | repositories { mavenCentral() } 19 | 20 | val flinkVersion = "1.20.1" 21 | 22 | dependencies { 23 | compileOnly("org.apache.flink:flink-java:$flinkVersion") 24 | compileOnly("org.apache.flink:flink-streaming-java:$flinkVersion") 25 | 26 | testImplementation("org.junit.jupiter:junit-jupiter-api:5.13.0") 27 | testImplementation("org.apache.flink:flink-test-utils:$flinkVersion") 28 | testImplementation("org.assertj:assertj-core:3.27.3") 29 | testImplementation("org.mockito:mockito-junit-jupiter:5.18.0") 30 | 31 | testRuntimeOnly("org.junit.platform:junit-platform-launcher") 32 | testRuntimeOnly("org.slf4j:slf4j-simple:2.0.17") 33 | } 34 | 35 | tasks.named("test") { 36 | useJUnitPlatform() 37 | } 38 | 39 | testing { 40 | suites { 41 | configureEach { 42 | if (this is JvmTestSuite) { 43 | dependencies { 44 | implementation("org.assertj:assertj-core:3.27.3") 45 | } 46 | } 47 | } 48 | 49 | val test by getting(JvmTestSuite::class) { 50 | dependencies { 51 | val junitVersion = "5.13.0" 52 | implementation("org.junit.jupiter:junit-jupiter-api:$junitVersion") 53 | implementation("org.apache.flink:flink-test-utils:$flinkVersion") 54 | 55 | implementation("org.mockito:mockito-junit-jupiter:5.18.0") 56 | 57 | runtimeOnly("org.junit.jupiter:junit-jupiter-engine:$junitVersion") 58 | runtimeOnly("org.slf4j:slf4j-simple:2.0.17") 59 | } 60 | } 61 | register("integrationTest") { 62 | dependencies { 63 | implementation("com.mashape.unirest:unirest-java:1.4.9") 64 | implementation("org.awaitility:awaitility:4.3.0") 65 | implementation("com.github.docker-java:docker-java:3.5.1") 66 | } 67 | } 68 | } 69 | } 70 | 71 | tasks { 72 | "jacocoTestReport"(JacocoReport::class) { 73 | reports { 74 | xml.required.set(true) 75 | html.required.set(true) 76 | } 77 | } 78 | "check" { dependsOn("jacocoTestReport") } 79 | } 80 | 81 | dockerCompose { 82 | isRequiredBy(tasks["integrationTest"]) 83 | setProjectName(null) 84 | } 85 | 86 | spotless { 87 | java { googleJavaFormat() } 88 | kotlinGradle { ktlint() } 89 | } 90 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | services: 3 | job-cluster: 4 | build: . 5 | container_name: job-cluster 6 | ports: 7 | - "8081:8081" 8 | - "9249:9249" 9 | volumes: 10 | - state:/state 11 | command: standalone-job --job-classname com.github.mbode.flink_prometheus_example.PrometheusExampleJob --job-id 00000000000000000000000000000000 12 | environment: 13 | - | 14 | FLINK_PROPERTIES= 15 | jobmanager.rpc.address: job-cluster 16 | state.checkpoints.dir: file:///state 17 | 18 | taskmanager1: 19 | build: . 20 | container_name: taskmanager1 21 | ports: 22 | - "9250:9249" 23 | volumes: 24 | - state:/state 25 | command: taskmanager 26 | environment: 27 | - | 28 | FLINK_PROPERTIES= 29 | jobmanager.rpc.address: job-cluster 30 | state.checkpoints.dir: file:///state 31 | 32 | taskmanager2: 33 | build: . 34 | container_name: taskmanager2 35 | ports: 36 | - "9251:9249" 37 | volumes: 38 | - state:/state 39 | command: taskmanager 40 | environment: 41 | - | 42 | FLINK_PROPERTIES= 43 | jobmanager.rpc.address: job-cluster 44 | state.checkpoints.dir: file:///state 45 | 46 | prometheus: 47 | image: prom/prometheus:v2.37.1 48 | container_name: prometheus 49 | ports: 50 | - "9090:9090" 51 | volumes: 52 | - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml 53 | - ./prometheus/flink.rules.yml:/etc/prometheus/flink.rules.yml 54 | 55 | grafana: 56 | image: grafana/grafana:8.4.0 57 | container_name: grafana 58 | ports: 59 | - "3000:3000" 60 | environment: 61 | - GF_SECURITY_ADMIN_PASSWORD=flink 62 | volumes: 63 | - ./grafana/provisioning/:/etc/grafana/provisioning/ 64 | 65 | volumes: 66 | state: 67 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbode/flink-prometheus-example/057f19c9543c52dccd6651eb935b2049cb5b01e4/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.1-bin.zip 4 | networkTimeout=10000 5 | validateDistributionUrl=true 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | # SPDX-License-Identifier: Apache-2.0 19 | # 20 | 21 | ############################################################################## 22 | # 23 | # Gradle start up script for POSIX generated by Gradle. 24 | # 25 | # Important for running: 26 | # 27 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 28 | # noncompliant, but you have some other compliant shell such as ksh or 29 | # bash, then to run this script, type that shell name before the whole 30 | # command line, like: 31 | # 32 | # ksh Gradle 33 | # 34 | # Busybox and similar reduced shells will NOT work, because this script 35 | # requires all of these POSIX shell features: 36 | # * functions; 37 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 38 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 39 | # * compound commands having a testable exit status, especially «case»; 40 | # * various built-in commands including «command», «set», and «ulimit». 41 | # 42 | # Important for patching: 43 | # 44 | # (2) This script targets any POSIX shell, so it avoids extensions provided 45 | # by Bash, Ksh, etc; in particular arrays are avoided. 46 | # 47 | # The "traditional" practice of packing multiple parameters into a 48 | # space-separated string is a well documented source of bugs and security 49 | # problems, so this is (mostly) avoided, by progressively accumulating 50 | # options in "$@", and eventually passing that to Java. 51 | # 52 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 53 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 54 | # see the in-line comments for details. 55 | # 56 | # There are tweaks for specific operating systems such as AIX, CygWin, 57 | # Darwin, MinGW, and NonStop. 58 | # 59 | # (3) This script is generated from the Groovy template 60 | # https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 61 | # within the Gradle project. 62 | # 63 | # You can find Gradle at https://github.com/gradle/gradle/. 64 | # 65 | ############################################################################## 66 | 67 | # Attempt to set APP_HOME 68 | 69 | # Resolve links: $0 may be a link 70 | app_path=$0 71 | 72 | # Need this for daisy-chained symlinks. 73 | while 74 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 75 | [ -h "$app_path" ] 76 | do 77 | ls=$( ls -ld "$app_path" ) 78 | link=${ls#*' -> '} 79 | case $link in #( 80 | /*) app_path=$link ;; #( 81 | *) app_path=$APP_HOME$link ;; 82 | esac 83 | done 84 | 85 | # This is normally unused 86 | # shellcheck disable=SC2034 87 | APP_BASE_NAME=${0##*/} 88 | # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) 89 | APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit 90 | 91 | # Use the maximum available, or set MAX_FD != -1 to use that value. 92 | MAX_FD=maximum 93 | 94 | warn () { 95 | echo "$*" 96 | } >&2 97 | 98 | die () { 99 | echo 100 | echo "$*" 101 | echo 102 | exit 1 103 | } >&2 104 | 105 | # OS specific support (must be 'true' or 'false'). 106 | cygwin=false 107 | msys=false 108 | darwin=false 109 | nonstop=false 110 | case "$( uname )" in #( 111 | CYGWIN* ) cygwin=true ;; #( 112 | Darwin* ) darwin=true ;; #( 113 | MSYS* | MINGW* ) msys=true ;; #( 114 | NONSTOP* ) nonstop=true ;; 115 | esac 116 | 117 | CLASSPATH="\\\"\\\"" 118 | 119 | 120 | # Determine the Java command to use to start the JVM. 121 | if [ -n "$JAVA_HOME" ] ; then 122 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 123 | # IBM's JDK on AIX uses strange locations for the executables 124 | JAVACMD=$JAVA_HOME/jre/sh/java 125 | else 126 | JAVACMD=$JAVA_HOME/bin/java 127 | fi 128 | if [ ! -x "$JAVACMD" ] ; then 129 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 130 | 131 | Please set the JAVA_HOME variable in your environment to match the 132 | location of your Java installation." 133 | fi 134 | else 135 | JAVACMD=java 136 | if ! command -v java >/dev/null 2>&1 137 | then 138 | die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 139 | 140 | Please set the JAVA_HOME variable in your environment to match the 141 | location of your Java installation." 142 | fi 143 | fi 144 | 145 | # Increase the maximum file descriptors if we can. 146 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 147 | case $MAX_FD in #( 148 | max*) 149 | # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. 150 | # shellcheck disable=SC2039,SC3045 151 | MAX_FD=$( ulimit -H -n ) || 152 | warn "Could not query maximum file descriptor limit" 153 | esac 154 | case $MAX_FD in #( 155 | '' | soft) :;; #( 156 | *) 157 | # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. 158 | # shellcheck disable=SC2039,SC3045 159 | ulimit -n "$MAX_FD" || 160 | warn "Could not set maximum file descriptor limit to $MAX_FD" 161 | esac 162 | fi 163 | 164 | # Collect all arguments for the java command, stacking in reverse order: 165 | # * args from the command line 166 | # * the main class name 167 | # * -classpath 168 | # * -D...appname settings 169 | # * --module-path (only if needed) 170 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 171 | 172 | # For Cygwin or MSYS, switch paths to Windows format before running java 173 | if "$cygwin" || "$msys" ; then 174 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 175 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 176 | 177 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 178 | 179 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 180 | for arg do 181 | if 182 | case $arg in #( 183 | -*) false ;; # don't mess with options #( 184 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 185 | [ -e "$t" ] ;; #( 186 | *) false ;; 187 | esac 188 | then 189 | arg=$( cygpath --path --ignore --mixed "$arg" ) 190 | fi 191 | # Roll the args list around exactly as many times as the number of 192 | # args, so each arg winds up back in the position where it started, but 193 | # possibly modified. 194 | # 195 | # NB: a `for` loop captures its iteration list before it begins, so 196 | # changing the positional parameters here affects neither the number of 197 | # iterations, nor the values presented in `arg`. 198 | shift # remove old arg 199 | set -- "$@" "$arg" # push replacement arg 200 | done 201 | fi 202 | 203 | 204 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 205 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 206 | 207 | # Collect all arguments for the java command: 208 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, 209 | # and any embedded shellness will be escaped. 210 | # * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be 211 | # treated as '${Hostname}' itself on the command line. 212 | 213 | set -- \ 214 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 215 | -classpath "$CLASSPATH" \ 216 | -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ 217 | "$@" 218 | 219 | # Stop when "xargs" is not available. 220 | if ! command -v xargs >/dev/null 2>&1 221 | then 222 | die "xargs is not available" 223 | fi 224 | 225 | # Use "xargs" to parse quoted args. 226 | # 227 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 228 | # 229 | # In Bash we could simply go: 230 | # 231 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 232 | # set -- "${ARGS[@]}" "$@" 233 | # 234 | # but POSIX shell has neither arrays nor command substitution, so instead we 235 | # post-process each arg (as a line of input to sed) to backslash-escape any 236 | # character that might be a shell metacharacter, then use eval to reverse 237 | # that process (while maintaining the separation between arguments), and wrap 238 | # the whole thing up as a single "set" statement. 239 | # 240 | # This will of course break if any of these variables contains a newline or 241 | # an unmatched quote. 242 | # 243 | 244 | eval "set -- $( 245 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 246 | xargs -n1 | 247 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 248 | tr '\n' ' ' 249 | )" '"$@"' 250 | 251 | exec "$JAVACMD" "$@" 252 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | @rem SPDX-License-Identifier: Apache-2.0 17 | @rem 18 | 19 | @if "%DEBUG%"=="" @echo off 20 | @rem ########################################################################## 21 | @rem 22 | @rem Gradle startup script for Windows 23 | @rem 24 | @rem ########################################################################## 25 | 26 | @rem Set local scope for the variables with windows NT shell 27 | if "%OS%"=="Windows_NT" setlocal 28 | 29 | set DIRNAME=%~dp0 30 | if "%DIRNAME%"=="" set DIRNAME=. 31 | @rem This is normally unused 32 | set APP_BASE_NAME=%~n0 33 | set APP_HOME=%DIRNAME% 34 | 35 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 36 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 37 | 38 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 39 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 40 | 41 | @rem Find java.exe 42 | if defined JAVA_HOME goto findJavaFromJavaHome 43 | 44 | set JAVA_EXE=java.exe 45 | %JAVA_EXE% -version >NUL 2>&1 46 | if %ERRORLEVEL% equ 0 goto execute 47 | 48 | echo. 1>&2 49 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 50 | echo. 1>&2 51 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 52 | echo location of your Java installation. 1>&2 53 | 54 | goto fail 55 | 56 | :findJavaFromJavaHome 57 | set JAVA_HOME=%JAVA_HOME:"=% 58 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 59 | 60 | if exist "%JAVA_EXE%" goto execute 61 | 62 | echo. 1>&2 63 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 64 | echo. 1>&2 65 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 66 | echo location of your Java installation. 1>&2 67 | 68 | goto fail 69 | 70 | :execute 71 | @rem Setup the command line 72 | 73 | set CLASSPATH= 74 | 75 | 76 | @rem Execute Gradle 77 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* 78 | 79 | :end 80 | @rem End local scope for the variables with windows NT shell 81 | if %ERRORLEVEL% equ 0 goto mainEnd 82 | 83 | :fail 84 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 85 | rem the _cmd.exe /c_ return code! 86 | set EXIT_CODE=%ERRORLEVEL% 87 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 88 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 89 | exit /b %EXIT_CODE% 90 | 91 | :mainEnd 92 | if "%OS%"=="Windows_NT" endlocal 93 | 94 | :omega 95 | -------------------------------------------------------------------------------- /grafana/provisioning/dashboards/Flink.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "editable": true, 16 | "gnetId": null, 17 | "graphTooltip": 0, 18 | "iteration": 1535902255604, 19 | "links": [], 20 | "panels": [ 21 | { 22 | "collapsed": true, 23 | "gridPos": { 24 | "h": 1, 25 | "w": 24, 26 | "x": 0, 27 | "y": 0 28 | }, 29 | "id": 9, 30 | "panels": [ 31 | { 32 | "aliasColors": {}, 33 | "bars": false, 34 | "dashLength": 10, 35 | "dashes": false, 36 | "datasource": "Prometheus", 37 | "fill": 1, 38 | "gridPos": { 39 | "h": 9, 40 | "w": 24, 41 | "x": 0, 42 | "y": 1 43 | }, 44 | "id": 2, 45 | "legend": { 46 | "alignAsTable": true, 47 | "avg": true, 48 | "current": true, 49 | "max": true, 50 | "min": true, 51 | "rightSide": true, 52 | "show": true, 53 | "total": false, 54 | "values": true 55 | }, 56 | "lines": true, 57 | "linewidth": 1, 58 | "links": [], 59 | "nullPointMode": "null", 60 | "percentage": false, 61 | "pointradius": 5, 62 | "points": false, 63 | "renderer": "flot", 64 | "seriesOverrides": [], 65 | "spaceLength": 10, 66 | "stack": false, 67 | "steppedLine": false, 68 | "targets": [ 69 | { 70 | "expr": "flink_jobmanager_Status_JVM_Memory_Heap_Used", 71 | "format": "time_series", 72 | "intervalFactor": 1, 73 | "legendFormat": "heap memory [{{instance}}]", 74 | "refId": "C" 75 | }, 76 | { 77 | "expr": "flink_jobmanager_Status_JVM_Memory_NonHeap_Used", 78 | "format": "time_series", 79 | "intervalFactor": 1, 80 | "legendFormat": "non-heap memory [{{instance}}]", 81 | "refId": "D" 82 | } 83 | ], 84 | "thresholds": [], 85 | "timeFrom": null, 86 | "timeShift": null, 87 | "title": "Memory used", 88 | "tooltip": { 89 | "shared": true, 90 | "sort": 0, 91 | "value_type": "individual" 92 | }, 93 | "type": "graph", 94 | "xaxis": { 95 | "buckets": null, 96 | "mode": "time", 97 | "name": null, 98 | "show": true, 99 | "values": [] 100 | }, 101 | "yaxes": [ 102 | { 103 | "format": "bytes", 104 | "label": null, 105 | "logBase": 1, 106 | "max": null, 107 | "min": null, 108 | "show": true 109 | }, 110 | { 111 | "format": "short", 112 | "label": null, 113 | "logBase": 1, 114 | "max": null, 115 | "min": null, 116 | "show": true 117 | } 118 | ], 119 | "yaxis": { 120 | "align": false, 121 | "alignLevel": null 122 | } 123 | }, 124 | { 125 | "aliasColors": {}, 126 | "bars": false, 127 | "dashLength": 10, 128 | "dashes": false, 129 | "datasource": "Prometheus", 130 | "fill": 1, 131 | "gridPos": { 132 | "h": 9, 133 | "w": 12, 134 | "x": 0, 135 | "y": 10 136 | }, 137 | "id": 10, 138 | "legend": { 139 | "avg": false, 140 | "current": false, 141 | "max": false, 142 | "min": false, 143 | "show": true, 144 | "total": false, 145 | "values": false 146 | }, 147 | "lines": true, 148 | "linewidth": 1, 149 | "links": [], 150 | "nullPointMode": "null", 151 | "percentage": false, 152 | "pointradius": 5, 153 | "points": false, 154 | "renderer": "flot", 155 | "seriesOverrides": [], 156 | "spaceLength": 10, 157 | "stack": false, 158 | "steppedLine": false, 159 | "targets": [ 160 | { 161 | "expr": "rate(flink_jobmanager_Status_JVM_GarbageCollector_PS_MarkSweep_Time[5m])/1000 + rate(flink_jobmanager_Status_JVM_GarbageCollector_PS_Scavenge_Time[5m])/1000", 162 | "format": "time_series", 163 | "intervalFactor": 1, 164 | "legendFormat": "{{instance}}", 165 | "refId": "B" 166 | } 167 | ], 168 | "thresholds": [], 169 | "timeFrom": null, 170 | "timeShift": null, 171 | "title": "GC Overhead", 172 | "tooltip": { 173 | "shared": true, 174 | "sort": 0, 175 | "value_type": "individual" 176 | }, 177 | "type": "graph", 178 | "xaxis": { 179 | "buckets": null, 180 | "mode": "time", 181 | "name": null, 182 | "show": true, 183 | "values": [] 184 | }, 185 | "yaxes": [ 186 | { 187 | "format": "percentunit", 188 | "label": null, 189 | "logBase": 1, 190 | "max": null, 191 | "min": null, 192 | "show": true 193 | }, 194 | { 195 | "format": "short", 196 | "label": null, 197 | "logBase": 1, 198 | "max": null, 199 | "min": null, 200 | "show": true 201 | } 202 | ], 203 | "yaxis": { 204 | "align": false, 205 | "alignLevel": null 206 | } 207 | } 208 | ], 209 | "title": "JobManager", 210 | "type": "row" 211 | }, 212 | { 213 | "collapsed": true, 214 | "gridPos": { 215 | "h": 1, 216 | "w": 24, 217 | "x": 0, 218 | "y": 1 219 | }, 220 | "id": 6, 221 | "panels": [ 222 | { 223 | "aliasColors": {}, 224 | "bars": false, 225 | "dashLength": 10, 226 | "dashes": false, 227 | "datasource": "Prometheus", 228 | "fill": 1, 229 | "gridPos": { 230 | "h": 9, 231 | "w": 24, 232 | "x": 0, 233 | "y": 2 234 | }, 235 | "id": 7, 236 | "legend": { 237 | "alignAsTable": true, 238 | "avg": true, 239 | "current": true, 240 | "max": true, 241 | "min": true, 242 | "rightSide": true, 243 | "show": true, 244 | "total": false, 245 | "values": true 246 | }, 247 | "lines": true, 248 | "linewidth": 1, 249 | "links": [], 250 | "nullPointMode": "null", 251 | "percentage": false, 252 | "pointradius": 5, 253 | "points": false, 254 | "renderer": "flot", 255 | "seriesOverrides": [], 256 | "spaceLength": 10, 257 | "stack": false, 258 | "steppedLine": false, 259 | "targets": [ 260 | { 261 | "expr": "flink_taskmanager_Status_JVM_Memory_Heap_Used", 262 | "format": "time_series", 263 | "intervalFactor": 1, 264 | "legendFormat": "heap memory [{{instance}}]", 265 | "refId": "A" 266 | }, 267 | { 268 | "expr": "flink_taskmanager_Status_JVM_Memory_NonHeap_Used", 269 | "format": "time_series", 270 | "intervalFactor": 1, 271 | "legendFormat": "non-heap memory [{{instance}}]", 272 | "refId": "B" 273 | } 274 | ], 275 | "thresholds": [], 276 | "timeFrom": null, 277 | "timeShift": null, 278 | "title": "Memory used", 279 | "tooltip": { 280 | "shared": true, 281 | "sort": 0, 282 | "value_type": "individual" 283 | }, 284 | "type": "graph", 285 | "xaxis": { 286 | "buckets": null, 287 | "mode": "time", 288 | "name": null, 289 | "show": true, 290 | "values": [] 291 | }, 292 | "yaxes": [ 293 | { 294 | "format": "bytes", 295 | "label": null, 296 | "logBase": 1, 297 | "max": null, 298 | "min": null, 299 | "show": true 300 | }, 301 | { 302 | "format": "short", 303 | "label": null, 304 | "logBase": 1, 305 | "max": null, 306 | "min": null, 307 | "show": true 308 | } 309 | ], 310 | "yaxis": { 311 | "align": false, 312 | "alignLevel": null 313 | } 314 | }, 315 | { 316 | "aliasColors": {}, 317 | "bars": false, 318 | "dashLength": 10, 319 | "dashes": false, 320 | "datasource": "Prometheus", 321 | "fill": 1, 322 | "gridPos": { 323 | "h": 9, 324 | "w": 12, 325 | "x": 0, 326 | "y": 11 327 | }, 328 | "id": 4, 329 | "legend": { 330 | "avg": false, 331 | "current": false, 332 | "max": false, 333 | "min": false, 334 | "show": true, 335 | "total": false, 336 | "values": false 337 | }, 338 | "lines": true, 339 | "linewidth": 1, 340 | "links": [], 341 | "nullPointMode": "null", 342 | "percentage": false, 343 | "pointradius": 5, 344 | "points": false, 345 | "renderer": "flot", 346 | "seriesOverrides": [], 347 | "spaceLength": 10, 348 | "stack": false, 349 | "steppedLine": false, 350 | "targets": [ 351 | { 352 | "expr": "rate(flink_taskmanager_Status_JVM_GarbageCollector_G1_Old_Generation_Time[5m])/1000 + rate(flink_taskmanager_Status_JVM_GarbageCollector_G1_Young_Generation_Time[5m])/1000", 353 | "format": "time_series", 354 | "intervalFactor": 1, 355 | "legendFormat": "{{instance}}", 356 | "refId": "A" 357 | } 358 | ], 359 | "thresholds": [], 360 | "timeFrom": null, 361 | "timeShift": null, 362 | "title": "GC Overhead", 363 | "tooltip": { 364 | "shared": true, 365 | "sort": 0, 366 | "value_type": "individual" 367 | }, 368 | "type": "graph", 369 | "xaxis": { 370 | "buckets": null, 371 | "mode": "time", 372 | "name": null, 373 | "show": true, 374 | "values": [] 375 | }, 376 | "yaxes": [ 377 | { 378 | "format": "percentunit", 379 | "label": null, 380 | "logBase": 1, 381 | "max": null, 382 | "min": null, 383 | "show": true 384 | }, 385 | { 386 | "format": "short", 387 | "label": null, 388 | "logBase": 1, 389 | "max": null, 390 | "min": null, 391 | "show": true 392 | } 393 | ], 394 | "yaxis": { 395 | "align": false, 396 | "alignLevel": null 397 | } 398 | } 399 | ], 400 | "repeat": null, 401 | "title": "TaskManager", 402 | "type": "row" 403 | }, 404 | { 405 | "collapsed": false, 406 | "gridPos": { 407 | "h": 1, 408 | "w": 24, 409 | "x": 0, 410 | "y": 2 411 | }, 412 | "id": 25, 413 | "panels": [], 414 | "title": "PrometheusExampleJob", 415 | "type": "row" 416 | }, 417 | { 418 | "aliasColors": {}, 419 | "bars": false, 420 | "dashLength": 10, 421 | "dashes": false, 422 | "datasource": null, 423 | "fill": 1, 424 | "gridPos": { 425 | "h": 9, 426 | "w": 12, 427 | "x": 0, 428 | "y": 3 429 | }, 430 | "id": 20, 431 | "legend": { 432 | "avg": false, 433 | "current": false, 434 | "max": false, 435 | "min": false, 436 | "show": true, 437 | "total": false, 438 | "values": false 439 | }, 440 | "lines": true, 441 | "linewidth": 1, 442 | "links": [], 443 | "nullPointMode": "null", 444 | "percentage": false, 445 | "pointradius": 5, 446 | "points": false, 447 | "renderer": "flot", 448 | "seriesOverrides": [], 449 | "spaceLength": 10, 450 | "stack": false, 451 | "steppedLine": false, 452 | "targets": [ 453 | { 454 | "expr": "flink_taskmanager_job_task_operator_events", 455 | "format": "time_series", 456 | "intervalFactor": 1, 457 | "legendFormat": "{{job}}", 458 | "refId": "A" 459 | } 460 | ], 461 | "thresholds": [], 462 | "timeFrom": null, 463 | "timeShift": null, 464 | "title": "Events", 465 | "tooltip": { 466 | "shared": true, 467 | "sort": 0, 468 | "value_type": "individual" 469 | }, 470 | "type": "graph", 471 | "xaxis": { 472 | "buckets": null, 473 | "mode": "time", 474 | "name": null, 475 | "show": true, 476 | "values": [] 477 | }, 478 | "yaxes": [ 479 | { 480 | "format": "short", 481 | "label": null, 482 | "logBase": 1, 483 | "max": null, 484 | "min": null, 485 | "show": true 486 | }, 487 | { 488 | "format": "short", 489 | "label": null, 490 | "logBase": 1, 491 | "max": null, 492 | "min": null, 493 | "show": true 494 | } 495 | ], 496 | "yaxis": { 497 | "align": false, 498 | "alignLevel": null 499 | } 500 | }, 501 | { 502 | "aliasColors": {}, 503 | "bars": false, 504 | "dashLength": 10, 505 | "dashes": false, 506 | "datasource": null, 507 | "fill": 1, 508 | "gridPos": { 509 | "h": 9, 510 | "w": 12, 511 | "x": 12, 512 | "y": 3 513 | }, 514 | "id": 22, 515 | "legend": { 516 | "alignAsTable": false, 517 | "avg": false, 518 | "current": false, 519 | "hideZero": false, 520 | "max": false, 521 | "min": false, 522 | "show": true, 523 | "total": false, 524 | "values": false 525 | }, 526 | "lines": true, 527 | "linewidth": 1, 528 | "links": [], 529 | "nullPointMode": "null", 530 | "percentage": false, 531 | "pointradius": 5, 532 | "points": false, 533 | "renderer": "flot", 534 | "seriesOverrides": [], 535 | "spaceLength": 10, 536 | "stack": false, 537 | "steppedLine": false, 538 | "targets": [ 539 | { 540 | "expr": "flink_taskmanager_job_task_operator_value_histogram", 541 | "format": "time_series", 542 | "intervalFactor": 1, 543 | "legendFormat": "{{quantile}}", 544 | "refId": "A" 545 | } 546 | ], 547 | "thresholds": [], 548 | "timeFrom": null, 549 | "timeShift": null, 550 | "title": "Values", 551 | "tooltip": { 552 | "shared": true, 553 | "sort": 0, 554 | "value_type": "individual" 555 | }, 556 | "type": "graph", 557 | "xaxis": { 558 | "buckets": null, 559 | "mode": "time", 560 | "name": null, 561 | "show": true, 562 | "values": [] 563 | }, 564 | "yaxes": [ 565 | { 566 | "format": "short", 567 | "label": null, 568 | "logBase": 1, 569 | "max": null, 570 | "min": null, 571 | "show": true 572 | }, 573 | { 574 | "format": "short", 575 | "label": null, 576 | "logBase": 1, 577 | "max": null, 578 | "min": null, 579 | "show": true 580 | } 581 | ], 582 | "yaxis": { 583 | "align": false, 584 | "alignLevel": null 585 | } 586 | }, 587 | { 588 | "aliasColors": {}, 589 | "bars": false, 590 | "dashLength": 10, 591 | "dashes": false, 592 | "datasource": null, 593 | "fill": 1, 594 | "gridPos": { 595 | "h": 9, 596 | "w": 12, 597 | "x": 0, 598 | "y": 12 599 | }, 600 | "id": 28, 601 | "legend": { 602 | "avg": false, 603 | "current": false, 604 | "max": false, 605 | "min": false, 606 | "show": true, 607 | "total": false, 608 | "values": false 609 | }, 610 | "lines": true, 611 | "linewidth": 1, 612 | "links": [], 613 | "nullPointMode": "null", 614 | "percentage": false, 615 | "pointradius": 5, 616 | "points": false, 617 | "renderer": "flot", 618 | "seriesOverrides": [], 619 | "spaceLength": 10, 620 | "stack": false, 621 | "steppedLine": false, 622 | "targets": [ 623 | { 624 | "expr": "flink_taskmanager_job_task_numBytesInLocalPerSecond{job_name=\"PrometheusExampleJob\"}", 625 | "format": "time_series", 626 | "interval": "", 627 | "intervalFactor": 1, 628 | "legendFormat": "{{task_name}}", 629 | "refId": "A" 630 | } 631 | ], 632 | "thresholds": [], 633 | "timeFrom": null, 634 | "timeShift": null, 635 | "title": "Bytes in", 636 | "tooltip": { 637 | "shared": true, 638 | "sort": 0, 639 | "value_type": "individual" 640 | }, 641 | "type": "graph", 642 | "xaxis": { 643 | "buckets": null, 644 | "mode": "time", 645 | "name": null, 646 | "show": true, 647 | "values": [] 648 | }, 649 | "yaxes": [ 650 | { 651 | "format": "Bps", 652 | "label": null, 653 | "logBase": 1, 654 | "max": null, 655 | "min": null, 656 | "show": true 657 | }, 658 | { 659 | "format": "short", 660 | "label": null, 661 | "logBase": 1, 662 | "max": null, 663 | "min": null, 664 | "show": true 665 | } 666 | ], 667 | "yaxis": { 668 | "align": false, 669 | "alignLevel": null 670 | } 671 | }, 672 | { 673 | "aliasColors": {}, 674 | "bars": false, 675 | "dashLength": 10, 676 | "dashes": false, 677 | "datasource": null, 678 | "fill": 1, 679 | "gridPos": { 680 | "h": 9, 681 | "w": 12, 682 | "x": 12, 683 | "y": 12 684 | }, 685 | "id": 29, 686 | "legend": { 687 | "avg": false, 688 | "current": false, 689 | "max": false, 690 | "min": false, 691 | "show": true, 692 | "total": false, 693 | "values": false 694 | }, 695 | "lines": true, 696 | "linewidth": 1, 697 | "links": [], 698 | "nullPointMode": "null", 699 | "percentage": false, 700 | "pointradius": 5, 701 | "points": false, 702 | "renderer": "flot", 703 | "seriesOverrides": [], 704 | "spaceLength": 10, 705 | "stack": false, 706 | "steppedLine": false, 707 | "targets": [ 708 | { 709 | "expr": "flink_taskmanager_job_task_numBytesOutPerSecond{job_name=\"PrometheusExampleJob\"}", 710 | "format": "time_series", 711 | "interval": "", 712 | "intervalFactor": 1, 713 | "legendFormat": "{{task_name}}", 714 | "refId": "A" 715 | } 716 | ], 717 | "thresholds": [], 718 | "timeFrom": null, 719 | "timeShift": null, 720 | "title": "Bytes out", 721 | "tooltip": { 722 | "shared": true, 723 | "sort": 0, 724 | "value_type": "individual" 725 | }, 726 | "type": "graph", 727 | "xaxis": { 728 | "buckets": null, 729 | "mode": "time", 730 | "name": null, 731 | "show": true, 732 | "values": [] 733 | }, 734 | "yaxes": [ 735 | { 736 | "format": "Bps", 737 | "label": null, 738 | "logBase": 1, 739 | "max": null, 740 | "min": null, 741 | "show": true 742 | }, 743 | { 744 | "format": "short", 745 | "label": null, 746 | "logBase": 1, 747 | "max": null, 748 | "min": null, 749 | "show": true 750 | } 751 | ], 752 | "yaxis": { 753 | "align": false, 754 | "alignLevel": null 755 | } 756 | }, 757 | { 758 | "cacheTimeout": null, 759 | "colorBackground": false, 760 | "colorValue": false, 761 | "colors": [ 762 | "#d44a3a", 763 | "rgba(237, 129, 40, 0.89)", 764 | "#299c46" 765 | ], 766 | "datasource": "Prometheus", 767 | "decimals": 1, 768 | "format": "ms", 769 | "gauge": { 770 | "maxValue": 100, 771 | "minValue": 0, 772 | "show": false, 773 | "thresholdLabels": false, 774 | "thresholdMarkers": true 775 | }, 776 | "gridPos": { 777 | "h": 3, 778 | "w": 5, 779 | "x": 0, 780 | "y": 21 781 | }, 782 | "id": 14, 783 | "interval": null, 784 | "links": [], 785 | "mappingType": 1, 786 | "mappingTypes": [ 787 | { 788 | "name": "value to text", 789 | "value": 1 790 | }, 791 | { 792 | "name": "range to text", 793 | "value": 2 794 | } 795 | ], 796 | "maxDataPoints": 100, 797 | "nullPointMode": "connected", 798 | "nullText": null, 799 | "postfix": "", 800 | "postfixFontSize": "50%", 801 | "prefix": "", 802 | "prefixFontSize": "50%", 803 | "rangeMaps": [ 804 | { 805 | "from": "null", 806 | "text": "N/A", 807 | "to": "null" 808 | } 809 | ], 810 | "repeat": "job", 811 | "repeatDirection": "h", 812 | "scopedVars": { 813 | "job": { 814 | "selected": false, 815 | "text": "PrometheusExampleJob", 816 | "value": "PrometheusExampleJob" 817 | } 818 | }, 819 | "sparkline": { 820 | "fillColor": "rgba(31, 118, 189, 0.18)", 821 | "full": false, 822 | "lineColor": "rgb(31, 120, 193)", 823 | "show": false 824 | }, 825 | "tableColumn": "", 826 | "targets": [ 827 | { 828 | "expr": "flink_jobmanager_job_uptime{job_name=\"$job\"}", 829 | "format": "time_series", 830 | "intervalFactor": 1, 831 | "refId": "A" 832 | } 833 | ], 834 | "thresholds": ".5,.8", 835 | "title": "Uptime", 836 | "type": "singlestat", 837 | "valueFontSize": "80%", 838 | "valueMaps": [ 839 | { 840 | "op": "=", 841 | "text": "N/A", 842 | "value": "null" 843 | } 844 | ], 845 | "valueName": "current" 846 | } 847 | ], 848 | "refresh": "5s", 849 | "schemaVersion": 16, 850 | "style": "dark", 851 | "tags": [], 852 | "templating": { 853 | "list": [] 854 | }, 855 | "time": { 856 | "from": "now-5m", 857 | "to": "now" 858 | }, 859 | "timepicker": { 860 | "refresh_intervals": [ 861 | "5s", 862 | "10s", 863 | "30s", 864 | "1m", 865 | "5m", 866 | "15m", 867 | "30m", 868 | "1h", 869 | "2h", 870 | "1d" 871 | ], 872 | "time_options": [ 873 | "5m", 874 | "15m", 875 | "1h", 876 | "6h", 877 | "12h", 878 | "24h", 879 | "2d", 880 | "7d", 881 | "30d" 882 | ] 883 | }, 884 | "timezone": "", 885 | "title": "Flink", 886 | "uid": "veLveEOiz", 887 | "version": 1 888 | } -------------------------------------------------------------------------------- /grafana/provisioning/dashboards/dashboards.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: 1 3 | 4 | providers: 5 | - name: 'default' 6 | orgId: 1 7 | folder: '' 8 | type: file 9 | disableDeletion: true 10 | editable: false 11 | updateIntervalSeconds: 3 12 | options: 13 | path: /etc/grafana/provisioning/dashboards 14 | -------------------------------------------------------------------------------- /grafana/provisioning/datasources/prometheus.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: 1 3 | 4 | deleteDatasources: 5 | - name: Prometheus 6 | orgId: 1 7 | 8 | datasources: 9 | - name: Prometheus 10 | type: prometheus 11 | access: proxy 12 | orgId: 1 13 | url: http://prometheus:9090 14 | isDefault: true 15 | version: 1 16 | editable: false 17 | -------------------------------------------------------------------------------- /prometheus/flink.rules.yml: -------------------------------------------------------------------------------- 1 | --- 2 | groups: 3 | - name: flink 4 | rules: 5 | - alert: FlinkTaskManagerMissing 6 | expr: count(flink_taskmanager_Status_JVM_CPU_Time) < 2 7 | for: 1m 8 | annotations: 9 | summary: Fewer Flink TaskManagers than expected are running. 10 | -------------------------------------------------------------------------------- /prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | --- 2 | global: 3 | scrape_interval: 1s 4 | evaluation_interval: 1s 5 | 6 | rule_files: 7 | - flink.rules.yml 8 | 9 | scrape_configs: 10 | - job_name: 'flink' 11 | static_configs: 12 | - targets: ['job-cluster:9249', 'taskmanager1:9249', 'taskmanager2:9249'] 13 | -------------------------------------------------------------------------------- /src/integrationTest/java/com/github/mbode/flink_prometheus_example/FlinkIT.java: -------------------------------------------------------------------------------- 1 | package com.github.mbode.flink_prometheus_example; 2 | 3 | import static org.assertj.core.api.Assertions.assertThat; 4 | import static org.awaitility.Awaitility.await; 5 | 6 | import com.github.dockerjava.core.DockerClientBuilder; 7 | import com.mashape.unirest.http.Unirest; 8 | import com.mashape.unirest.http.exceptions.UnirestException; 9 | import java.util.concurrent.Callable; 10 | import java.util.concurrent.TimeUnit; 11 | import org.json.JSONException; 12 | import org.json.JSONObject; 13 | import org.junit.jupiter.api.Test; 14 | 15 | class FlinkIT { 16 | private static final String FLINK_URL = 17 | "http://" 18 | + System.getProperty("job-cluster.host") 19 | + ":" 20 | + Integer.getInteger("job-cluster.tcp.8081") 21 | + "/"; 22 | private static final String JOB_ID = "00000000000000000000000000000000"; 23 | 24 | @Test 25 | void exampleJobIsRunning() { 26 | await().atMost(1, TimeUnit.MINUTES).ignoreException(JSONException.class).until(jobIsRunning()); 27 | } 28 | 29 | @Test 30 | void jobCanBeRestartedFromCheckpoint() throws UnirestException { 31 | await() 32 | .atMost(1, TimeUnit.MINUTES) 33 | .ignoreException(JSONException.class) 34 | .untilAsserted(() -> assertThat(getActiveTaskManager()).doesNotContain("unassigned")); 35 | 36 | final String firstActiveTaskManager = getActiveTaskManager(); 37 | 38 | DockerClientBuilder.getInstance().build().killContainerCmd(firstActiveTaskManager).exec(); 39 | 40 | await() 41 | .atMost(1, TimeUnit.MINUTES) 42 | .untilAsserted( 43 | () -> assertThat(getActiveTaskManager()).isNotEqualTo(firstActiveTaskManager)); 44 | } 45 | 46 | private static String getActiveTaskManager() throws UnirestException { 47 | final String vertexId = 48 | Unirest.get(FLINK_URL + "jobs/" + JOB_ID) 49 | .asJson() 50 | .getBody() 51 | .getObject() 52 | .getJSONArray("vertices") 53 | .getJSONObject(0) 54 | .getString("id"); 55 | return Unirest.get(FLINK_URL + "jobs/" + JOB_ID + "/vertices/" + vertexId) 56 | .asJson() 57 | .getBody() 58 | .getObject() 59 | .getJSONArray("subtasks") 60 | .getJSONObject(0) 61 | .getString("host") 62 | .split(":")[0]; 63 | } 64 | 65 | private Callable jobIsRunning() { 66 | return () -> { 67 | final JSONObject responseJson = 68 | Unirest.get(FLINK_URL + "jobs/" + JOB_ID).asJson().getBody().getObject(); 69 | return "PrometheusExampleJob".equals(responseJson.getString("name")) 70 | && "RUNNING".equals(responseJson.getString("state")); 71 | }; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/integrationTest/java/com/github/mbode/flink_prometheus_example/GrafanaIT.java: -------------------------------------------------------------------------------- 1 | package com.github.mbode.flink_prometheus_example; 2 | 3 | import static org.awaitility.Awaitility.await; 4 | 5 | import com.mashape.unirest.http.Unirest; 6 | import org.junit.jupiter.api.Test; 7 | 8 | class GrafanaIT { 9 | private static final String GRAFANA_URL = 10 | "http://" 11 | + System.getProperty("grafana.host") 12 | + ":" 13 | + Integer.getInteger("grafana.tcp.3000") 14 | + "/"; 15 | 16 | @Test 17 | void flinkDashboardHasBeenImported() { 18 | await() 19 | .until( 20 | () -> { 21 | final String responseBody = 22 | Unirest.get(GRAFANA_URL + "api/dashboards/uid/veLveEOiz") 23 | .basicAuth("admin", "flink") 24 | .asString() 25 | .getBody(); 26 | return responseBody.contains("Flink"); 27 | }); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/integrationTest/java/com/github/mbode/flink_prometheus_example/PrometheusIT.java: -------------------------------------------------------------------------------- 1 | package com.github.mbode.flink_prometheus_example; 2 | 3 | import static org.awaitility.Awaitility.await; 4 | 5 | import com.mashape.unirest.http.Unirest; 6 | import com.mashape.unirest.http.exceptions.UnirestException; 7 | import java.util.concurrent.TimeUnit; 8 | import org.json.JSONArray; 9 | import org.junit.jupiter.api.Test; 10 | 11 | class PrometheusIT { 12 | private static final String PROMETHEUS_URL = 13 | "http://" 14 | + System.getProperty("prometheus.host") 15 | + ":" 16 | + Integer.getInteger("prometheus.tcp.9090") 17 | + "/"; 18 | 19 | @Test 20 | void genericFlinkMetricsAreAvailable() { 21 | await().until(() -> dataIsAvailableInPrometheusFor("flink_jobmanager_job_uptime")); 22 | } 23 | 24 | @Test 25 | void exampleJobMetricsAreAvailable() { 26 | await() 27 | .atMost(1, TimeUnit.MINUTES) 28 | .until( 29 | () -> 30 | dataIsAvailableInPrometheusFor("flink_taskmanager_job_task_operator_events") 31 | && dataIsAvailableInPrometheusFor( 32 | "flink_taskmanager_job_task_operator_value_histogram")); 33 | } 34 | 35 | private static boolean dataIsAvailableInPrometheusFor(String metricsName) 36 | throws UnirestException { 37 | final JSONArray resultArray = 38 | Unirest.get(PROMETHEUS_URL + "api/v1/query?query=" + metricsName) 39 | .asJson() 40 | .getBody() 41 | .getObject() 42 | .getJSONObject("data") 43 | .getJSONArray("result"); 44 | return resultArray.length() > 0; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/github/mbode/flink_prometheus_example/FlinkMetricsExposingMapFunction.java: -------------------------------------------------------------------------------- 1 | package com.github.mbode.flink_prometheus_example; 2 | 3 | import org.apache.flink.api.common.functions.RichMapFunction; 4 | import org.apache.flink.configuration.Configuration; 5 | import org.apache.flink.metrics.Counter; 6 | import org.apache.flink.metrics.Histogram; 7 | import org.apache.flink.runtime.metrics.DescriptiveStatisticsHistogram; 8 | 9 | class FlinkMetricsExposingMapFunction extends RichMapFunction { 10 | private static final long serialVersionUID = 1L; 11 | 12 | private transient Counter eventCounter; 13 | private transient Histogram valueHistogram; 14 | 15 | @Override 16 | public void open(Configuration parameters) { 17 | eventCounter = getRuntimeContext().getMetricGroup().counter("events"); 18 | valueHistogram = 19 | getRuntimeContext() 20 | .getMetricGroup() 21 | .histogram("value_histogram", new DescriptiveStatisticsHistogram(10_000)); 22 | } 23 | 24 | @Override 25 | public Integer map(Integer value) { 26 | eventCounter.inc(); 27 | valueHistogram.update(value); 28 | return value; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/github/mbode/flink_prometheus_example/PrometheusExampleJob.java: -------------------------------------------------------------------------------- 1 | package com.github.mbode.flink_prometheus_example; 2 | 3 | import org.apache.flink.api.java.utils.ParameterTool; 4 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 5 | import org.apache.flink.streaming.api.functions.sink.DiscardingSink; 6 | 7 | public class PrometheusExampleJob { 8 | private final ParameterTool parameters; 9 | 10 | public static void main(String[] args) throws Exception { 11 | new PrometheusExampleJob(ParameterTool.fromArgs(args)).run(); 12 | } 13 | 14 | private PrometheusExampleJob(ParameterTool parameters) { 15 | this.parameters = parameters; 16 | } 17 | 18 | private void run() throws Exception { 19 | final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 20 | env.enableCheckpointing(500); 21 | env.disableOperatorChaining(); 22 | 23 | env.addSource(new RandomSourceFunction(parameters.getInt("elements", Integer.MAX_VALUE))) 24 | .name(RandomSourceFunction.class.getSimpleName()) 25 | .map(new FlinkMetricsExposingMapFunction()) 26 | .name(FlinkMetricsExposingMapFunction.class.getSimpleName()) 27 | .addSink(new DiscardingSink<>()) 28 | .name(DiscardingSink.class.getSimpleName()); 29 | 30 | env.execute(PrometheusExampleJob.class.getSimpleName()); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/github/mbode/flink_prometheus_example/RandomSourceFunction.java: -------------------------------------------------------------------------------- 1 | package com.github.mbode.flink_prometheus_example; 2 | 3 | import java.util.concurrent.ThreadLocalRandom; 4 | import org.apache.flink.streaming.api.functions.source.SourceFunction; 5 | 6 | class RandomSourceFunction implements SourceFunction { 7 | private int count = 0; 8 | private volatile boolean isRunning = true; 9 | private int elements; 10 | 11 | RandomSourceFunction(int elements) { 12 | this.elements = elements; 13 | } 14 | 15 | public void run(SourceContext ctx) throws InterruptedException { 16 | while (isRunning && count < elements) { 17 | Thread.sleep(1); 18 | ctx.collect(ThreadLocalRandom.current().nextInt(10_000)); 19 | count++; 20 | } 21 | } 22 | 23 | public void cancel() { 24 | isRunning = false; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/test/java/com/github/mbode/flink_prometheus_example/CollectSink.java: -------------------------------------------------------------------------------- 1 | package com.github.mbode.flink_prometheus_example; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import org.apache.flink.streaming.api.functions.sink.SinkFunction; 6 | 7 | class CollectSink implements SinkFunction { 8 | static final List values = new ArrayList<>(); 9 | 10 | @Override 11 | public void invoke(Integer value, Context context) { 12 | values.add(value); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/test/java/com/github/mbode/flink_prometheus_example/FlinkMetricsExposingMapFunctionTest.java: -------------------------------------------------------------------------------- 1 | package com.github.mbode.flink_prometheus_example; 2 | 3 | import static org.assertj.core.api.Assertions.assertThat; 4 | 5 | import org.apache.flink.metrics.Counter; 6 | import org.apache.flink.metrics.Histogram; 7 | import org.junit.jupiter.api.Test; 8 | import org.junit.jupiter.api.extension.ExtendWith; 9 | import org.mockito.InjectMocks; 10 | import org.mockito.Mock; 11 | import org.mockito.Mockito; 12 | import org.mockito.junit.jupiter.MockitoExtension; 13 | 14 | @ExtendWith(MockitoExtension.class) 15 | class FlinkMetricsExposingMapFunctionTest { 16 | private static final Integer TEST_VALUE = 42; 17 | 18 | @Mock private Counter eventCounter; 19 | 20 | @Mock private Histogram valueHistogram; 21 | 22 | @InjectMocks 23 | private final FlinkMetricsExposingMapFunction flinkMetricsExposingMapFunction = 24 | new FlinkMetricsExposingMapFunction(); 25 | 26 | @Test 27 | void mapActsAsIdentity() { 28 | assertThat(flinkMetricsExposingMapFunction.map(TEST_VALUE)).isEqualTo(TEST_VALUE); 29 | } 30 | 31 | @Test 32 | void eventsAreCounted() { 33 | flinkMetricsExposingMapFunction.map(TEST_VALUE); 34 | Mockito.verify(eventCounter).inc(); 35 | } 36 | 37 | @Test 38 | void valueIsReportedToHistogram() { 39 | flinkMetricsExposingMapFunction.map(TEST_VALUE); 40 | Mockito.verify(valueHistogram).update(TEST_VALUE); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/test/java/com/github/mbode/flink_prometheus_example/PrometheusExampleJobTest.java: -------------------------------------------------------------------------------- 1 | package com.github.mbode.flink_prometheus_example; 2 | 3 | import org.apache.flink.test.util.AbstractTestBase; 4 | import org.junit.jupiter.api.Test; 5 | 6 | class PrometheusExampleJobTest extends AbstractTestBase { 7 | @Test 8 | void jobRuns() throws Exception { 9 | PrometheusExampleJob.main(new String[] {"--elements", "7"}); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/test/java/com/github/mbode/flink_prometheus_example/RandomSourceFunctionTest.java: -------------------------------------------------------------------------------- 1 | package com.github.mbode.flink_prometheus_example; 2 | 3 | import static org.assertj.core.api.Assertions.assertThat; 4 | 5 | import java.time.Duration; 6 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 7 | import org.apache.flink.test.util.AbstractTestBase; 8 | import org.junit.jupiter.api.Assertions; 9 | import org.junit.jupiter.api.BeforeEach; 10 | import org.junit.jupiter.api.Test; 11 | 12 | class RandomSourceFunctionTest extends AbstractTestBase { 13 | private StreamExecutionEnvironment env; 14 | 15 | @BeforeEach 16 | void prepareExecutionEnvironmentAndTestSink() { 17 | env = StreamExecutionEnvironment.getExecutionEnvironment(); 18 | env.setParallelism(1); 19 | 20 | CollectSink.values.clear(); 21 | } 22 | 23 | @Test 24 | void expectedNumberOfElementsIsEmitted() throws Exception { 25 | env.addSource(new RandomSourceFunction(42)).addSink(new CollectSink()); 26 | env.execute(); 27 | assertThat(CollectSink.values).hasSize(42); 28 | } 29 | 30 | @Test 31 | void canCancel() { 32 | final RandomSourceFunction function = new RandomSourceFunction(Integer.MAX_VALUE); 33 | function.cancel(); 34 | 35 | env.addSource(function).addSink(new CollectSink()); 36 | Assertions.assertTimeout(Duration.ofSeconds(10), () -> env.execute()); 37 | 38 | assertThat(CollectSink.values).isEmpty(); 39 | } 40 | } 41 | --------------------------------------------------------------------------------