├── .gitignore ├── README.md ├── build.gradle ├── conduktor-platform ├── docker-compose.yml └── platform-config.yml ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── images └── kafka-beginners.png ├── kafka-basics ├── build.gradle └── src │ └── main │ └── java │ └── io │ └── conduktor │ └── demos │ └── kafka │ ├── ConsumerDemo.java │ ├── ConsumerDemoCooperative.java │ ├── ConsumerDemoWithShutdown.java │ ├── ProducerDemo.java │ ├── ProducerDemoKeys.java │ ├── ProducerDemoWithCallback.java │ └── advanced │ ├── ConsumerDemoAssignSeek.java │ ├── ConsumerDemoRebalanceListener.java │ ├── ConsumerDemoThreads.java │ └── ConsumerRebalanceListenerImpl.java ├── kafka-consumer-opensearch ├── build.gradle ├── docker-compose.yml └── src │ └── main │ └── java │ └── io │ └── conduktor │ └── demos │ └── kafka │ └── opensearch │ └── OpenSearchConsumer.java ├── kafka-producer-wikimedia ├── build.gradle └── src │ └── main │ └── java │ └── io │ └── conduktor │ └── demos │ └── kafka │ └── wikimedia │ ├── WikimediaChangeHandler.java │ └── WikimediaChangesProducer.java ├── kafka-streams-wikimedia ├── build.gradle └── src │ └── main │ └── java │ └── io │ └── conduktor │ └── demos │ └── kafka │ └── streams │ └── wikimedia │ ├── WikimediaStreamsApp.java │ └── processor │ ├── BotCountStreamBuilder.java │ ├── EventCountTimeseriesBuilder.java │ └── WebsiteCountStreamBuilder.java └── settings.gradle /.gitignore: -------------------------------------------------------------------------------- 1 | # From https://github.com/github/gitignore/blob/master/Gradle.gitignore 2 | .gradle 3 | **/build/ 4 | 5 | # Ignore Gradle GUI config 6 | gradle-app.setting 7 | 8 | 9 | # Cache of project 10 | .gradletasknamecache 11 | 12 | # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 13 | # gradle/wrapper/gradle-wrapper.properties 14 | 15 | 16 | 17 | # From https://github.com/github/gitignore/blob/master/Java.gitignore 18 | *.class 19 | 20 | # Mobile Tools for Java (J2ME) 21 | .mtj.tmp/ 22 | 23 | # Package Files # 24 | *.jar 25 | *.war 26 | *.ear 27 | 28 | # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) 29 | !gradle-wrapper.jar 30 | 31 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 32 | hs_err_pid* 33 | 34 | 35 | # From https://github.com/github/gitignore/blob/master/Global/JetBrains.gitignore 36 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 37 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 38 | 39 | # User-specific stuff: 40 | .idea/ 41 | 42 | # Sensitive or high-churn files: 43 | .idea/dataSources.ids 44 | .idea/dataSources.xml 45 | .idea/dataSources.local.xml 46 | .idea/sqlDataSources.xml 47 | .idea/dynamic.xml 48 | .idea/uiDesigner.xml 49 | 50 | # Gradle: 51 | .idea/gradle.xml 52 | .idea/libraries 53 | 54 | # Mongo Explorer plugin: 55 | .idea/mongoSettings.xml 56 | 57 | ## File-based project format: 58 | *.iws 59 | 60 | ## Plugin-specific files: 61 | 62 | # IntelliJ 63 | out/ 64 | 65 | # mpeltonen/sbt-idea plugin 66 | .idea_modules/ 67 | 68 | # JIRA plugin 69 | atlassian-ide-plugin.xml 70 | 71 | # Crashlytics plugin (for Android Studio and IntelliJ) 72 | com_crashlytics_export_strings.xml 73 | crashlytics.properties 74 | crashlytics-build.properties 75 | fabric.properties 76 | 77 | 78 | *.DS_Store 79 | .AppleDouble 80 | .LSOverride 81 | 82 | # Icon must end with two \r 83 | Icon 84 | 85 | 86 | # Thumbnails 87 | ._* 88 | 89 | # Files that might appear in the root of a volume 90 | .DocumentRevisions-V100 91 | .fseventsd 92 | .Spotlight-V100 93 | .TemporaryItems 94 | .Trashes 95 | .VolumeIcon.icns 96 | .com.apple.timemachine.donotpresent 97 | 98 | # Directories potentially created on remote AFP share 99 | .AppleDB 100 | .AppleDesktop 101 | Network Trash Folder 102 | Temporary Items 103 | .apdisk -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kafka for Beginners 2 | 3 | This is a companion repository for the [Kafka for Beginners course](https://links.datacumulus.com/apache-kafka-coupon) by Conduktor 4 | 5 | ![Conduktor](images/kafka-beginners.png) 6 | 7 | ## Content 8 | - Basics of Kafka Java Programming 9 | - Wikimedia Producer 10 | - OpenSearch Consumer 11 | - Kafka Streams Sample Application 12 | 13 | # By Conduktor 14 | 15 | [Conduktor](https://www.conduktor.io) is about making Kafka accessible to everyone. Check out our free Kafka learning website [Kafkademy](https://kafkademy.com/) and our Apache Kafka Desktop Client [Conduktor DevTools](https://conduktor.io/download) 16 | 17 | ![Conduktor](https://www.conduktor.io/images/logo.svg) -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'io.conduktor.demos' 6 | version '1.0-SNAPSHOT' 7 | 8 | repositories { 9 | mavenCentral() 10 | } 11 | 12 | dependencies { 13 | testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.2' 14 | testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.2' 15 | } 16 | 17 | test { 18 | useJUnitPlatform() 19 | } -------------------------------------------------------------------------------- /conduktor-platform/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | zookeeper: 3 | image: confluentinc/cp-zookeeper:7.8.0 4 | hostname: zookeeper 5 | container_name: zookeeper 6 | ports: 7 | - "2181:2181" 8 | environment: 9 | ZOOKEEPER_CLIENT_PORT: 2181 10 | ZOOKEEPER_SERVER_ID: 1 11 | ZOOKEEPER_SERVERS: zookeeper:2888:3888 12 | kafka: 13 | image: confluentinc/cp-kafka:7.8.0 14 | hostname: kafka 15 | container_name: kafka 16 | ports: 17 | - "9092:9092" 18 | - "29092:29092" 19 | - "9999:9999" 20 | environment: 21 | KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 22 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT 23 | KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL 24 | KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" 25 | KAFKA_BROKER_ID: 1 26 | KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" 27 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 28 | KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 29 | KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 30 | KAFKA_JMX_PORT: 9999 31 | KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1} 32 | KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer 33 | KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" 34 | depends_on: 35 | - zookeeper 36 | schema-registry: 37 | image: confluentinc/cp-schema-registry:7.8.0 38 | hostname: schema-registry 39 | container_name: schema-registry 40 | ports: 41 | - "8081:8081" 42 | environment: 43 | SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:19092 44 | SCHEMA_REGISTRY_HOST_NAME: schema-registry 45 | SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 46 | depends_on: 47 | - zookeeper 48 | - kafka 49 | postgresql: 50 | image: postgres:14 51 | hostname: postgresql 52 | container_name: postgresql 53 | environment: 54 | POSTGRES_DB: "conduktor-console" 55 | POSTGRES_USER: "conduktor" 56 | POSTGRES_PASSWORD: "change_me" 57 | POSTGRES_HOST_AUTH_METHOD: "scram-sha-256" 58 | conduktor-console: 59 | image: conduktor/conduktor-console:1.34.3 60 | hostname: conduktor-console 61 | container_name: conduktor-console 62 | depends_on: 63 | - postgresql 64 | ports: 65 | - "8080:8080" 66 | volumes: 67 | - type: bind 68 | source: "./platform-config.yml" 69 | target: /opt/conduktor/platform-config.yaml 70 | read_only: true 71 | environment: 72 | CDK_IN_CONF_FILE: /opt/conduktor/platform-config.yaml 73 | conduktor-monitoring: 74 | hostname: conduktor-monitoring 75 | container_name: conduktor-monitoring 76 | image: conduktor/conduktor-console-cortex:1.34.3 77 | environment: 78 | CDK_CONSOLE-URL: "http://conduktor-console:8080" 79 | -------------------------------------------------------------------------------- /conduktor-platform/platform-config.yml: -------------------------------------------------------------------------------- 1 | organization: # Name of your organization (mandatory) 2 | name: "My Organization" 3 | 4 | database: # Database credentials, must match what you have in the docker-compose (mandatory) 5 | host: "postgresql" 6 | port: 5432 7 | name: "conduktor-console" 8 | username: "conduktor" 9 | password: "change_me" 10 | 11 | admin: # Define the local admin for the initial setup (mandatory) 12 | email: admin@conduktor.io 13 | password: adminP4ss! 14 | 15 | auth: # Define a local user (optional) 16 | local-users: 17 | - email: user@conduktor.io 18 | password: userP4ss! 19 | 20 | monitoring: # Monitoring settings (optional) 21 | cortex-url: http://conduktor-monitoring:9009/ 22 | alert-manager-url: http://conduktor-monitoring:9010/ 23 | callback-url: http://conduktor-console:8080/monitoring/api/ 24 | notifications-callback-url: http://localhost:8080 25 | 26 | clusters: # Cluster and schema registry configuration (optional) 27 | - id: my-local-kafka-cluster 28 | name: My Local Kafka Cluster 29 | bootstrapServers: "kafka:19092" 30 | schemaRegistry: 31 | url: "http://schema-registry:8081" 32 | # kafkaConnects: 33 | # - url: http://kafka-connect:8083 34 | # name: full stack kafka connect 35 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/kafka-beginners-course/662d8e3efd9fb3f2986414dbfb509a468ecb1161/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # 4 | # Copyright 2015 the original author or authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | ## 21 | ## Gradle start up script for UN*X 22 | ## 23 | ############################################################################## 24 | 25 | # Attempt to set APP_HOME 26 | # Resolve links: $0 may be a link 27 | PRG="$0" 28 | # Need this for relative symlinks. 29 | while [ -h "$PRG" ] ; do 30 | ls=`ls -ld "$PRG"` 31 | link=`expr "$ls" : '.*-> \(.*\)$'` 32 | if expr "$link" : '/.*' > /dev/null; then 33 | PRG="$link" 34 | else 35 | PRG=`dirname "$PRG"`"/$link" 36 | fi 37 | done 38 | SAVED="`pwd`" 39 | cd "`dirname \"$PRG\"`/" >/dev/null 40 | APP_HOME="`pwd -P`" 41 | cd "$SAVED" >/dev/null 42 | 43 | APP_NAME="Gradle" 44 | APP_BASE_NAME=`basename "$0"` 45 | 46 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 47 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 48 | 49 | # Use the maximum available, or set MAX_FD != -1 to use that value. 50 | MAX_FD="maximum" 51 | 52 | warn () { 53 | echo "$*" 54 | } 55 | 56 | die () { 57 | echo 58 | echo "$*" 59 | echo 60 | exit 1 61 | } 62 | 63 | # OS specific support (must be 'true' or 'false'). 64 | cygwin=false 65 | msys=false 66 | darwin=false 67 | nonstop=false 68 | case "`uname`" in 69 | CYGWIN* ) 70 | cygwin=true 71 | ;; 72 | Darwin* ) 73 | darwin=true 74 | ;; 75 | MSYS* | MINGW* ) 76 | msys=true 77 | ;; 78 | NONSTOP* ) 79 | nonstop=true 80 | ;; 81 | esac 82 | 83 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 84 | 85 | 86 | # Determine the Java command to use to start the JVM. 87 | if [ -n "$JAVA_HOME" ] ; then 88 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 89 | # IBM's JDK on AIX uses strange locations for the executables 90 | JAVACMD="$JAVA_HOME/jre/sh/java" 91 | else 92 | JAVACMD="$JAVA_HOME/bin/java" 93 | fi 94 | if [ ! -x "$JAVACMD" ] ; then 95 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 96 | 97 | Please set the JAVA_HOME variable in your environment to match the 98 | location of your Java installation." 99 | fi 100 | else 101 | JAVACMD="java" 102 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 103 | 104 | Please set the JAVA_HOME variable in your environment to match the 105 | location of your Java installation." 106 | fi 107 | 108 | # Increase the maximum file descriptors if we can. 109 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 110 | MAX_FD_LIMIT=`ulimit -H -n` 111 | if [ $? -eq 0 ] ; then 112 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 113 | MAX_FD="$MAX_FD_LIMIT" 114 | fi 115 | ulimit -n $MAX_FD 116 | if [ $? -ne 0 ] ; then 117 | warn "Could not set maximum file descriptor limit: $MAX_FD" 118 | fi 119 | else 120 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 121 | fi 122 | fi 123 | 124 | # For Darwin, add options to specify how the application appears in the dock 125 | if $darwin; then 126 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 127 | fi 128 | 129 | # For Cygwin or MSYS, switch paths to Windows format before running java 130 | if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then 131 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 132 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 133 | 134 | JAVACMD=`cygpath --unix "$JAVACMD"` 135 | 136 | # We build the pattern for arguments to be converted via cygpath 137 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 138 | SEP="" 139 | for dir in $ROOTDIRSRAW ; do 140 | ROOTDIRS="$ROOTDIRS$SEP$dir" 141 | SEP="|" 142 | done 143 | OURCYGPATTERN="(^($ROOTDIRS))" 144 | # Add a user-defined pattern to the cygpath arguments 145 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 146 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 147 | fi 148 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 149 | i=0 150 | for arg in "$@" ; do 151 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 152 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 153 | 154 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 155 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 156 | else 157 | eval `echo args$i`="\"$arg\"" 158 | fi 159 | i=`expr $i + 1` 160 | done 161 | case $i in 162 | 0) set -- ;; 163 | 1) set -- "$args0" ;; 164 | 2) set -- "$args0" "$args1" ;; 165 | 3) set -- "$args0" "$args1" "$args2" ;; 166 | 4) set -- "$args0" "$args1" "$args2" "$args3" ;; 167 | 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 168 | 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 169 | 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 170 | 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 171 | 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 172 | esac 173 | fi 174 | 175 | # Escape application args 176 | save () { 177 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 178 | echo " " 179 | } 180 | APP_ARGS=`save "$@"` 181 | 182 | # Collect all arguments for the java command, following the shell quoting and substitution rules 183 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 184 | 185 | exec "$JAVACMD" "$@" 186 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /images/kafka-beginners.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/kafka-beginners-course/662d8e3efd9fb3f2986414dbfb509a468ecb1161/images/kafka-beginners.png -------------------------------------------------------------------------------- /kafka-basics/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'io.conduktor.demos' 6 | version '1.0-SNAPSHOT' 7 | 8 | repositories { 9 | mavenCentral() 10 | } 11 | 12 | dependencies { 13 | 14 | // https://mvnrepository.com/artifact/org.apache.kafka/kafka-clients 15 | implementation 'org.apache.kafka:kafka-clients:3.3.1' 16 | 17 | // https://mvnrepository.com/artifact/org.slf4j/slf4j-api 18 | implementation 'org.slf4j:slf4j-api:1.7.36' 19 | 20 | // https://mvnrepository.com/artifact/org.slf4j/slf4j-simple 21 | implementation 'org.slf4j:slf4j-simple:1.7.36' 22 | 23 | } 24 | 25 | test { 26 | useJUnitPlatform() 27 | } -------------------------------------------------------------------------------- /kafka-basics/src/main/java/io/conduktor/demos/kafka/ConsumerDemo.java: -------------------------------------------------------------------------------- 1 | package io.conduktor.demos.kafka; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRecord; 4 | import org.apache.kafka.clients.consumer.ConsumerRecords; 5 | import org.apache.kafka.clients.consumer.KafkaConsumer; 6 | import org.apache.kafka.common.serialization.StringDeserializer; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.time.Duration; 11 | import java.util.Arrays; 12 | import java.util.Properties; 13 | 14 | public class ConsumerDemo { 15 | 16 | private static final Logger log = LoggerFactory.getLogger(ConsumerDemo.class.getSimpleName()); 17 | 18 | public static void main(String[] args) { 19 | log.info("I am a Kafka Consumer!"); 20 | 21 | String groupId = "my-java-application"; 22 | String topic = "demo_java"; 23 | 24 | // create Producer Properties 25 | Properties properties = new Properties(); 26 | 27 | // connect to Localhost 28 | // properties.setProperty("bootstrap.servers", "127.0.0.1:9092"); 29 | 30 | // connect to Conduktor Playground 31 | properties.setProperty("bootstrap.servers", "cluster.playground.cdkt.io:9092"); 32 | properties.setProperty("security.protocol", "SASL_SSL"); 33 | properties.setProperty("sasl.jaas.config", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"your-username\" password=\"your-password\";"); 34 | properties.setProperty("sasl.mechanism", "PLAIN"); 35 | 36 | // create consumer configs 37 | properties.setProperty("key.deserializer", StringDeserializer.class.getName()); 38 | properties.setProperty("value.deserializer", StringDeserializer.class.getName()); 39 | properties.setProperty("group.id", groupId); 40 | properties.setProperty("auto.offset.reset", "earliest"); 41 | 42 | // create a consumer 43 | KafkaConsumer consumer = new KafkaConsumer<>(properties); 44 | 45 | // subscribe to a topic 46 | consumer.subscribe(Arrays.asList(topic)); 47 | 48 | // poll for data 49 | while (true) { 50 | 51 | log.info("Polling"); 52 | 53 | ConsumerRecords records = 54 | consumer.poll(Duration.ofMillis(1000)); 55 | 56 | for (ConsumerRecord record: records) { 57 | log.info("Key: " + record.key() + ", Value: " + record.value()); 58 | log.info("Partition: " + record.partition() + ", Offset: " + record.offset()); 59 | } 60 | 61 | 62 | } 63 | 64 | 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /kafka-basics/src/main/java/io/conduktor/demos/kafka/ConsumerDemoCooperative.java: -------------------------------------------------------------------------------- 1 | package io.conduktor.demos.kafka; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRecord; 4 | import org.apache.kafka.clients.consumer.ConsumerRecords; 5 | import org.apache.kafka.clients.consumer.CooperativeStickyAssignor; 6 | import org.apache.kafka.clients.consumer.KafkaConsumer; 7 | import org.apache.kafka.common.errors.WakeupException; 8 | import org.apache.kafka.common.serialization.StringDeserializer; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.time.Duration; 13 | import java.util.Arrays; 14 | import java.util.Properties; 15 | 16 | public class ConsumerDemoCooperative { 17 | 18 | private static final Logger log = LoggerFactory.getLogger(ConsumerDemoCooperative.class.getSimpleName()); 19 | 20 | public static void main(String[] args) { 21 | log.info("I am a Kafka Consumer!"); 22 | 23 | String groupId = "my-java-application"; 24 | String topic = "demo_java"; 25 | 26 | // create Producer Properties 27 | Properties properties = new Properties(); 28 | 29 | // connect to Localhost 30 | // properties.setProperty("bootstrap.servers", "127.0.0.1:9092"); 31 | 32 | // connect to Conduktor Playground 33 | properties.setProperty("bootstrap.servers", "cluster.playground.cdkt.io:9092"); 34 | properties.setProperty("security.protocol", "SASL_SSL"); 35 | properties.setProperty("sasl.jaas.config", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"your-username\" password=\"your-password\";"); 36 | properties.setProperty("sasl.mechanism", "PLAIN"); 37 | 38 | // create consumer configs 39 | properties.setProperty("key.deserializer", StringDeserializer.class.getName()); 40 | properties.setProperty("value.deserializer", StringDeserializer.class.getName()); 41 | properties.setProperty("group.id", groupId); 42 | properties.setProperty("auto.offset.reset", "earliest"); 43 | properties.setProperty("partition.assignment.strategy", CooperativeStickyAssignor.class.getName()); 44 | // properties.setProperty("group.instance.id", "...."); // strategy for static assignments 45 | 46 | 47 | // create a consumer 48 | KafkaConsumer consumer = new KafkaConsumer<>(properties); 49 | 50 | // get a reference to the main thread 51 | final Thread mainThread = Thread.currentThread(); 52 | 53 | // adding the shutdown hook 54 | Runtime.getRuntime().addShutdownHook(new Thread() { 55 | public void run() { 56 | log.info("Detected a shutdown, let's exit by calling consumer.wakeup()..."); 57 | consumer.wakeup(); 58 | 59 | // join the main thread to allow the execution of the code in the main thread 60 | try { 61 | mainThread.join(); 62 | } catch (InterruptedException e) { 63 | e.printStackTrace(); 64 | } 65 | } 66 | }); 67 | 68 | 69 | 70 | try { 71 | // subscribe to a topic 72 | consumer.subscribe(Arrays.asList(topic)); 73 | // poll for data 74 | while (true) { 75 | 76 | ConsumerRecords records = 77 | consumer.poll(Duration.ofMillis(1000)); 78 | 79 | for (ConsumerRecord record : records) { 80 | log.info("Key: " + record.key() + ", Value: " + record.value()); 81 | log.info("Partition: " + record.partition() + ", Offset: " + record.offset()); 82 | } 83 | 84 | } 85 | 86 | } catch (WakeupException e) { 87 | log.info("Consumer is starting to shut down"); 88 | } catch (Exception e) { 89 | log.error("Unexpected exception in the consumer", e); 90 | } finally { 91 | consumer.close(); // close the consumer, this will also commit offsets 92 | log.info("The consumer is now gracefully shut down"); 93 | } 94 | 95 | 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /kafka-basics/src/main/java/io/conduktor/demos/kafka/ConsumerDemoWithShutdown.java: -------------------------------------------------------------------------------- 1 | package io.conduktor.demos.kafka; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRecord; 4 | import org.apache.kafka.clients.consumer.ConsumerRecords; 5 | import org.apache.kafka.clients.consumer.KafkaConsumer; 6 | import org.apache.kafka.common.errors.WakeupException; 7 | import org.apache.kafka.common.serialization.StringDeserializer; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.time.Duration; 12 | import java.util.Arrays; 13 | import java.util.Properties; 14 | 15 | public class ConsumerDemoWithShutdown { 16 | 17 | private static final Logger log = LoggerFactory.getLogger(ConsumerDemoWithShutdown.class.getSimpleName()); 18 | 19 | public static void main(String[] args) { 20 | log.info("I am a Kafka Consumer!"); 21 | 22 | String groupId = "my-java-application"; 23 | String topic = "demo_java"; 24 | 25 | // create Producer Properties 26 | Properties properties = new Properties(); 27 | 28 | // connect to Localhost 29 | // properties.setProperty("bootstrap.servers", "127.0.0.1:9092"); 30 | 31 | // connect to Conduktor Playground 32 | properties.setProperty("bootstrap.servers", "cluster.playground.cdkt.io:9092"); 33 | properties.setProperty("security.protocol", "SASL_SSL"); 34 | properties.setProperty("sasl.jaas.config", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"your-username\" password=\"your-password\";"); 35 | properties.setProperty("sasl.mechanism", "PLAIN"); 36 | 37 | // create consumer configs 38 | properties.setProperty("key.deserializer", StringDeserializer.class.getName()); 39 | properties.setProperty("value.deserializer", StringDeserializer.class.getName()); 40 | properties.setProperty("group.id", groupId); 41 | properties.setProperty("auto.offset.reset", "earliest"); 42 | 43 | // create a consumer 44 | KafkaConsumer consumer = new KafkaConsumer<>(properties); 45 | 46 | // get a reference to the main thread 47 | final Thread mainThread = Thread.currentThread(); 48 | 49 | // adding the shutdown hook 50 | Runtime.getRuntime().addShutdownHook(new Thread() { 51 | public void run() { 52 | log.info("Detected a shutdown, let's exit by calling consumer.wakeup()..."); 53 | consumer.wakeup(); 54 | 55 | // join the main thread to allow the execution of the code in the main thread 56 | try { 57 | mainThread.join(); 58 | } catch (InterruptedException e) { 59 | e.printStackTrace(); 60 | } 61 | } 62 | }); 63 | 64 | 65 | 66 | try { 67 | // subscribe to a topic 68 | consumer.subscribe(Arrays.asList(topic)); 69 | // poll for data 70 | while (true) { 71 | 72 | ConsumerRecords records = 73 | consumer.poll(Duration.ofMillis(1000)); 74 | 75 | for (ConsumerRecord record : records) { 76 | log.info("Key: " + record.key() + ", Value: " + record.value()); 77 | log.info("Partition: " + record.partition() + ", Offset: " + record.offset()); 78 | } 79 | 80 | } 81 | 82 | } catch (WakeupException e) { 83 | log.info("Consumer is starting to shut down"); 84 | } catch (Exception e) { 85 | log.error("Unexpected exception in the consumer", e); 86 | } finally { 87 | consumer.close(); // close the consumer, this will also commit offsets 88 | log.info("The consumer is now gracefully shut down"); 89 | } 90 | 91 | 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /kafka-basics/src/main/java/io/conduktor/demos/kafka/ProducerDemo.java: -------------------------------------------------------------------------------- 1 | package io.conduktor.demos.kafka; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.ProducerRecord; 5 | import org.apache.kafka.common.serialization.StringSerializer; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.util.Properties; 10 | 11 | public class ProducerDemo { 12 | 13 | private static final Logger log = LoggerFactory.getLogger(ProducerDemo.class.getSimpleName()); 14 | 15 | public static void main(String[] args) { 16 | log.info("I am a Kafka Producer!"); 17 | 18 | // create Producer Properties 19 | Properties properties = new Properties(); 20 | 21 | // connect to Localhost 22 | // properties.setProperty("bootstrap.servers", "127.0.0.1:9092"); 23 | 24 | // connect to Conduktor Playground 25 | properties.setProperty("bootstrap.servers", "cluster.playground.cdkt.io:9092"); 26 | properties.setProperty("security.protocol", "SASL_SSL"); 27 | properties.setProperty("sasl.jaas.config", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"your-username\" password=\"your-password\";"); 28 | properties.setProperty("sasl.mechanism", "PLAIN"); 29 | 30 | // set producer properties 31 | properties.setProperty("key.serializer", StringSerializer.class.getName()); 32 | properties.setProperty("value.serializer", StringSerializer.class.getName()); 33 | 34 | // create the Producer 35 | KafkaProducer producer = new KafkaProducer<>(properties); 36 | 37 | // create a Producer Record 38 | ProducerRecord producerRecord = 39 | new ProducerRecord<>("demo_java", "hello world"); 40 | 41 | // send data 42 | producer.send(producerRecord); 43 | 44 | // tell the producer to send all data and block until done -- synchronous 45 | producer.flush(); 46 | 47 | // flush and close the producer 48 | producer.close(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /kafka-basics/src/main/java/io/conduktor/demos/kafka/ProducerDemoKeys.java: -------------------------------------------------------------------------------- 1 | package io.conduktor.demos.kafka; 2 | 3 | import org.apache.kafka.clients.producer.Callback; 4 | import org.apache.kafka.clients.producer.KafkaProducer; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.apache.kafka.clients.producer.RecordMetadata; 7 | import org.apache.kafka.common.serialization.StringSerializer; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.util.Properties; 12 | 13 | public class ProducerDemoKeys { 14 | 15 | private static final Logger log = LoggerFactory.getLogger(ProducerDemoKeys.class.getSimpleName()); 16 | 17 | public static void main(String[] args) { 18 | log.info("I am a Kafka Producer!"); 19 | 20 | // create Producer Properties 21 | Properties properties = new Properties(); 22 | 23 | // connect to Localhost 24 | // properties.setProperty("bootstrap.servers", "127.0.0.1:9092"); 25 | 26 | // connect to Conduktor Playground 27 | properties.setProperty("bootstrap.servers", "cluster.playground.cdkt.io:9092"); 28 | properties.setProperty("security.protocol", "SASL_SSL"); 29 | properties.setProperty("sasl.jaas.config", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"your-username\" password=\"your-password\";"); 30 | properties.setProperty("sasl.mechanism", "PLAIN"); 31 | 32 | // set producer properties 33 | properties.setProperty("key.serializer", StringSerializer.class.getName()); 34 | properties.setProperty("value.serializer", StringSerializer.class.getName()); 35 | 36 | // create the Producer 37 | KafkaProducer producer = new KafkaProducer<>(properties); 38 | 39 | 40 | for (int j=0; j<2; j++){ 41 | 42 | for (int i=0; i<10; i++){ 43 | 44 | String topic = "demo_java"; 45 | String key = "id_" + i; 46 | String value = "hello world " + i; 47 | 48 | // create a Producer Record 49 | ProducerRecord producerRecord = 50 | new ProducerRecord<>(topic, key, value); 51 | 52 | // send data 53 | producer.send(producerRecord, new Callback() { 54 | @Override 55 | public void onCompletion(RecordMetadata metadata, Exception e) { 56 | // executes every time a record successfully sent or an exception is thrown 57 | if (e == null) { 58 | // the record was successfully sent 59 | log.info("Key: " + key + " | Partition: " + metadata.partition()); 60 | } else { 61 | log.error("Error while producing", e); 62 | } 63 | } 64 | }); 65 | } 66 | 67 | try { 68 | Thread.sleep(500); 69 | } catch (InterruptedException e) { 70 | e.printStackTrace(); 71 | } 72 | } 73 | 74 | 75 | 76 | // tell the producer to send all data and block until done -- synchronous 77 | producer.flush(); 78 | 79 | // flush and close the producer 80 | producer.close(); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /kafka-basics/src/main/java/io/conduktor/demos/kafka/ProducerDemoWithCallback.java: -------------------------------------------------------------------------------- 1 | package io.conduktor.demos.kafka; 2 | 3 | import org.apache.kafka.clients.producer.Callback; 4 | import org.apache.kafka.clients.producer.KafkaProducer; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.apache.kafka.clients.producer.RecordMetadata; 7 | import org.apache.kafka.common.serialization.StringSerializer; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.util.Properties; 12 | 13 | public class ProducerDemoWithCallback { 14 | 15 | private static final Logger log = LoggerFactory.getLogger(ProducerDemoWithCallback.class.getSimpleName()); 16 | 17 | public static void main(String[] args) { 18 | log.info("I am a Kafka Producer!"); 19 | 20 | // create Producer Properties 21 | Properties properties = new Properties(); 22 | 23 | // connect to Localhost 24 | // properties.setProperty("bootstrap.servers", "127.0.0.1:9092"); 25 | 26 | // connect to Conduktor Playground 27 | properties.setProperty("bootstrap.servers", "cluster.playground.cdkt.io:9092"); 28 | properties.setProperty("security.protocol", "SASL_SSL"); 29 | properties.setProperty("sasl.jaas.config", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"your-username\" password=\"your-password\";"); 30 | properties.setProperty("sasl.mechanism", "PLAIN"); 31 | 32 | // set producer properties 33 | properties.setProperty("key.serializer", StringSerializer.class.getName()); 34 | properties.setProperty("value.serializer", StringSerializer.class.getName()); 35 | 36 | properties.setProperty("batch.size", "400"); 37 | 38 | // properties.setProperty("partitioner.class", RoundRobinPartitioner.class.getName()); 39 | 40 | // create the Producer 41 | KafkaProducer producer = new KafkaProducer<>(properties); 42 | 43 | 44 | for (int j=0; j<10; j++){ 45 | 46 | for (int i=0; i<30; i++){ 47 | 48 | // create a Producer Record 49 | ProducerRecord producerRecord = 50 | new ProducerRecord<>("demo_java", "hello world " + i); 51 | 52 | // send data 53 | producer.send(producerRecord, new Callback() { 54 | @Override 55 | public void onCompletion(RecordMetadata metadata, Exception e) { 56 | // executes every time a record successfully sent or an exception is thrown 57 | if (e == null) { 58 | // the record was successfully sent 59 | log.info("Received new metadata \n" + 60 | "Topic: " + metadata.topic() + "\n" + 61 | "Partition: " + metadata.partition() + "\n" + 62 | "Offset: " + metadata.offset() + "\n" + 63 | "Timestamp: " + metadata.timestamp()); 64 | } else { 65 | log.error("Error while producing", e); 66 | } 67 | } 68 | }); 69 | } 70 | 71 | try { 72 | Thread.sleep(500); 73 | } catch (InterruptedException e) { 74 | e.printStackTrace(); 75 | } 76 | 77 | } 78 | 79 | 80 | // tell the producer to send all data and block until done -- synchronous 81 | producer.flush(); 82 | 83 | // flush and close the producer 84 | producer.close(); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /kafka-basics/src/main/java/io/conduktor/demos/kafka/advanced/ConsumerDemoAssignSeek.java: -------------------------------------------------------------------------------- 1 | package io.conduktor.demos.kafka.advanced; 2 | 3 | import org.apache.kafka.clients.CommonClientConfigs; 4 | import org.apache.kafka.clients.consumer.ConsumerConfig; 5 | import org.apache.kafka.clients.consumer.ConsumerRecord; 6 | import org.apache.kafka.clients.consumer.ConsumerRecords; 7 | import org.apache.kafka.clients.consumer.KafkaConsumer; 8 | import org.apache.kafka.common.TopicPartition; 9 | import org.apache.kafka.common.config.SaslConfigs; 10 | import org.apache.kafka.common.serialization.StringDeserializer; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.time.Duration; 15 | import java.util.Arrays; 16 | import java.util.Properties; 17 | 18 | public class ConsumerDemoAssignSeek { 19 | public static void main(String[] args) { 20 | 21 | Logger log = LoggerFactory.getLogger(ConsumerDemoAssignSeek.class.getName()); 22 | 23 | String bootstrapServers = "127.0.0.1:9092"; 24 | String topic = "demo_java"; 25 | 26 | // create consumer configs 27 | Properties properties = new Properties(); 28 | // properties.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:9092"); 29 | properties.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "cluster.playground.cdkt.io:9092"); 30 | properties.setProperty(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_SSL"); 31 | properties.setProperty(SaslConfigs.SASL_JAAS_CONFIG, "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"6hiWlwa3a3RibZIBq2lNEP\" password=\"8a575166-4c7d-4900-ad2c-f2b4a510f0ce\";"); 32 | properties.setProperty(SaslConfigs.SASL_MECHANISM, "PLAIN"); 33 | properties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 34 | properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 35 | properties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); 36 | 37 | // create consumer 38 | KafkaConsumer consumer = new KafkaConsumer(properties); 39 | 40 | // assign and seek are mostly used to replay data or fetch a specific message 41 | 42 | // assign 43 | TopicPartition partitionToReadFrom = new TopicPartition(topic, 0); 44 | long offsetToReadFrom = 7L; 45 | consumer.assign(Arrays.asList(partitionToReadFrom)); 46 | 47 | // seek 48 | consumer.seek(partitionToReadFrom, offsetToReadFrom); 49 | 50 | int numberOfMessagesToRead = 5; 51 | boolean keepOnReading = true; 52 | int numberOfMessagesReadSoFar = 0; 53 | 54 | // poll for new data 55 | while(keepOnReading){ 56 | ConsumerRecords records = 57 | consumer.poll(Duration.ofMillis(100)); 58 | 59 | for (ConsumerRecord record : records){ 60 | numberOfMessagesReadSoFar += 1; 61 | log.info("Key: " + record.key() + ", Value: " + record.value()); 62 | log.info("Partition: " + record.partition() + ", Offset:" + record.offset()); 63 | if (numberOfMessagesReadSoFar >= numberOfMessagesToRead){ 64 | keepOnReading = false; // to exit the while loop 65 | break; // to exit the for loop 66 | } 67 | } 68 | } 69 | 70 | log.info("Exiting the application"); 71 | 72 | } 73 | } -------------------------------------------------------------------------------- /kafka-basics/src/main/java/io/conduktor/demos/kafka/advanced/ConsumerDemoRebalanceListener.java: -------------------------------------------------------------------------------- 1 | package io.conduktor.demos.kafka.advanced; 2 | 3 | import org.apache.kafka.clients.CommonClientConfigs; 4 | import org.apache.kafka.clients.consumer.ConsumerConfig; 5 | import org.apache.kafka.clients.consumer.ConsumerRecord; 6 | import org.apache.kafka.clients.consumer.ConsumerRecords; 7 | import org.apache.kafka.clients.consumer.KafkaConsumer; 8 | import org.apache.kafka.common.config.SaslConfigs; 9 | import org.apache.kafka.common.errors.WakeupException; 10 | import org.apache.kafka.common.serialization.StringDeserializer; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.time.Duration; 15 | import java.util.Arrays; 16 | import java.util.Properties; 17 | 18 | public class ConsumerDemoRebalanceListener { 19 | private static final Logger log = LoggerFactory.getLogger(ConsumerDemoRebalanceListener.class); 20 | 21 | public static void main(String[] args) { 22 | log.info("I am a Kafka Consumer with a Rebalance"); 23 | 24 | String bootstrapServers = "127.0.0.1:9092"; 25 | String groupId = "my-fifth-application"; 26 | String topic = "demo_java"; 27 | 28 | // create consumer configs 29 | Properties properties = new Properties(); 30 | // properties.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:9092"); 31 | properties.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "cluster.playground.cdkt.io:9092"); 32 | properties.setProperty(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_SSL"); 33 | properties.setProperty(SaslConfigs.SASL_JAAS_CONFIG, "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"6hiWlwa3a3RibZIBq2lNEP\" password=\"8a575166-4c7d-4900-ad2c-f2b4a510f0ce\";"); 34 | properties.setProperty(SaslConfigs.SASL_MECHANISM, "PLAIN"); 35 | properties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 36 | properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 37 | properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, groupId); 38 | properties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); 39 | 40 | // we disable Auto Commit of offsets 41 | properties.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); 42 | 43 | // create consumer 44 | KafkaConsumer consumer = new KafkaConsumer<>(properties); 45 | 46 | ConsumerRebalanceListenerImpl listener = new ConsumerRebalanceListenerImpl(consumer); 47 | 48 | // get a reference to the current thread 49 | final Thread mainThread = Thread.currentThread(); 50 | 51 | // adding the shutdown hook 52 | Runtime.getRuntime().addShutdownHook(new Thread() { 53 | public void run() { 54 | log.info("Detected a shutdown, let's exit by calling consumer.wakeup()..."); 55 | consumer.wakeup(); 56 | 57 | // join the main thread to allow the execution of the code in the main thread 58 | try { 59 | mainThread.join(); 60 | } catch (InterruptedException e) { 61 | e.printStackTrace(); 62 | } 63 | } 64 | }); 65 | 66 | try { 67 | // subscribe consumer to our topic(s) 68 | consumer.subscribe(Arrays.asList(topic), listener); 69 | 70 | // poll for new data 71 | while (true) { 72 | ConsumerRecords records = 73 | consumer.poll(Duration.ofMillis(100)); 74 | 75 | for (ConsumerRecord record : records) { 76 | log.info("Key: " + record.key() + ", Value: " + record.value()); 77 | log.info("Partition: " + record.partition() + ", Offset:" + record.offset()); 78 | 79 | // we track the offset we have been committed in the listener 80 | listener.addOffsetToTrack(record.topic(), record.partition(), record.offset()); 81 | } 82 | 83 | // We commitAsync as we have processed all data and we don't want to block until the next .poll() call 84 | consumer.commitAsync(); 85 | } 86 | } catch (WakeupException e) { 87 | log.info("Wake up exception!"); 88 | // we ignore this as this is an expected exception when closing a consumer 89 | } catch (Exception e) { 90 | log.error("Unexpected exception", e); 91 | } finally { 92 | try { 93 | consumer.commitSync(listener.getCurrentOffsets()); // we must commit the offsets synchronously here 94 | } finally { 95 | consumer.close(); 96 | log.info("The consumer is now gracefully closed."); 97 | } 98 | } 99 | } 100 | } -------------------------------------------------------------------------------- /kafka-basics/src/main/java/io/conduktor/demos/kafka/advanced/ConsumerDemoThreads.java: -------------------------------------------------------------------------------- 1 | package io.conduktor.demos.kafka.advanced; 2 | 3 | import org.apache.kafka.clients.CommonClientConfigs; 4 | import org.apache.kafka.clients.consumer.*; 5 | import org.apache.kafka.common.config.SaslConfigs; 6 | import org.apache.kafka.common.errors.WakeupException; 7 | import org.apache.kafka.common.serialization.StringDeserializer; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.time.Duration; 12 | import java.util.Collections; 13 | import java.util.Date; 14 | import java.util.Properties; 15 | import java.util.concurrent.CountDownLatch; 16 | 17 | public class ConsumerDemoThreads { 18 | 19 | public static void main(String[] args) { 20 | ConsumerDemoWorker consumerDemoWorker = new ConsumerDemoWorker(); 21 | new Thread(consumerDemoWorker).start(); 22 | Runtime.getRuntime().addShutdownHook(new Thread(new ConsumerDemoCloser(consumerDemoWorker))); 23 | } 24 | 25 | private static class ConsumerDemoWorker implements Runnable { 26 | 27 | private static final Logger log = LoggerFactory.getLogger(ConsumerDemoWorker.class); 28 | 29 | private CountDownLatch countDownLatch; 30 | private Consumer consumer; 31 | 32 | @Override 33 | public void run() { 34 | countDownLatch = new CountDownLatch(1); 35 | final Properties properties = new Properties(); 36 | 37 | // properties.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:9092"); 38 | properties.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "cluster.playground.cdkt.io:9092"); 39 | properties.setProperty(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_SSL"); 40 | properties.setProperty(SaslConfigs.SASL_JAAS_CONFIG, "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"6hiWlwa3a3RibZIBq2lNEP\" password=\"8a575166-4c7d-4900-ad2c-f2b4a510f0ce\";"); 41 | properties.setProperty(SaslConfigs.SASL_MECHANISM, "PLAIN"); 42 | properties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 43 | properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 44 | properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "my-sixth-application"); 45 | properties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); 46 | 47 | consumer = new KafkaConsumer<>(properties); 48 | consumer.subscribe(Collections.singleton("demo_java")); 49 | 50 | final Duration pollTimeout = Duration.ofMillis(100); 51 | 52 | try { 53 | while (true) { 54 | final ConsumerRecords consumerRecords = consumer.poll(pollTimeout); 55 | for (final ConsumerRecord consumerRecord : consumerRecords) { 56 | log.info("Getting consumer record key: '" + consumerRecord.key() + "', value: '" + consumerRecord.value() + "', partition: " + consumerRecord.partition() + " and offset: " + consumerRecord.offset() + " at " + new Date(consumerRecord.timestamp())); 57 | } 58 | } 59 | } catch (WakeupException e) { 60 | log.info("Consumer poll woke up"); 61 | } finally { 62 | consumer.close(); 63 | countDownLatch.countDown(); 64 | } 65 | } 66 | 67 | void shutdown() throws InterruptedException { 68 | consumer.wakeup(); 69 | countDownLatch.await(); 70 | log.info("Consumer closed"); 71 | } 72 | 73 | } 74 | 75 | private static class ConsumerDemoCloser implements Runnable { 76 | 77 | private static final Logger log = LoggerFactory.getLogger(ConsumerDemoCloser.class); 78 | 79 | private final ConsumerDemoWorker consumerDemoWorker; 80 | 81 | ConsumerDemoCloser(final ConsumerDemoWorker consumerDemoWorker) { 82 | this.consumerDemoWorker = consumerDemoWorker; 83 | } 84 | 85 | @Override 86 | public void run() { 87 | try { 88 | consumerDemoWorker.shutdown(); 89 | } catch (InterruptedException e) { 90 | log.error("Error shutting down consumer", e); 91 | } 92 | } 93 | } 94 | } -------------------------------------------------------------------------------- /kafka-basics/src/main/java/io/conduktor/demos/kafka/advanced/ConsumerRebalanceListenerImpl.java: -------------------------------------------------------------------------------- 1 | package io.conduktor.demos.kafka.advanced; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; 4 | import org.apache.kafka.clients.consumer.KafkaConsumer; 5 | import org.apache.kafka.clients.consumer.OffsetAndMetadata; 6 | import org.apache.kafka.common.TopicPartition; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.util.Collection; 11 | import java.util.HashMap; 12 | import java.util.Map; 13 | 14 | public class ConsumerRebalanceListenerImpl implements ConsumerRebalanceListener { 15 | 16 | private static final Logger log = LoggerFactory.getLogger(ConsumerRebalanceListenerImpl.class); 17 | 18 | private KafkaConsumer consumer; 19 | private Map currentOffsets = new HashMap<>(); 20 | 21 | public ConsumerRebalanceListenerImpl(KafkaConsumer consumer) { 22 | this.consumer = consumer; 23 | } 24 | 25 | public void addOffsetToTrack(String topic, int partition, long offset){ 26 | currentOffsets.put( 27 | new TopicPartition(topic, partition), 28 | new OffsetAndMetadata(offset + 1, null)); 29 | } 30 | 31 | @Override 32 | public void onPartitionsRevoked(Collection partitions) { 33 | log.info("onPartitionsRevoked callback triggered"); 34 | log.info("Committing offsets: " + currentOffsets); 35 | 36 | consumer.commitSync(currentOffsets); 37 | } 38 | 39 | @Override 40 | public void onPartitionsAssigned(Collection partitions) { 41 | log.info("onPartitionsAssigned callback triggered"); 42 | } 43 | 44 | // this is used when we shut down our consumer gracefully 45 | public Map getCurrentOffsets() { 46 | return currentOffsets; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /kafka-consumer-opensearch/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'io.conduktor.demos' 6 | version '1.0-SNAPSHOT' 7 | 8 | repositories { 9 | mavenCentral() 10 | } 11 | 12 | dependencies { 13 | 14 | // https://mvnrepository.com/artifact/org.apache.kafka/kafka-clients 15 | implementation 'org.apache.kafka:kafka-clients:3.3.1' 16 | 17 | // https://mvnrepository.com/artifact/org.slf4j/slf4j-api 18 | implementation 'org.slf4j:slf4j-api:1.7.36' 19 | 20 | // https://mvnrepository.com/artifact/org.slf4j/slf4j-simple 21 | implementation 'org.slf4j:slf4j-simple:1.7.36' 22 | 23 | // https://search.maven.org/artifact/org.opensearch.client/opensearch-rest-high-level-client/1.2.4/jar 24 | implementation 'org.opensearch.client:opensearch-rest-high-level-client:1.3.2' 25 | 26 | // https://search.maven.org/artifact/com.google.code.gson/gson/2.9.0/jar 27 | implementation 'com.google.code.gson:gson:2.9.0' 28 | 29 | } 30 | test { 31 | useJUnitPlatform() 32 | } -------------------------------------------------------------------------------- /kafka-consumer-opensearch/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | opensearch: 4 | image: opensearchproject/opensearch:1.2.4 5 | environment: 6 | discovery.type: single-node 7 | plugins.security.disabled: "true" # disable https and logins 8 | compatibility.override_main_response_version: "true" 9 | ports: 10 | - 9200:9200 11 | - 9600:9600 # required for Performance Analyzer 12 | 13 | # console at http://localhost:5601/app/dev_tools#/console 14 | opensearch-dashboards: 15 | image: opensearchproject/opensearch-dashboards:1.2.0 16 | ports: 17 | - 5601:5601 18 | environment: 19 | OPENSEARCH_HOSTS: '["http://opensearch:9200"]' 20 | DISABLE_SECURITY_DASHBOARDS_PLUGIN: "true" 21 | -------------------------------------------------------------------------------- /kafka-consumer-opensearch/src/main/java/io/conduktor/demos/kafka/opensearch/OpenSearchConsumer.java: -------------------------------------------------------------------------------- 1 | package opensearch; 2 | 3 | import com.google.gson.JsonParser; 4 | import org.apache.http.HttpHost; 5 | import org.apache.http.auth.AuthScope; 6 | import org.apache.http.auth.UsernamePasswordCredentials; 7 | import org.apache.http.client.CredentialsProvider; 8 | import org.apache.http.impl.client.BasicCredentialsProvider; 9 | import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy; 10 | import org.apache.kafka.clients.CommonClientConfigs; 11 | import org.apache.kafka.clients.consumer.ConsumerConfig; 12 | import org.apache.kafka.clients.consumer.ConsumerRecord; 13 | import org.apache.kafka.clients.consumer.ConsumerRecords; 14 | import org.apache.kafka.clients.consumer.KafkaConsumer; 15 | import org.apache.kafka.common.config.SaslConfigs; 16 | import org.apache.kafka.common.errors.WakeupException; 17 | import org.apache.kafka.common.serialization.StringDeserializer; 18 | import org.opensearch.action.bulk.BulkRequest; 19 | import org.opensearch.action.bulk.BulkResponse; 20 | import org.opensearch.action.index.IndexRequest; 21 | import org.opensearch.client.RequestOptions; 22 | import org.opensearch.client.RestClient; 23 | import org.opensearch.client.RestHighLevelClient; 24 | import org.opensearch.client.indices.CreateIndexRequest; 25 | import org.opensearch.client.indices.GetIndexRequest; 26 | import org.opensearch.common.xcontent.XContentType; 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | import java.io.IOException; 31 | import java.net.URI; 32 | import java.time.Duration; 33 | import java.util.Collections; 34 | import java.util.Properties; 35 | 36 | public class OpenSearchConsumer { 37 | 38 | public static RestHighLevelClient createOpenSearchClient() { 39 | String connString = "http://localhost:9200"; 40 | // String connString = "https://c9p5mwld41:45zeygn9hy@kafka-course-2322630105.eu-west-1.bonsaisearch.net:443"; 41 | 42 | // we build a URI from the connection string 43 | RestHighLevelClient restHighLevelClient; 44 | URI connUri = URI.create(connString); 45 | // extract login information if it exists 46 | String userInfo = connUri.getUserInfo(); 47 | 48 | if (userInfo == null) { 49 | // REST client without security 50 | restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost(connUri.getHost(), connUri.getPort(), "http"))); 51 | 52 | } else { 53 | // REST client with security 54 | String[] auth = userInfo.split(":"); 55 | 56 | CredentialsProvider cp = new BasicCredentialsProvider(); 57 | cp.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(auth[0], auth[1])); 58 | 59 | restHighLevelClient = new RestHighLevelClient( 60 | RestClient.builder(new HttpHost(connUri.getHost(), connUri.getPort(), connUri.getScheme())) 61 | .setHttpClientConfigCallback( 62 | httpAsyncClientBuilder -> httpAsyncClientBuilder.setDefaultCredentialsProvider(cp) 63 | .setKeepAliveStrategy(new DefaultConnectionKeepAliveStrategy()))); 64 | 65 | 66 | } 67 | 68 | return restHighLevelClient; 69 | } 70 | 71 | private static KafkaConsumer createKafkaConsumer(){ 72 | 73 | String groupId = "consumer-opensearch-demo"; 74 | 75 | // create consumer configs 76 | Properties properties = new Properties(); 77 | properties.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:9092"); 78 | properties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 79 | properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 80 | properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, groupId); 81 | properties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest"); 82 | properties.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); 83 | 84 | // create consumer 85 | return new KafkaConsumer<>(properties); 86 | 87 | } 88 | 89 | private static String extractId(String json){ 90 | // gson library 91 | return JsonParser.parseString(json) 92 | .getAsJsonObject() 93 | .get("meta") 94 | .getAsJsonObject() 95 | .get("id") 96 | .getAsString(); 97 | } 98 | 99 | public static void main(String[] args) throws IOException { 100 | 101 | Logger log = LoggerFactory.getLogger(OpenSearchConsumer.class.getSimpleName()); 102 | 103 | // first create an OpenSearch Client 104 | RestHighLevelClient openSearchClient = createOpenSearchClient(); 105 | 106 | // create our Kafka Client 107 | KafkaConsumer consumer = createKafkaConsumer(); 108 | 109 | // get a reference to the main thread 110 | final Thread mainThread = Thread.currentThread(); 111 | 112 | // adding the shutdown hook 113 | Runtime.getRuntime().addShutdownHook(new Thread() { 114 | public void run() { 115 | log.info("Detected a shutdown, let's exit by calling consumer.wakeup()..."); 116 | consumer.wakeup(); 117 | 118 | // join the main thread to allow the execution of the code in the main thread 119 | try { 120 | mainThread.join(); 121 | } catch (InterruptedException e) { 122 | e.printStackTrace(); 123 | } 124 | } 125 | }); 126 | 127 | // we need to create the index on OpenSearch if it doesn't exist already 128 | 129 | try(openSearchClient; consumer){ 130 | 131 | boolean indexExists = openSearchClient.indices().exists(new GetIndexRequest("wikimedia"), RequestOptions.DEFAULT); 132 | 133 | if (!indexExists){ 134 | CreateIndexRequest createIndexRequest = new CreateIndexRequest("wikimedia"); 135 | openSearchClient.indices().create(createIndexRequest, RequestOptions.DEFAULT); 136 | log.info("The Wikimedia Index has been created!"); 137 | } else { 138 | log.info("The Wikimedia Index already exits"); 139 | } 140 | 141 | // we subscribe the consumer 142 | consumer.subscribe(Collections.singleton("wikimedia.recentchange")); 143 | 144 | 145 | while(true) { 146 | 147 | ConsumerRecords records = consumer.poll(Duration.ofMillis(3000)); 148 | 149 | int recordCount = records.count(); 150 | log.info("Received " + recordCount + " record(s)"); 151 | 152 | BulkRequest bulkRequest = new BulkRequest(); 153 | 154 | for (ConsumerRecord record : records) { 155 | 156 | // send the record into OpenSearch 157 | 158 | // strategy 1 159 | // define an ID using Kafka Record coordinates 160 | // String id = record.topic() + "_" + record.partition() + "_" + record.offset(); 161 | 162 | try { 163 | // strategy 2 164 | // we extract the ID from the JSON value 165 | String id = extractId(record.value()); 166 | 167 | IndexRequest indexRequest = new IndexRequest("wikimedia") 168 | .source(record.value(), XContentType.JSON) 169 | .id(id); 170 | 171 | // IndexResponse response = openSearchClient.index(indexRequest, RequestOptions.DEFAULT); 172 | 173 | bulkRequest.add(indexRequest); 174 | 175 | // log.info(response.getId()); 176 | } catch (Exception e){ 177 | 178 | } 179 | 180 | } 181 | 182 | 183 | if (bulkRequest.numberOfActions() > 0){ 184 | BulkResponse bulkResponse = openSearchClient.bulk(bulkRequest, RequestOptions.DEFAULT); 185 | log.info("Inserted " + bulkResponse.getItems().length + " record(s)."); 186 | 187 | try { 188 | Thread.sleep(1000); 189 | } catch (InterruptedException e) { 190 | e.printStackTrace(); 191 | } 192 | 193 | // commit offsets after the batch is consumed 194 | consumer.commitSync(); 195 | log.info("Offsets have been committed!"); 196 | } 197 | 198 | 199 | 200 | 201 | } 202 | 203 | 204 | } catch (WakeupException e) { 205 | log.info("Consumer is starting to shut down"); 206 | } catch (Exception e) { 207 | log.error("Unexpected exception in the consumer", e); 208 | } finally { 209 | consumer.close(); // close the consumer, this will also commit offsets 210 | openSearchClient.close(); 211 | log.info("The consumer is now gracefully shut down"); 212 | } 213 | 214 | } 215 | } 216 | -------------------------------------------------------------------------------- /kafka-producer-wikimedia/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'io.conduktor.demos' 6 | version '1.0-SNAPSHOT' 7 | 8 | repositories { 9 | mavenCentral() 10 | } 11 | 12 | 13 | dependencies { 14 | 15 | // https://mvnrepository.com/artifact/org.apache.kafka/kafka-clients 16 | implementation 'org.apache.kafka:kafka-clients:3.3.1' 17 | 18 | // https://mvnrepository.com/artifact/org.slf4j/slf4j-api 19 | implementation 'org.slf4j:slf4j-api:2.0.5' 20 | 21 | // https://mvnrepository.com/artifact/org.slf4j/slf4j-simple 22 | implementation 'org.slf4j:slf4j-simple:2.0.5' 23 | 24 | // https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp 25 | implementation 'com.squareup.okhttp3:okhttp:4.9.3' 26 | 27 | // https://mvnrepository.com/artifact/com.launchdarkly/okhttp-eventsource 28 | implementation 'com.launchdarkly:okhttp-eventsource:2.5.0' 29 | 30 | } 31 | 32 | test { 33 | useJUnitPlatform() 34 | } -------------------------------------------------------------------------------- /kafka-producer-wikimedia/src/main/java/io/conduktor/demos/kafka/wikimedia/WikimediaChangeHandler.java: -------------------------------------------------------------------------------- 1 | package io.conduktor.demos.kafka.wikimedia; 2 | 3 | import com.launchdarkly.eventsource.EventHandler; 4 | import com.launchdarkly.eventsource.MessageEvent; 5 | import org.apache.kafka.clients.producer.KafkaProducer; 6 | import org.apache.kafka.clients.producer.ProducerRecord; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | public class WikimediaChangeHandler implements EventHandler { 11 | 12 | KafkaProducer kafkaProducer; 13 | String topic; 14 | private final Logger log = LoggerFactory.getLogger(WikimediaChangeHandler.class.getSimpleName()); 15 | 16 | public WikimediaChangeHandler(KafkaProducer kafkaProducer, String topic){ 17 | this.kafkaProducer = kafkaProducer; 18 | this.topic = topic; 19 | } 20 | 21 | 22 | @Override 23 | public void onOpen() { 24 | // nothing here 25 | } 26 | 27 | @Override 28 | public void onClosed() { 29 | kafkaProducer.close(); 30 | } 31 | 32 | @Override 33 | public void onMessage(String event, MessageEvent messageEvent) { 34 | log.info(messageEvent.getData()); 35 | // asynchronous 36 | kafkaProducer.send(new ProducerRecord<>(topic, messageEvent.getData())); 37 | } 38 | 39 | @Override 40 | public void onComment(String comment) { 41 | // nothing here 42 | } 43 | 44 | @Override 45 | public void onError(Throwable t) { 46 | log.error("Error in Stream Reading", t); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /kafka-producer-wikimedia/src/main/java/io/conduktor/demos/kafka/wikimedia/WikimediaChangesProducer.java: -------------------------------------------------------------------------------- 1 | package io.conduktor.demos.kafka.wikimedia; 2 | 3 | import com.launchdarkly.eventsource.EventHandler; 4 | import com.launchdarkly.eventsource.EventSource; 5 | import org.apache.kafka.clients.producer.KafkaProducer; 6 | import org.apache.kafka.clients.producer.ProducerConfig; 7 | import org.apache.kafka.common.serialization.StringSerializer; 8 | 9 | import java.net.URI; 10 | import java.util.Properties; 11 | import java.util.concurrent.TimeUnit; 12 | 13 | public class WikimediaChangesProducer { 14 | 15 | public static void main(String[] args) throws InterruptedException { 16 | 17 | // create Producer Properties 18 | Properties properties = new Properties(); 19 | properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:9092"); 20 | properties.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 21 | properties.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 22 | 23 | // create the Producer 24 | KafkaProducer producer = new KafkaProducer<>(properties); 25 | 26 | String topic = "wikimedia.recentchange"; 27 | 28 | EventHandler eventHandler = new WikimediaChangeHandler(producer, topic); 29 | String url = "https://stream.wikimedia.org/v2/stream/recentchange"; 30 | EventSource.Builder builder = new EventSource.Builder(eventHandler, URI.create(url)); 31 | EventSource eventSource = builder.build(); 32 | 33 | 34 | // start the producer in another thread 35 | eventSource.start(); 36 | 37 | // we produce for 10 minutes and block the program until then 38 | TimeUnit.MINUTES.sleep(10); 39 | 40 | 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /kafka-streams-wikimedia/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'io.conduktor.demos' 6 | version '1.0-SNAPSHOT' 7 | 8 | repositories { 9 | mavenCentral() 10 | } 11 | 12 | dependencies { 13 | // https://mvnrepository.com/artifact/org.apache.kafka/kafka-streams 14 | implementation 'org.apache.kafka:kafka-streams:3.3.1' 15 | 16 | // https://mvnrepository.com/artifact/org.slf4j/slf4j-api 17 | implementation 'org.slf4j:slf4j-api:1.7.36' 18 | 19 | // https://mvnrepository.com/artifact/org.slf4j/slf4j-simple 20 | implementation 'org.slf4j:slf4j-simple:1.7.36' 21 | 22 | implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.3' 23 | } 24 | 25 | test { 26 | useJUnitPlatform() 27 | } -------------------------------------------------------------------------------- /kafka-streams-wikimedia/src/main/java/io/conduktor/demos/kafka/streams/wikimedia/WikimediaStreamsApp.java: -------------------------------------------------------------------------------- 1 | package io.conduktor.demos.kafka.streams.wikimedia; 2 | 3 | import io.conduktor.demos.kafka.streams.wikimedia.processor.BotCountStreamBuilder; 4 | import io.conduktor.demos.kafka.streams.wikimedia.processor.EventCountTimeseriesBuilder; 5 | import io.conduktor.demos.kafka.streams.wikimedia.processor.WebsiteCountStreamBuilder; 6 | import org.apache.kafka.common.serialization.Serdes; 7 | import org.apache.kafka.streams.KafkaStreams; 8 | import org.apache.kafka.streams.StreamsBuilder; 9 | import org.apache.kafka.streams.StreamsConfig; 10 | import org.apache.kafka.streams.Topology; 11 | import org.apache.kafka.streams.kstream.KStream; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import java.util.Properties; 16 | 17 | public class WikimediaStreamsApp { 18 | private static final Logger LOGGER = LoggerFactory.getLogger(WikimediaStreamsApp.class); 19 | private static final Properties properties; 20 | private static final String INPUT_TOPIC = "wikimedia.recentchange"; 21 | 22 | static { 23 | properties = new Properties(); 24 | properties.put(StreamsConfig.APPLICATION_ID_CONFIG, "wikimedia-stats-application"); 25 | properties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:9092"); 26 | properties.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 27 | properties.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 28 | } 29 | 30 | public static void main(String[] args) { 31 | StreamsBuilder builder = new StreamsBuilder(); 32 | KStream changeJsonStream = builder.stream(INPUT_TOPIC); 33 | 34 | BotCountStreamBuilder botCountStreamBuilder = new BotCountStreamBuilder(changeJsonStream); 35 | botCountStreamBuilder.setup(); 36 | 37 | WebsiteCountStreamBuilder websiteCountStreamBuilder = new WebsiteCountStreamBuilder(changeJsonStream); 38 | websiteCountStreamBuilder.setup(); 39 | 40 | EventCountTimeseriesBuilder eventCountTimeseriesBuilder = new EventCountTimeseriesBuilder(changeJsonStream); 41 | eventCountTimeseriesBuilder.setup(); 42 | 43 | final Topology appTopology = builder.build(); 44 | LOGGER.info("Topology: {}", appTopology.describe()); 45 | KafkaStreams streams = new KafkaStreams(appTopology, properties); 46 | streams.start(); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /kafka-streams-wikimedia/src/main/java/io/conduktor/demos/kafka/streams/wikimedia/processor/BotCountStreamBuilder.java: -------------------------------------------------------------------------------- 1 | package io.conduktor.demos.kafka.streams.wikimedia.processor; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.JsonNode; 5 | import com.fasterxml.jackson.databind.ObjectMapper; 6 | import org.apache.kafka.streams.kstream.KStream; 7 | import org.apache.kafka.streams.kstream.Materialized; 8 | 9 | import java.io.IOException; 10 | import java.util.Map; 11 | 12 | public class BotCountStreamBuilder { 13 | 14 | private static final String BOT_COUNT_STORE = "bot-count-store"; 15 | private static final String BOT_COUNT_TOPIC = "wikimedia.stats.bots"; 16 | private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); 17 | 18 | private final KStream inputStream; 19 | 20 | public BotCountStreamBuilder(KStream inputStream) { 21 | this.inputStream = inputStream; 22 | } 23 | 24 | public void setup() { 25 | this.inputStream 26 | .mapValues(changeJson -> { 27 | try { 28 | final JsonNode jsonNode = OBJECT_MAPPER.readTree(changeJson); 29 | if (jsonNode.get("bot").asBoolean()) { 30 | return "bot"; 31 | } 32 | return "non-bot"; 33 | } catch (IOException e) { 34 | return "parse-error"; 35 | } 36 | }) 37 | .groupBy((key, botOrNot) -> botOrNot) 38 | .count(Materialized.as(BOT_COUNT_STORE)) 39 | .toStream() 40 | .mapValues((key, value) -> { 41 | final Map kvMap = Map.of(String.valueOf(key), value); 42 | try { 43 | return OBJECT_MAPPER.writeValueAsString(kvMap); 44 | } catch (JsonProcessingException e) { 45 | return null; 46 | } 47 | }) 48 | .to(BOT_COUNT_TOPIC); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /kafka-streams-wikimedia/src/main/java/io/conduktor/demos/kafka/streams/wikimedia/processor/EventCountTimeseriesBuilder.java: -------------------------------------------------------------------------------- 1 | package io.conduktor.demos.kafka.streams.wikimedia.processor; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import org.apache.kafka.common.serialization.Serdes; 6 | import org.apache.kafka.streams.kstream.*; 7 | 8 | import java.time.Duration; 9 | import java.util.Map; 10 | 11 | public class EventCountTimeseriesBuilder { 12 | 13 | private static final String TIMESERIES_TOPIC = "wikimedia.stats.timeseries"; 14 | private static final String TIMESERIES_STORE = "event-count-store"; 15 | private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); 16 | 17 | private final KStream inputStream; 18 | 19 | public EventCountTimeseriesBuilder(KStream inputStream) { 20 | this.inputStream = inputStream; 21 | } 22 | 23 | public void setup() { 24 | final TimeWindows timeWindows = TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(10)); 25 | this.inputStream 26 | .selectKey((key, value) -> "key-to-group") 27 | .groupByKey() 28 | .windowedBy(timeWindows) 29 | .count(Materialized.as(TIMESERIES_STORE)) 30 | .toStream() 31 | .mapValues((readOnlyKey, value) -> { 32 | final Map kvMap = Map.of( 33 | "start_time", readOnlyKey.window().startTime().toString(), 34 | "end_time", readOnlyKey.window().endTime().toString(), 35 | "window_size", timeWindows.size(), 36 | "event_count", value 37 | ); 38 | try { 39 | return OBJECT_MAPPER.writeValueAsString(kvMap); 40 | } catch (JsonProcessingException e) { 41 | return null; 42 | } 43 | }) 44 | .to(TIMESERIES_TOPIC, Produced.with( 45 | WindowedSerdes.timeWindowedSerdeFrom(String.class, timeWindows.size()), 46 | Serdes.String() 47 | )); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /kafka-streams-wikimedia/src/main/java/io/conduktor/demos/kafka/streams/wikimedia/processor/WebsiteCountStreamBuilder.java: -------------------------------------------------------------------------------- 1 | package io.conduktor.demos.kafka.streams.wikimedia.processor; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.JsonNode; 5 | import com.fasterxml.jackson.databind.ObjectMapper; 6 | import org.apache.kafka.common.serialization.Serdes; 7 | import org.apache.kafka.streams.kstream.*; 8 | 9 | import java.io.IOException; 10 | import java.time.Duration; 11 | import java.util.Map; 12 | 13 | public class WebsiteCountStreamBuilder { 14 | private static final String WEBSITE_COUNT_STORE = "website-count-store"; 15 | private static final String WEBSITE_COUNT_TOPIC = "wikimedia.stats.website"; 16 | private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); 17 | 18 | private final KStream inputStream; 19 | 20 | public WebsiteCountStreamBuilder(KStream inputStream) { 21 | this.inputStream = inputStream; 22 | } 23 | 24 | public void setup() { 25 | final TimeWindows timeWindows = TimeWindows.ofSizeWithNoGrace(Duration.ofMinutes(1L)); 26 | this.inputStream 27 | .selectKey((k, changeJson) -> { 28 | try { 29 | final JsonNode jsonNode = OBJECT_MAPPER.readTree(changeJson); 30 | return jsonNode.get("server_name").asText(); 31 | } catch (IOException e) { 32 | return "parse-error"; 33 | } 34 | }) 35 | .groupByKey() 36 | .windowedBy(timeWindows) 37 | .count(Materialized.as(WEBSITE_COUNT_STORE)) 38 | .toStream() 39 | .mapValues((key, value) -> { 40 | final Map kvMap = Map.of( 41 | "website", key.key(), 42 | "count", value 43 | ); 44 | try { 45 | return OBJECT_MAPPER.writeValueAsString(kvMap); 46 | } catch (JsonProcessingException e) { 47 | return null; 48 | } 49 | }) 50 | .to(WEBSITE_COUNT_TOPIC, Produced.with( 51 | WindowedSerdes.timeWindowedSerdeFrom(String.class, timeWindows.size()), 52 | Serdes.String() 53 | )); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-beginners-course' 2 | include 'kafka-basics' 3 | include 'kafka-producer-wikimedia' 4 | include 'kafka-consumer-opensearch' 5 | include 'kafka-streams-wikimedia' 6 | --------------------------------------------------------------------------------