├── settings.gradle ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── .gitignore ├── src └── main │ └── java │ └── com │ └── inovatrend │ └── mtcdemo │ ├── Task.java │ └── MultithreadedKafkaConsumer.java ├── gradlew.bat └── gradlew /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'mtc-demo' 2 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/inovatrend/mtc-demo/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.4.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | .gradle 3 | build/ 4 | !gradle/wrapper/gradle-wrapper.jar 5 | !**/src/main/** 6 | !**/src/test/** 7 | 8 | ### IntelliJ IDEA ### 9 | .idea 10 | *.iws 11 | *.iml 12 | *.ipr 13 | out/ 14 | 15 | ### NetBeans ### 16 | /nbproject/private/ 17 | /nbbuild/ 18 | /dist/ 19 | /nbdist/ 20 | /.nb-gradle/ 21 | 22 | ### VS Code ### 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /src/main/java/com/inovatrend/mtcdemo/Task.java: -------------------------------------------------------------------------------- 1 | package com.inovatrend.mtcdemo; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRecord; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | import java.util.List; 8 | import java.util.concurrent.CompletableFuture; 9 | import java.util.concurrent.ExecutionException; 10 | import java.util.concurrent.atomic.AtomicLong; 11 | import java.util.concurrent.locks.ReentrantLock; 12 | 13 | 14 | public class Task implements Runnable { 15 | 16 | private final List> records; 17 | 18 | private volatile boolean stopped = false; 19 | 20 | private volatile boolean started = false; 21 | 22 | private volatile boolean finished = false; 23 | 24 | private final CompletableFuture completion = new CompletableFuture<>(); 25 | 26 | private final ReentrantLock startStopLock = new ReentrantLock(); 27 | 28 | private final AtomicLong currentOffset = new AtomicLong(); 29 | 30 | private Logger log = LoggerFactory.getLogger(Task.class); 31 | 32 | public Task(List> records) { 33 | this.records = records; 34 | } 35 | 36 | 37 | public void run() { 38 | startStopLock.lock(); 39 | if (stopped){ 40 | return; 41 | } 42 | started = true; 43 | startStopLock.unlock(); 44 | 45 | for (ConsumerRecord record : records) { 46 | if (stopped) 47 | break; 48 | // process record here and make sure you catch all exceptions; 49 | currentOffset.set(record.offset() + 1); 50 | } 51 | finished = true; 52 | completion.complete(currentOffset.get()); 53 | } 54 | 55 | public long getCurrentOffset() { 56 | return currentOffset.get(); 57 | } 58 | 59 | public void stop() { 60 | startStopLock.lock(); 61 | this.stopped = true; 62 | if (!started) { 63 | finished = true; 64 | completion.complete(currentOffset.get()); 65 | } 66 | startStopLock.unlock(); 67 | } 68 | 69 | public long waitForCompletion() { 70 | try { 71 | return completion.get(); 72 | } catch (InterruptedException | ExecutionException e) { 73 | return -1; 74 | } 75 | } 76 | 77 | public boolean isFinished() { 78 | return finished; 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto init 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto init 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :init 68 | @rem Get command-line arguments, handling Windows variants 69 | 70 | if not "%OS%" == "Windows_NT" goto win9xME_args 71 | 72 | :win9xME_args 73 | @rem Slurp the command line arguments. 74 | set CMD_LINE_ARGS= 75 | set _SKIP=2 76 | 77 | :win9xME_args_slurp 78 | if "x%~1" == "x" goto execute 79 | 80 | set CMD_LINE_ARGS=%* 81 | 82 | :execute 83 | @rem Setup the command line 84 | 85 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 86 | 87 | 88 | @rem Execute Gradle 89 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 90 | 91 | :end 92 | @rem End local scope for the variables with windows NT shell 93 | if "%ERRORLEVEL%"=="0" goto mainEnd 94 | 95 | :fail 96 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 97 | rem the _cmd.exe /c_ return code! 98 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 99 | exit /b 1 100 | 101 | :mainEnd 102 | if "%OS%"=="Windows_NT" endlocal 103 | 104 | :omega 105 | -------------------------------------------------------------------------------- /src/main/java/com/inovatrend/mtcdemo/MultithreadedKafkaConsumer.java: -------------------------------------------------------------------------------- 1 | package com.inovatrend.mtcdemo; 2 | 3 | import org.apache.kafka.clients.consumer.*; 4 | import org.apache.kafka.common.TopicPartition; 5 | import org.apache.kafka.common.errors.WakeupException; 6 | import org.apache.kafka.common.serialization.StringDeserializer; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.time.Duration; 11 | import java.time.temporal.ChronoUnit; 12 | import java.util.*; 13 | import java.util.concurrent.ExecutorService; 14 | import java.util.concurrent.Executors; 15 | import java.util.concurrent.atomic.AtomicBoolean; 16 | 17 | public class MultithreadedKafkaConsumer implements Runnable, ConsumerRebalanceListener { 18 | 19 | private final KafkaConsumer consumer; 20 | private final ExecutorService executor = Executors.newFixedThreadPool(8); 21 | private final Map activeTasks = new HashMap<>(); 22 | private final Map offsetsToCommit = new HashMap<>(); 23 | private final AtomicBoolean stopped = new AtomicBoolean(false); 24 | private long lastCommitTime = System.currentTimeMillis(); 25 | private final Logger log = LoggerFactory.getLogger(MultithreadedKafkaConsumer.class); 26 | 27 | 28 | public MultithreadedKafkaConsumer(String topic) { 29 | Properties config = new Properties(); 30 | config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); 31 | config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); 32 | config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); 33 | config.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); 34 | config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); 35 | config.put(ConsumerConfig.GROUP_ID_CONFIG, "multithreaded-consumer-demo"); 36 | consumer = new KafkaConsumer<>(config); 37 | new Thread(this).start(); 38 | } 39 | 40 | 41 | @Override 42 | public void run() { 43 | try { 44 | consumer.subscribe(Collections.singleton("topic-name"), this); 45 | while (!stopped.get()) { 46 | ConsumerRecords records = consumer.poll(Duration.of(100, ChronoUnit.MILLIS)); 47 | handleFetchedRecords(records); 48 | checkActiveTasks(); 49 | commitOffsets(); 50 | } 51 | } catch (WakeupException we) { 52 | if (!stopped.get()) 53 | throw we; 54 | } finally { 55 | consumer.close(); 56 | } 57 | } 58 | 59 | 60 | private void handleFetchedRecords(ConsumerRecords records) { 61 | if (records.count() > 0) { 62 | List partitionsToPause = new ArrayList<>(); 63 | records.partitions().forEach(partition -> { 64 | List> partitionRecords = records.records(partition); 65 | Task task = new Task(partitionRecords); 66 | partitionsToPause.add(partition); 67 | executor.submit(task); 68 | activeTasks.put(partition, task); 69 | }); 70 | consumer.pause(partitionsToPause); 71 | } 72 | } 73 | 74 | private void commitOffsets() { 75 | try { 76 | long currentTimeMillis = System.currentTimeMillis(); 77 | if (currentTimeMillis - lastCommitTime > 5000) { 78 | if(!offsetsToCommit.isEmpty()) { 79 | consumer.commitSync(offsetsToCommit); 80 | offsetsToCommit.clear(); 81 | } 82 | lastCommitTime = currentTimeMillis; 83 | } 84 | } catch (Exception e) { 85 | log.error("Failed to commit offsets!", e); 86 | } 87 | } 88 | 89 | 90 | private void checkActiveTasks() { 91 | List finishedTasksPartitions = new ArrayList<>(); 92 | activeTasks.forEach((partition, task) -> { 93 | if (task.isFinished()) 94 | finishedTasksPartitions.add(partition); 95 | long offset = task.getCurrentOffset(); 96 | if (offset > 0) 97 | offsetsToCommit.put(partition, new OffsetAndMetadata(offset)); 98 | }); 99 | finishedTasksPartitions.forEach(partition -> activeTasks.remove(partition)); 100 | consumer.resume(finishedTasksPartitions); 101 | } 102 | 103 | 104 | @Override 105 | public void onPartitionsRevoked(Collection partitions) { 106 | 107 | // 1. Stop all tasks handling records from revoked partitions 108 | Map stoppedTask = new HashMap<>(); 109 | for (TopicPartition partition : partitions) { 110 | Task task = activeTasks.remove(partition); 111 | if (task != null) { 112 | task.stop(); 113 | stoppedTask.put(partition, task); 114 | } 115 | } 116 | 117 | // 2. Wait for stopped tasks to complete processing of current record 118 | stoppedTask.forEach((partition, task) -> { 119 | long offset = task.waitForCompletion(); 120 | if (offset > 0) 121 | offsetsToCommit.put(partition, new OffsetAndMetadata(offset)); 122 | }); 123 | 124 | 125 | // 3. collect offsets for revoked partitions 126 | Map revokedPartitionOffsets = new HashMap<>(); 127 | partitions.forEach( partition -> { 128 | OffsetAndMetadata offset = offsetsToCommit.remove(partition); 129 | if (offset != null) 130 | revokedPartitionOffsets.put(partition, offset); 131 | }); 132 | 133 | // 4. commit offsets for revoked partitions 134 | try { 135 | consumer.commitSync(revokedPartitionOffsets); 136 | } catch (Exception e) { 137 | log.warn("Failed to commit offsets for revoked partitions!"); 138 | } 139 | } 140 | 141 | @Override 142 | public void onPartitionsAssigned(Collection partitions) { 143 | consumer.resume(partitions); 144 | } 145 | 146 | 147 | public void stopConsuming() { 148 | stopped.set(true); 149 | consumer.wakeup(); 150 | } 151 | 152 | } 153 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # 4 | # Copyright 2015 the original author or authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | ## 21 | ## Gradle start up script for UN*X 22 | ## 23 | ############################################################################## 24 | 25 | # Attempt to set APP_HOME 26 | # Resolve links: $0 may be a link 27 | PRG="$0" 28 | # Need this for relative symlinks. 29 | while [ -h "$PRG" ] ; do 30 | ls=`ls -ld "$PRG"` 31 | link=`expr "$ls" : '.*-> \(.*\)$'` 32 | if expr "$link" : '/.*' > /dev/null; then 33 | PRG="$link" 34 | else 35 | PRG=`dirname "$PRG"`"/$link" 36 | fi 37 | done 38 | SAVED="`pwd`" 39 | cd "`dirname \"$PRG\"`/" >/dev/null 40 | APP_HOME="`pwd -P`" 41 | cd "$SAVED" >/dev/null 42 | 43 | APP_NAME="Gradle" 44 | APP_BASE_NAME=`basename "$0"` 45 | 46 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 47 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 48 | 49 | # Use the maximum available, or set MAX_FD != -1 to use that value. 50 | MAX_FD="maximum" 51 | 52 | warn () { 53 | echo "$*" 54 | } 55 | 56 | die () { 57 | echo 58 | echo "$*" 59 | echo 60 | exit 1 61 | } 62 | 63 | # OS specific support (must be 'true' or 'false'). 64 | cygwin=false 65 | msys=false 66 | darwin=false 67 | nonstop=false 68 | case "`uname`" in 69 | CYGWIN* ) 70 | cygwin=true 71 | ;; 72 | Darwin* ) 73 | darwin=true 74 | ;; 75 | MINGW* ) 76 | msys=true 77 | ;; 78 | NONSTOP* ) 79 | nonstop=true 80 | ;; 81 | esac 82 | 83 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 84 | 85 | 86 | # Determine the Java command to use to start the JVM. 87 | if [ -n "$JAVA_HOME" ] ; then 88 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 89 | # IBM's JDK on AIX uses strange locations for the executables 90 | JAVACMD="$JAVA_HOME/jre/sh/java" 91 | else 92 | JAVACMD="$JAVA_HOME/bin/java" 93 | fi 94 | if [ ! -x "$JAVACMD" ] ; then 95 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 96 | 97 | Please set the JAVA_HOME variable in your environment to match the 98 | location of your Java installation." 99 | fi 100 | else 101 | JAVACMD="java" 102 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 103 | 104 | Please set the JAVA_HOME variable in your environment to match the 105 | location of your Java installation." 106 | fi 107 | 108 | # Increase the maximum file descriptors if we can. 109 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 110 | MAX_FD_LIMIT=`ulimit -H -n` 111 | if [ $? -eq 0 ] ; then 112 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 113 | MAX_FD="$MAX_FD_LIMIT" 114 | fi 115 | ulimit -n $MAX_FD 116 | if [ $? -ne 0 ] ; then 117 | warn "Could not set maximum file descriptor limit: $MAX_FD" 118 | fi 119 | else 120 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 121 | fi 122 | fi 123 | 124 | # For Darwin, add options to specify how the application appears in the dock 125 | if $darwin; then 126 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 127 | fi 128 | 129 | # For Cygwin or MSYS, switch paths to Windows format before running java 130 | if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then 131 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 132 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 133 | 134 | JAVACMD=`cygpath --unix "$JAVACMD"` 135 | 136 | # We build the pattern for arguments to be converted via cygpath 137 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 138 | SEP="" 139 | for dir in $ROOTDIRSRAW ; do 140 | ROOTDIRS="$ROOTDIRS$SEP$dir" 141 | SEP="|" 142 | done 143 | OURCYGPATTERN="(^($ROOTDIRS))" 144 | # Add a user-defined pattern to the cygpath arguments 145 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 146 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 147 | fi 148 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 149 | i=0 150 | for arg in "$@" ; do 151 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 152 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 153 | 154 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 155 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 156 | else 157 | eval `echo args$i`="\"$arg\"" 158 | fi 159 | i=`expr $i + 1` 160 | done 161 | case $i in 162 | 0) set -- ;; 163 | 1) set -- "$args0" ;; 164 | 2) set -- "$args0" "$args1" ;; 165 | 3) set -- "$args0" "$args1" "$args2" ;; 166 | 4) set -- "$args0" "$args1" "$args2" "$args3" ;; 167 | 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 168 | 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 169 | 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 170 | 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 171 | 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 172 | esac 173 | fi 174 | 175 | # Escape application args 176 | save () { 177 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 178 | echo " " 179 | } 180 | APP_ARGS=`save "$@"` 181 | 182 | # Collect all arguments for the java command, following the shell quoting and substitution rules 183 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 184 | 185 | exec "$JAVACMD" "$@" 186 | --------------------------------------------------------------------------------