├── .github ├── dependabot.yml └── workflows │ └── maven.yml ├── .gitignore ├── .mvn └── wrapper │ ├── MavenWrapperDownloader.java │ └── maven-wrapper.properties ├── Apache2LicenseText ├── KafkaInAction_AppendixB ├── Commands.md ├── pom.xml └── pythonexamples │ ├── pythonconsumer.py │ └── pythonproducer.py ├── KafkaInAction_Chapter1 └── README.md ├── KafkaInAction_Chapter10 ├── Commands.md └── pom.xml ├── KafkaInAction_Chapter11 ├── Commands.md ├── pom.xml └── src │ └── main │ ├── avro │ ├── alert.avsc │ └── alert_v2.avsc │ ├── java │ └── org │ │ └── kafkainaction │ │ ├── consumer │ │ └── AlertConsumer.java │ │ └── producer │ │ └── AlertProducer.java │ └── resources │ └── logback.xml ├── KafkaInAction_Chapter12 ├── Commands.adoc ├── pom.xml └── src │ ├── main │ ├── avro │ │ ├── account.avsc │ │ ├── funds.avsc │ │ ├── transaction.avsc │ │ └── transaction_result.avsc │ ├── java │ │ └── org │ │ │ └── kafkainaction │ │ │ └── kstreams2 │ │ │ ├── SchemaSerdes.java │ │ │ ├── TransactionProcessor.java │ │ │ └── TransactionTransformer.java │ └── resources │ │ ├── Ksql.sql │ │ └── logback.xml │ └── test │ └── java │ └── org │ └── kafkainaction │ └── kstreams2 │ ├── AccountProducerTest.java │ ├── TransactionProcessorTest.java │ ├── TransactionProducerTest.java │ ├── TransactionTransformerExtendedTest.java │ └── TransactionTransformerTest.java ├── KafkaInAction_Chapter2 ├── Commands.md ├── pom.xml ├── scripts │ ├── portInUse.sh │ ├── starteverything.sh │ └── stopeverything.sh └── src │ └── main │ ├── java │ └── org │ │ └── kafkainaction │ │ ├── consumer │ │ └── HelloWorldConsumer.java │ │ └── producer │ │ └── HelloWorldProducer.java │ └── resources │ └── logback.xml ├── KafkaInAction_Chapter3 ├── Commands.md ├── pom.xml └── src │ └── main │ ├── avro │ └── kinaction_alert.avsc │ ├── java │ └── org │ │ └── kafkainaction │ │ ├── consumer │ │ └── HelloWorldConsumer.java │ │ └── producer │ │ └── HelloWorldProducer.java │ └── resources │ └── logback.xml ├── KafkaInAction_Chapter4 ├── .mvn │ └── wrapper │ │ ├── MavenWrapperDownloader.java │ │ └── maven-wrapper.properties ├── Commands.md ├── mvnw ├── mvnw.cmd ├── pom.xml └── src │ └── main │ ├── java │ └── org │ │ └── kafkainaction │ │ ├── callback │ │ └── AlertCallback.java │ │ ├── consumer │ │ └── AlertConsumer.java │ │ ├── model │ │ └── Alert.java │ │ ├── partitioner │ │ └── AlertLevelPartitioner.java │ │ ├── producer │ │ ├── AlertProducer.java │ │ ├── AlertTrendingProducer.java │ │ ├── AuditProducer.java │ │ └── FlumeSinkProducer.java │ │ └── serde │ │ └── AlertKeySerde.java │ └── resources │ ├── kafkasink.conf │ └── logback.xml ├── KafkaInAction_Chapter5 ├── Commands.md ├── pom.xml └── src │ └── main │ ├── java │ └── org │ │ └── kafkainaction │ │ ├── consumer │ │ ├── ASyncCommit.java │ │ ├── AlertTrendConsumer.java │ │ ├── AuditConsumer.java │ │ ├── KinactionStopConsumer.java │ │ └── WebClickConsumer.java │ │ ├── model │ │ └── Alert.java │ │ └── serde │ │ └── AlertKeySerde.java │ └── resources │ └── logback.xml ├── KafkaInAction_Chapter6 ├── Commands.md └── pom.xml ├── KafkaInAction_Chapter7 ├── Commands.md ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── kafkainaction │ │ │ ├── consumer │ │ │ └── AlertConsumer.java │ │ │ ├── model │ │ │ └── Alert.java │ │ │ ├── partitioner │ │ │ └── AlertLevelPartitioner.java │ │ │ ├── producer │ │ │ └── AlertProducer.java │ │ │ └── serde │ │ │ └── AlertKeySerde.java │ └── resources │ │ └── logback.xml │ └── test │ └── java │ └── org │ └── kafkainaction │ └── producer │ └── AlertLevelPartitionerTest.java ├── KafkaInAction_Chapter8 ├── Commands.md └── pom.xml ├── KafkaInAction_Chapter9 ├── Commands.md ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── kafkainaction │ │ ├── callback │ │ └── AlertCallback.java │ │ ├── consumer │ │ ├── AlertConsumer.java │ │ └── AlertConsumerMetricsInterceptor.java │ │ ├── model │ │ └── Alert.java │ │ ├── producer │ │ ├── AlertProducer.java │ │ └── AlertProducerMetricsInterceptor.java │ │ ├── serde │ │ └── AlertKeySerde.java │ │ └── util │ │ └── CreateTopic.java │ └── monitoring-interceptors-5.2.2.jar ├── LICENSE ├── Makefile ├── README.md ├── docker-compose.yaml ├── errata.md ├── licenses ├── ApacheFlumeLicense.txt ├── ApacheFlumeNotice.txt ├── ApacheKafkaLicense.txt ├── ApacheKafkaNotice.txt ├── Links.md └── README.md ├── mvnw ├── mvnw.cmd ├── pom.xml ├── renovate.json └── wait-for-services.sh /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "maven" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | -------------------------------------------------------------------------------- /.github/workflows/maven.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Maven 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven 3 | 4 | name: Java CI with Maven 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | strategy: 16 | matrix: 17 | java: [ '11', '17', '21' ] 18 | name: Java ${{ matrix.java }} build 19 | 20 | steps: 21 | - uses: actions/checkout@v3 22 | - name: Set up JDK ${{ matrix.java }} 23 | uses: actions/setup-java@v3 24 | with: 25 | java-version: ${{ matrix.java }} 26 | distribution: 'temurin' 27 | cache: maven 28 | - name: Build with Maven 29 | run: ./mvnw -B package --file pom.xml 30 | 31 | smoke-test: 32 | runs-on: ubuntu-latest 33 | needs: build 34 | name: Confluent Platform Smoke Test 35 | 36 | steps: 37 | - uses: actions/checkout@v3 38 | - name: Set up Docker 39 | uses: docker/setup-buildx-action@v2 40 | - name: Set up Docker Compose 41 | run: | 42 | sudo apt-get update 43 | sudo apt-get install -y docker-compose 44 | - name: Run Smoke Test 45 | run: make smoke-test 46 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.toptal.com/developers/gitignore/api/java,maven,intellij 3 | # Edit at https://www.toptal.com/developers/gitignore?templates=java,maven,intellij 4 | 5 | ### Intellij ### 6 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider 7 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 8 | 9 | # User-specific stuff 10 | .idea/**/workspace.xml 11 | .idea/**/tasks.xml 12 | .idea/**/usage.statistics.xml 13 | .idea/**/dictionaries 14 | .idea/**/shelf 15 | 16 | # Generated files 17 | .idea/**/contentModel.xml 18 | 19 | # Sensitive or high-churn files 20 | .idea/**/dataSources/ 21 | .idea/**/dataSources.ids 22 | .idea/**/dataSources.local.xml 23 | .idea/**/sqlDataSources.xml 24 | .idea/**/dynamic.xml 25 | .idea/**/uiDesigner.xml 26 | .idea/**/dbnavigator.xml 27 | 28 | # Gradle 29 | .idea/**/gradle.xml 30 | .idea/**/libraries 31 | 32 | # Gradle and Maven with auto-import 33 | # When using Gradle or Maven with auto-import, you should exclude module files, 34 | # since they will be recreated, and may cause churn. Uncomment if using 35 | # auto-import. 36 | # .idea/artifacts 37 | # .idea/compiler.xml 38 | # .idea/jarRepositories.xml 39 | # .idea/modules.xml 40 | # .idea/*.iml 41 | # .idea/modules 42 | # *.iml 43 | # *.ipr 44 | 45 | # CMake 46 | cmake-build-*/ 47 | 48 | # Mongo Explorer plugin 49 | .idea/**/mongoSettings.xml 50 | 51 | # File-based project format 52 | *.iws 53 | 54 | # IntelliJ 55 | out/ 56 | 57 | # mpeltonen/sbt-idea plugin 58 | .idea_modules/ 59 | 60 | # JIRA plugin 61 | atlassian-ide-plugin.xml 62 | 63 | # Cursive Clojure plugin 64 | .idea/replstate.xml 65 | 66 | # Crashlytics plugin (for Android Studio and IntelliJ) 67 | com_crashlytics_export_strings.xml 68 | crashlytics.properties 69 | crashlytics-build.properties 70 | fabric.properties 71 | 72 | # Editor-based Rest Client 73 | .idea/httpRequests 74 | 75 | # Android studio 3.1+ serialized cache file 76 | .idea/caches/build_file_checksums.ser 77 | 78 | ### Intellij Patch ### 79 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 80 | 81 | # *.iml 82 | # modules.xml 83 | # .idea/misc.xml 84 | # *.ipr 85 | 86 | # Sonarlint plugin 87 | # https://plugins.jetbrains.com/plugin/7973-sonarlint 88 | .idea/**/sonarlint/ 89 | 90 | # SonarQube Plugin 91 | # https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin 92 | .idea/**/sonarIssues.xml 93 | 94 | # Markdown Navigator plugin 95 | # https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced 96 | .idea/**/markdown-navigator.xml 97 | .idea/**/markdown-navigator-enh.xml 98 | .idea/**/markdown-navigator/ 99 | 100 | # Cache file creation bug 101 | # See https://youtrack.jetbrains.com/issue/JBR-2257 102 | .idea/$CACHE_FILE$ 103 | 104 | # CodeStream plugin 105 | # https://plugins.jetbrains.com/plugin/12206-codestream 106 | .idea/codestream.xml 107 | 108 | .idea/ 109 | 110 | ### Java ### 111 | # Compiled class file 112 | *.class 113 | 114 | # Log file 115 | *.log 116 | 117 | # BlueJ files 118 | *.ctxt 119 | 120 | # Mobile Tools for Java (J2ME) 121 | .mtj.tmp/ 122 | 123 | # Package Files # 124 | *.jar 125 | *.war 126 | *.nar 127 | *.ear 128 | *.zip 129 | *.tar.gz 130 | *.rar 131 | 132 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 133 | hs_err_pid* 134 | 135 | ### Maven ### 136 | target/ 137 | pom.xml.tag 138 | pom.xml.releaseBackup 139 | pom.xml.versionsBackup 140 | pom.xml.next 141 | release.properties 142 | dependency-reduced-pom.xml 143 | buildNumber.properties 144 | .mvn/timing.properties 145 | # https://github.com/takari/maven-wrapper#usage-without-binary-jar 146 | .mvn/wrapper/maven-wrapper.jar 147 | 148 | # End of https://www.toptal.com/developers/gitignore/api/java,maven,intellij 149 | 150 | .project 151 | .settings 152 | .classpath 153 | .vscode 154 | -------------------------------------------------------------------------------- /.mvn/wrapper/MavenWrapperDownloader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2007-present the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | import java.net.*; 17 | import java.io.*; 18 | import java.nio.channels.*; 19 | import java.util.Properties; 20 | 21 | public class MavenWrapperDownloader { 22 | 23 | private static final String WRAPPER_VERSION = "0.5.6"; 24 | /** 25 | * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. 26 | */ 27 | private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/" 28 | + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar"; 29 | 30 | /** 31 | * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to 32 | * use instead of the default one. 33 | */ 34 | private static final String MAVEN_WRAPPER_PROPERTIES_PATH = 35 | ".mvn/wrapper/maven-wrapper.properties"; 36 | 37 | /** 38 | * Path where the maven-wrapper.jar will be saved to. 39 | */ 40 | private static final String MAVEN_WRAPPER_JAR_PATH = 41 | ".mvn/wrapper/maven-wrapper.jar"; 42 | 43 | /** 44 | * Name of the property which should be used to override the default download url for the wrapper. 45 | */ 46 | private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; 47 | 48 | public static void main(String args[]) { 49 | System.out.println("- Downloader started"); 50 | File baseDirectory = new File(args[0]); 51 | System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); 52 | 53 | // If the maven-wrapper.properties exists, read it and check if it contains a custom 54 | // wrapperUrl parameter. 55 | File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); 56 | String url = DEFAULT_DOWNLOAD_URL; 57 | if(mavenWrapperPropertyFile.exists()) { 58 | FileInputStream mavenWrapperPropertyFileInputStream = null; 59 | try { 60 | mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); 61 | Properties mavenWrapperProperties = new Properties(); 62 | mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); 63 | url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); 64 | } catch (IOException e) { 65 | System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); 66 | } finally { 67 | try { 68 | if(mavenWrapperPropertyFileInputStream != null) { 69 | mavenWrapperPropertyFileInputStream.close(); 70 | } 71 | } catch (IOException e) { 72 | // Ignore ... 73 | } 74 | } 75 | } 76 | System.out.println("- Downloading from: " + url); 77 | 78 | File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); 79 | if(!outputFile.getParentFile().exists()) { 80 | if(!outputFile.getParentFile().mkdirs()) { 81 | System.out.println( 82 | "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'"); 83 | } 84 | } 85 | System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); 86 | try { 87 | downloadFileFromURL(url, outputFile); 88 | System.out.println("Done"); 89 | System.exit(0); 90 | } catch (Throwable e) { 91 | System.out.println("- Error downloading"); 92 | e.printStackTrace(); 93 | System.exit(1); 94 | } 95 | } 96 | 97 | private static void downloadFileFromURL(String urlString, File destination) throws Exception { 98 | if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) { 99 | String username = System.getenv("MVNW_USERNAME"); 100 | char[] password = System.getenv("MVNW_PASSWORD").toCharArray(); 101 | Authenticator.setDefault(new Authenticator() { 102 | @Override 103 | protected PasswordAuthentication getPasswordAuthentication() { 104 | return new PasswordAuthentication(username, password); 105 | } 106 | }); 107 | } 108 | URL website = new URL(urlString); 109 | ReadableByteChannel rbc; 110 | rbc = Channels.newChannel(website.openStream()); 111 | FileOutputStream fos = new FileOutputStream(destination); 112 | fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); 113 | fos.close(); 114 | rbc.close(); 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip 2 | wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar 3 | -------------------------------------------------------------------------------- /KafkaInAction_AppendixB/Commands.md: -------------------------------------------------------------------------------- 1 | # Commands used in Appendix B 2 | 3 | ## Python commands 4 | 5 | * `brew install librdkafka` - optional 6 | * `pip install confluent-kafka` 7 | * `python pythonproducer.py` 8 | * `python pythonconsumer.py` 9 | -------------------------------------------------------------------------------- /KafkaInAction_AppendixB/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | Kafka-In-Action 6 | org.kafkainaction 7 | 1.0.0-SNAPSHOT 8 | 9 | 10 | appendixB-no-java 11 | Appendix B 12 | 13 | 14 | appendixB 15 | 16 | 17 | -------------------------------------------------------------------------------- /KafkaInAction_AppendixB/pythonexamples/pythonconsumer.py: -------------------------------------------------------------------------------- 1 | from confluent_kafka import Consumer, KafkaError #(1) 2 | 3 | consumer = Consumer({ 4 | 'bootstrap.servers': 'localhost:9094', #(2) 5 | 'group.id': 'kinaction_team0group', 6 | 'auto.offset.reset': 'earliest' 7 | }) 8 | 9 | consumer.subscribe(['kinaction-python-topic']) #(3) 10 | 11 | try: 12 | while True: 13 | message = consumer.poll(2.5) #(4) 14 | 15 | if message is None: 16 | continue 17 | if message.error(): 18 | print('kinaction_error: %s' % message.error()) 19 | continue 20 | else: 21 | print('kinaction_info: %s for topic: %s\n' % 22 | (message.value().decode('utf-8'), message.topic())) 23 | 24 | except KeyboardInterrupt: 25 | print('kinaction_info: stopping\n') 26 | finally: 27 | consumer.close() #(5) 28 | 29 | -------------------------------------------------------------------------------- /KafkaInAction_AppendixB/pythonexamples/pythonproducer.py: -------------------------------------------------------------------------------- 1 | from confluent_kafka import Producer #(1) 2 | 3 | producer = Producer({'bootstrap.servers': 'localhost:9094'}) #(2) 4 | 5 | def result(err, message): #(3) 6 | if err: 7 | print('kinaction_error %s\n' % err) 8 | else: 9 | print('kinaction_info: topic=%s, and kinaction_offset=%d\n' % 10 | (message.topic(), message.offset())) 11 | 12 | messages = ["hello kinaction_python", "hello again"] #(4) 13 | 14 | for msg in messages: 15 | producer.poll(0) 16 | producer.produce("kinaction-python-topic", value=msg.encode('utf-8'), callback=result) #(5) 17 | 18 | producer.flush() #(6) 19 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter1/README.md: -------------------------------------------------------------------------------- 1 | # Source Code for Kafka in Action 2 | 3 | ## Notes 4 | 5 | Here are some notes regarding the source code: 6 | 7 | 1. Select shell commands will be presented in a [Markdown](https://daringfireball.net/projects/markdown/syntax) format in a file called Commands.md or in [AsciiDoc](https://docs.asciidoctor.org/asciidoc/latest/) called Commands.adoc for each Chapter folder if there are any commands selected in that chapter. 8 | 9 | ### Requirements 10 | 11 | This project was built with the following versions: 12 | 13 | 1. Java 11 14 | 1. Apache Maven 3.5.0 15 | 1. Kafka Client libraries 2.7.1 16 | 17 | ### IDE setup 18 | 19 | 1. We have used Eclipse for our IDE. 20 | To set up for Eclipse run `mvn eclipse:eclipse` from the base directory of this repo. 21 | Or, you can Import->Existing Maven Projects. 22 | 23 | ### Installing Kafka 24 | 25 | Run the following in a directory (without spaces in the path) once you get the artifact downloaded. Refer to Appendix A if needed. 26 | 27 | tar -xzf kafka_2.13-2.7.1.tgz 28 | cd kafka_2.13-2.7.1 29 | 30 | ### Running Kafka 31 | 32 | 1. To start Kafka go to `/kafka_2.13-2.7.1/` 33 | 2. Run `bin/zookeeper-server-start.sh config/zookeeper.properties` 34 | 3. Modify the Kafka server configs 35 | 36 | 37 | cp config/server.properties config/server0.properties 38 | cp config/server.properties config/server1.properties 39 | cp config/server.properties config/server2.properties 40 | 41 | # vi config/server0.properties 42 | 43 | ```` 44 | broker.id=0 45 | listeners=PLAINTEXT://localhost:9092 46 | log.dirs=/tmp/kafkainaction/kafka-logs-0 47 | ```` 48 | 49 | # vi config/server1.properties 50 | 51 | ```` 52 | broker.id=1 53 | listeners=PLAINTEXT://localhost:9093 54 | log.dirs=/tmp/kafkainaction/kafka-logs-1 55 | ```` 56 | 57 | # vi config/server2.properties 58 | 59 | ```` 60 | broker.id=2 61 | listeners=PLAINTEXT://localhost:9094 62 | log.dirs=/tmp/kafkainaction/kafka-logs-2 63 | ```` 64 | 65 | 4. Start the Kafka Brokers: 66 | 67 | ```` 68 | bin/kafka-server-start.sh config/server0.properties 69 | bin/kafka-server-start.sh config/server1.properties 70 | bin/kafka-server-start.sh config/server2.properties 71 | ```` 72 | 73 | ### Stopping Kafka 74 | 75 | 1. To stop Kafka go to /kafka_2.13-2.7.1/ 76 | 2. Run `bin/kafka-server-stop.sh` 77 | 3. Run `bin/zookeeper-server-stop.sh` 78 | 79 | ### Code by Chapter 80 | 81 | Most of the code from the book can be found in the project corresponding to the chapter. Some code has been moved to other chapters in order to reduce the number of replication of related classes. 82 | 83 | ### Running the examples 84 | 85 | Most of the example programs can be run from within an IDE or from the command line. 86 | Make sure that your ZooKeeper and Kafka Brokers are up and running before you can run any of the examples. 87 | 88 | The examples will usually write out to topics and print to the console. 89 | 90 | ### Shell Scripts 91 | 92 | In the Chapter 2 project, we have included a couple of scripts if you want to use them under src/main/resources. 93 | 94 | They include: 95 | * `starteverything.sh` //This will start your ZooKeeper and Kafka Brokers (you will still have to go through the first time setup with Appendix A before using this.) 96 | * `stopeverything.sh` // Will stop ZooKeeper and your brokers 97 | * `portInUse.sh` // If you get a port in use error on startup, this script will kill all processes using those ports (assuming you are using the same ports as in Appendix A setup). 98 | 99 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter10/Commands.md: -------------------------------------------------------------------------------- 1 | # Commands used in Chapter 10 2 | 3 | # Note 4 | * A security professional should be consulted for the correct way to 5 | set up your own environment. Our commands are meant as a guide for getting 6 | familiar and for learning, not as a production level of security. This is not 7 | a complete guide. Use it at your own risk. 8 | The EXAMPLE ONLY label notes that these commands are not likely to work for your setup due to the need for the domains in our example as well as keys needing created, etc. 9 | 10 | ## EXAMPLE ONLY - SSL Key Generation for a Broker 11 | 12 | keytool -genkey -noprompt \ 13 | -alias localhost \ 14 | -dname "CN=ka.manning.com, OU=TEST, O=TREASURE, L=Bend, S=Or, C=US" \ 15 | -keystore kafka.broker0.keystore.jks \ 16 | -keyalg RSA \ 17 | -storepass changeTreasure \ 18 | -keypass changeTreasure \ 19 | -validity 999 20 | 21 | ## EXAMPLE ONLY - Creating our own Certificate Authority 22 | openssl req -new -x509 -keyout cakey.crt -out ca.crt \ 23 | -days 999 -subj '/CN=localhost/OU=TEST/O=TREASURE/L=Bend/S=Or/C=US' \ 24 | -passin pass:changeTreasure -passout pass:changeTreasure 25 | 26 | 27 | ## EXAMPLE ONLY - Broker server properties changes 28 | 29 | listeners=PLAINTEXT://localhost:9092,SSL://localhost:9093 30 | ssl.truststore.location=/var/kafkainaction/private/kafka.broker0.truststore.jks 31 | ssl.truststore.password=changeTreasure 32 | ssl.keystore.location=/var/kafkainaction/private/kafka.broker0.keystore.jks 33 | ssl.keystore.password=changeTreasure 34 | ssl.key.password=changeTreasure 35 | 36 | ## EXAMPLE ONLY - SSL Configuration for Clients 37 | 38 | security.protocol=SSL 39 | ssl.truststore.location=/var/kafkainaction/ssl/client.truststore.jks 40 | ssl.truststore.password=changeTreasure 41 | 42 | ## EXAMPLE ONLY - Using SSL Configuration for Command line Clients 43 | 44 | bin/kafka-console-producer.sh --bootstrap-server localhost:9093 --topic kinaction_test_ssl \ 45 | --producer.config kinaction-ssl.properties 46 | bin/kafka-console-consumer.sh --bootstrap-server localhost:9093 --topic kinaction_test_ssl \ 47 | --consumer.config kinaction-ssl.properties 48 | 49 | ## EXAMPLE ONLY - Broker SASL JAAS File 50 | 51 | KafkaServer { 52 | com.sun.security.auth.module.Krb5LoginModule required 53 | useKeyTab=true 54 | storeKey=true 55 | keyTab=”/opt/kafkainaction/kafka_server0.keytab” 56 | principal=”kafka/kafka0.ka.manning.com@MANNING.COM”; 57 | }; 58 | 59 | 60 | ## EXAMPLE ONLY - Broker SASL properties changes 61 | 62 | listeners=PLAINTEXT://localhost:9092,SSL://localhost:9093,SASL_SSL://localhost:9094 63 | sasl.kerberos.service.name=kafka 64 | 65 | 66 | ## ACL Authorizer and Super Users 67 | 68 | authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer 69 | super.users=User:Franz 70 | 71 | ## Kafka ACLs To Read and Write to a Topic 72 | 73 | bin/kafka-acls.sh --authorizer-properties \ 74 | --bootstrap-server localhost:9094 --add \ 75 | --allow-principal User:Franz --allow-principal User:Hemingway \ 76 | --operation Read --operation Write --topic kinaction_clueful_secrets 77 | 78 | ## ACL ZooKeeper 79 | 80 | zookeeper.set.acl=true 81 | 82 | 83 | ## Creating a Network Bandwidth Quota for Client Clueful 84 | 85 | bin/kafka-configs.sh --bootstrap-server localhost:9094 --alter \ 86 | --add-config 'producer_byte_rate=1048576,consumer_byte_rate=5242880' \ 87 | --entity-type clients --entity-name kinaction_clueful 88 | 89 | ## Listing and Deleting a Quota for Client Clueful 90 | 91 | bin/kafka-configs.sh --bootstrap-server localhost:9094 \ 92 | ---describe \ 93 | --entity-type clients --entity-name kinaction_clueful 94 | 95 | bin/kafka-configs.sh --bootstrap-server localhost:9094 --alter \ 96 | --delete-config 'producer_byte_rate,consumer_byte_rate' \ 97 | --entity-type clients --entity-name kinaction_clueful 98 | 99 | ## Creating a Network Bandwidth Quota for Client Clueful 100 | 101 | bin/kafka-configs.sh --bootstrap-server localhost:9094 --alter \ 102 | --add-config 'request_percentage=100' \ 103 | --entity-type clients --entity-name kinaction_clueful 104 | 105 | 106 | # Reference 107 | * https://docs.confluent.io/platform/current/security/security_tutorial.html 108 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter10/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | Kafka-In-Action 6 | org.kafkainaction 7 | 1.0.0-SNAPSHOT 8 | 9 | 10 | chapter10-no-java 11 | Protecting Kafka 12 | 13 | 14 | chapter10 15 | 16 | 17 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter11/Commands.md: -------------------------------------------------------------------------------- 1 | # Commands used in Chapter 11 2 | 3 | ## Important Schema Registry Configuration 4 | 5 | ```` 6 | listeners=http://localhost:8081 7 | kafkastore.connection.url=localhost:2181 8 | kafkastore.topic=_schemas 9 | debug=false 10 | ```` 11 | 12 | ## Starting Schema Registry 13 | 14 | ```` 15 | bin/schema-registry-start \ 16 | ./etc/schema-registry/schema-registry.properties 17 | ```` 18 | 19 | ## Listing Schema Registry Configuration 20 | 21 | curl -X GET http://localhost:8081/config 22 | 23 | ## Checking Compatibility with the Schema Registry Maven Plugin 24 | 25 | ```` 26 | 27 | io.confluent 28 | kafka-schema-registry-maven-plugin 29 | 5.3.1 30 | 31 | 32 | http://localhost:8081 33 | 34 | 35 | 36 | src/main/avro/alert_v2.avsc 37 | 38 | 39 | 40 | 41 | test-compatibility 42 | 43 | 44 | ```` 45 | 46 | ### More Commands 47 | 48 | > curl -X GET http://localhost:8081/config 49 | > 50 | > curl http://localhost:8081/subjects/kinaction_schematest-value/versions/1 51 | > 52 | > curl -X POST -H "Content-Type: application/vnd.schemaregistry.v1+json" --data '{ "schema": "{ \"type\": \"record\", \"name\": \"Alert\", \"fields\": [{ \"name\": \"notafield\", \"type\": \"long\" } ]}" }' http://localhost:8081/compatibility/subjects/kinaction_schematest-value/versions/latest 53 | 54 | 55 | ### Maven Ouput 56 | 57 | [INFO] --- kafka-schema-registry-maven-plugin:6.2.1:test-compatibility (default-cli) @ chapter11 --- 58 | 59 | [INFO] Schema Kafka-In-Action-Source-Code/KafkaInAction_Chapter11/src/main/avro/alert_v2.avsc is compatible with subject(kinaction_schematest-value) 60 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter11/pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 5 | Kafka-In-Action 6 | org.kafkainaction 7 | 1.0.0-SNAPSHOT 8 | 9 | 4.0.0 10 | 11 | chapter11 12 | Schema Registry 13 | 14 | 15 | 16 | org.apache.avro 17 | avro 18 | ${avro.version} 19 | 20 | 21 | io.confluent 22 | kafka-avro-serializer 23 | ${confluent.version} 24 | 25 | 26 | 27 | 28 | 29 | confluent 30 | https://packages.confluent.io/maven/ 31 | 32 | 33 | 34 | 35 | chapter11 36 | 37 | 38 | maven-assembly-plugin 39 | 40 | 41 | org.apache.avro 42 | avro-maven-plugin 43 | ${avro.version} 44 | 45 | 46 | generate-sources 47 | 48 | schema 49 | 50 | 51 | ${project.basedir}/src/main/avro/ 52 | ${project.build.directory}/generated/src/main/java/ 53 | 54 | 55 | 56 | 57 | 58 | 59 | io.confluent 60 | kafka-schema-registry-maven-plugin 61 | 6.2.1 62 | 63 | 64 | http://localhost:8081 65 | 66 | 67 | src/main/avro/alert_v2.avsc 68 | 69 | 70 | test-compatibility 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter11/src/main/avro/alert.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "org.kafkainaction.avro", 3 | "type": "record", 4 | "name": "Alert", 5 | "fields": [ 6 | { 7 | "name": "sensor_id", 8 | "type": "long", 9 | "doc": "The unique id that identifies the sensor" 10 | }, 11 | { 12 | "name": "time", 13 | "type": "long", 14 | "doc": "Time the alert was generated as UTC milliseconds from the epoch" 15 | }, 16 | { 17 | "name": "status", 18 | "type": { 19 | "type": "enum", 20 | "name": "AlertStatus", 21 | "symbols": [ 22 | "Critical", 23 | "Major", 24 | "Minor", 25 | "Warning" 26 | ] 27 | }, 28 | "doc": "The allowed values that our sensors will use to emit current status" 29 | } 30 | ] 31 | } -------------------------------------------------------------------------------- /KafkaInAction_Chapter11/src/main/avro/alert_v2.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "org.kafkainaction.avro", 3 | "type": "record", 4 | "name": "Alert", 5 | "fields": [ 6 | { 7 | "name": "sensor_id", 8 | "type": "long", 9 | "doc": "The unique id that identifies the sensor" 10 | }, 11 | { 12 | "name": "time", 13 | "type": "long", 14 | "doc": "Time the alert was generated as UTC milliseconds from the epoch" 15 | }, 16 | { 17 | "name": "status", 18 | "type": { 19 | "type": "enum", 20 | "name": "AlertStatus", 21 | "symbols": [ 22 | "Critical", 23 | "Major", 24 | "Minor", 25 | "Warning" 26 | ] 27 | }, 28 | "doc": "The allowed values that our sensors will use to emit current status" 29 | }, 30 | { 31 | "name": "recovery_details", 32 | "type": "string", 33 | "default": "Analyst recovery needed" 34 | } 35 | ] 36 | } -------------------------------------------------------------------------------- /KafkaInAction_Chapter11/src/main/java/org/kafkainaction/consumer/AlertConsumer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.consumer; 2 | 3 | import org.kafkainaction.avro.Alert; 4 | 5 | import org.apache.kafka.clients.consumer.ConsumerRecord; 6 | import org.apache.kafka.clients.consumer.ConsumerRecords; 7 | import org.apache.kafka.clients.consumer.KafkaConsumer; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.time.Duration; 12 | import java.util.List; 13 | import java.util.Properties; 14 | 15 | import io.confluent.kafka.serializers.KafkaAvroDeserializerConfig; 16 | 17 | public class AlertConsumer { 18 | 19 | final static Logger log = LoggerFactory.getLogger(AlertConsumer.class); 20 | 21 | public static void main(String[] args) { 22 | Properties kaProperties = new Properties(); 23 | kaProperties.put("bootstrap.servers", "localhost:9094"); 24 | kaProperties.put("group.id", "alertinterceptor"); 25 | kaProperties.put("enable.auto.commit", "true"); 26 | kaProperties.put("auto.commit.interval.ms", "1000"); 27 | kaProperties.put("key.deserializer", "org.apache.kafka.common.serialization.LongDeserializer"); 28 | kaProperties.put("value.deserializer", "io.confluent.kafka.serializers.KafkaAvroDeserializer"); // <1> 29 | kaProperties.put("schema.registry.url", "http://localhost:8081"); // <2> 30 | kaProperties.put(KafkaAvroDeserializerConfig.SPECIFIC_AVRO_READER_CONFIG, true); 31 | 32 | KafkaConsumer consumer = new KafkaConsumer(kaProperties); //C <3> 33 | 34 | consumer.subscribe(List.of("kinaction_schematest")); //<4> 35 | 36 | while (true) { 37 | ConsumerRecords records = consumer.poll(Duration.ofMillis(250)); 38 | for (ConsumerRecord record : records) { 39 | log.info("kinaction_info Alert Content = {}", record.value().toString()); //<5> 40 | } 41 | } 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter11/src/main/java/org/kafkainaction/producer/AlertProducer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.producer; 2 | 3 | import org.kafkainaction.avro.Alert; 4 | 5 | import org.apache.kafka.clients.producer.KafkaProducer; 6 | import org.apache.kafka.clients.producer.Producer; 7 | import org.apache.kafka.clients.producer.ProducerRecord; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.util.Calendar; 12 | import java.util.Properties; 13 | 14 | import static org.kafkainaction.avro.AlertStatus.Critical; 15 | 16 | public class AlertProducer { 17 | 18 | final static Logger log = LoggerFactory.getLogger(AlertProducer.class); 19 | 20 | public static void main(String[] args) { 21 | 22 | Properties kaProperties = new Properties(); 23 | kaProperties.put("bootstrap.servers", "localhost:9092"); 24 | kaProperties.put("key.serializer", "org.apache.kafka.common.serialization.LongSerializer"); 25 | kaProperties.put("value.serializer", "io.confluent.kafka.serializers.KafkaAvroSerializer"); // <1> 26 | kaProperties.put("schema.registry.url", "http://localhost:8081"); // <2> 27 | 28 | try (Producer producer = new KafkaProducer<>(kaProperties)) { 29 | Alert alert = new Alert(); //<3> 30 | alert.setSensorId(12345L); 31 | alert.setTime(Calendar.getInstance().getTimeInMillis()); 32 | alert.setStatus(Critical); 33 | /* Uncomment the following line if alert_v2.avsc is the latest Alert model */ 34 | // alert.setRecoveryDetails("RecoveryDetails"); 35 | log.info(alert.toString()); 36 | 37 | ProducerRecord producerRecord = new ProducerRecord<>("kinaction_schematest", alert.getSensorId(), alert); // <4> 38 | 39 | producer.send(producerRecord); 40 | } 41 | 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter11/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %yellow(%d{yyyy-MM-dd HH:mm:ss}) %highlight([%p]) %magenta((%file:%line\)) - %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter12/Commands.adoc: -------------------------------------------------------------------------------- 1 | = Commands used in Chapter 12 2 | 3 | == start or components of the Confluent Platform 4 | 5 | docker-compose up -d 6 | 7 | == Create an interactive session with ksqlDB 8 | 9 | docker exec -it ksqldb-cli ksql http://ksqldb-server:8088 10 | 11 | == Start ksqlDB in headless mode 12 | 13 | bin/ksql-server-start.sh etc/ksql/ksql-server.properties \ 14 | --queries-file Ksql.sql 15 | 16 | == running a new version of Streams app 17 | 18 | . start Confluent Platform 19 | + 20 | 21 | `confluent local services ksql-server start` 22 | . Build and run an app 23 | + 24 | 25 | `./gradlew run` 26 | 27 | . produce some data using `kafka-avro-console-producer` 28 | + 29 | 30 | [source,shell script] 31 | .test-data 32 | ---- 33 | kafka-avro-console-producer --topic transaction-request --bootstrap-server localhost:9092 --property value.schema="$(< src/main/avro/transaction.avsc)" 34 | 35 | {"guid":"220acb8b-9353-4f63-950b-02e030902b2a","account":"1","amount":"'\u0010","type":"DEPOSIT","currency":"CAD","country":"CA"} 36 | {"guid":"987ec7c5-878c-438a-a33b-d48c142479dd","account":"1","amount":"N ","type":"DEPOSIT","currency":"CAD","country":"CA"} 37 | {"guid":"a7b99a64-851f-4886-8505-b99d2f74851c","account":"1","amount":"u0","type":"DEPOSIT","currency":"CAD","country":"CA"} 38 | {"guid":"0625d34b-e523-41f6-b015-4a3994684cb1","account":"1","amount":"u0","type":"WITHDRAW","currency":"CAD","country":"CA"} 39 | {"guid":"af92438d-ce6e-4c90-b957-75846f83183a","account":"1","amount":"\u0001† ","type":"WITHDRAW","currency":"CAD","country":"CA"} 40 | {"guid":"a2124811-de5c-4dca-8a85-912d098ea72f","account":"2","amount":"'\u0010","type":"DEPOSIT","currency":"USD","country":"USA"} 41 | {"guid":"78fcb5f2-64f7-4c15-8891-29caddc9bb90","account":"2","amount":"\u0013ˆ","type":"DEPOSIT","currency":"USD","country":"USA"} 42 | {"guid":"7bb790ec-96b9-47f6-b2b2-ebf36cb41da7","account":"2","amount":"u0","type":"DEPOSIT","currency":"USD","country":"USA"} 43 | ---- 44 | 45 | . consume result from 46 | + 47 | 48 | [source,shell script] 49 | .consume 50 | ---- 51 | kafka-avro-console-consumer --bootstrap-server localhost:9092 --from-beginning --topic transaction-failed --property schema.registry.url=http://localhost:8081 #<1> 52 | 53 | {"transaction":{"guid":"af92438d-ce6e-4c90-b957-75846f83183a","account":"1","amount":"\u0001† ","type":"WITHDRAW","currency":"CAD","country":"CA"},"funds":{"account":"1","balance":"u0"},"success":false,"errorType":{"org.kafkainaction.ErrorType":"INSUFFICIENT_FUNDS"}} 54 | 55 | kafka-avro-console-consumer --bootstrap-server localhost:9092 --from-beginning --topic transaction-success --property schema.registry.url=http://localhost:8081 #<2> 56 | 57 | {"transaction":{"guid":"220acb8b-9353-4f63-950b-02e030902b2a","account":"1","amount":"'\u0010","type":"DEPOSIT","currency":"CAD","country":"CA"},"funds":{"account":"1","balance":"'\u0010"},"success":true,"errorType":null} 58 | {"transaction":{"guid":"987ec7c5-878c-438a-a33b-d48c142479dd","account":"1","amount":"N ","type":"DEPOSIT","currency":"CAD","country":"CA"},"funds":{"account":"1","balance":"u0"},"success":true,"errorType":null} 59 | {"transaction":{"guid":"a7b99a64-851f-4886-8505-b99d2f74851c","account":"1","amount":"u0","type":"DEPOSIT","currency":"CAD","country":"CA"},"funds":{"account":"1","balance":"\u0000ê`"},"success":true,"errorType":null} 60 | {"transaction":{"guid":"0625d34b-e523-41f6-b015-4a3994684cb1","account":"1","amount":"u0","type":"WITHDRAW","currency":"CAD","country":"CA"},"funds":{"account":"1","balance":"u0"},"success":true,"errorType":null} 61 | {"transaction":{"guid":"a2124811-de5c-4dca-8a85-912d098ea72f","account":"2","amount":"'\u0010","type":"DEPOSIT","currency":"USD","country":"USA"},"funds":{"account":"2","balance":"'\u0010"},"success":true,"errorType":null} 62 | {"transaction":{"guid":"78fcb5f2-64f7-4c15-8891-29caddc9bb90","account":"2","amount":"\u0013ˆ","type":"DEPOSIT","currency":"USD","country":"USA"},"funds":{"account":"2","balance":":˜"},"success":true,"errorType":null} 63 | {"transaction":{"guid":"7bb790ec-96b9-47f6-b2b2-ebf36cb41da7","account":"2","amount":"u0","type":"DEPOSIT","currency":"USD","country":"USA"},"funds":{"account":"2","balance":"\u0000¯È"},"success":true,"errorType":null} 64 | ---- 65 | <1> Failed transactions 66 | <2> Successful transactions 67 | 68 | [source,sql] 69 | .account.sql 70 | ---- 71 | CREATE TABLE ACCOUNT ( 72 | numkey string PRIMARY KEY, 73 | number INT, 74 | firstName STRING, 75 | lastName STRING, 76 | numberAddress STRING, 77 | streetAddress STRING, 78 | cityAddress STRING, 79 | countryAddress STRING, 80 | creationDate BIGINT, 81 | updateDate BIGINT 82 | ) WITH (KAFKA_TOPIC = 'account', VALUE_FORMAT='avro', PARTITIONS=6, REPLICAS=1); 83 | ---- 84 | 85 | CREATE TABLE ACCOUNT (number INT PRIMARY KEY) WITH (KAFKA_TOPIC = 'account', VALUE_FORMAT='avro', PARTITIONS=6, REPLICAS=1); 86 | 87 | [source,sql] 88 | ---- 89 | CREATE STREAM TRANSACTION_SUCCESS ( 90 | numkey string KEY, 91 | transaction STRUCT, 92 | funds STRUCT, 93 | success boolean, 94 | errorType STRING 95 | --errorType STRUCT 96 | ) WITH (KAFKA_TOPIC='transaction-success', VALUE_FORMAT='avro'); 97 | ---- 98 | 99 | [source,sql] 100 | ---- 101 | CREATE STREAM TRANSACTION_STATEMENT AS 102 | SELECT * 103 | FROM TRANSACTION_SUCCESS 104 | LEFT JOIN ACCOUNT ON TRANSACTION_SUCCESS.numkey = ACCOUNT.numkey 105 | EMIT CHANGES; 106 | ---- 107 | 108 | 109 | # More examples link 110 | KGlobalTable and Processor API examples are at: 111 | https://github.com/Kafka-In-Action-Book/Kafka-In-Action-Source-Code/tree/master/KafkaInAction_Chapter12 112 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter12/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | Kafka-In-Action 6 | org.kafkainaction 7 | 1.0.0-SNAPSHOT 8 | 9 | 10 | chapter12 11 | Stream processing with Kafka Streams and ksqlDB 12 | 13 | 14 | 15 | org.apache.kafka 16 | kafka-streams 17 | ${kafka.version} 18 | 19 | 20 | org.apache.commons 21 | commons-text 22 | 1.10.0 23 | 24 | 25 | org.apache.avro 26 | avro 27 | ${avro.version} 28 | 29 | 30 | io.confluent 31 | kafka-avro-serializer 32 | ${confluent.version} 33 | 34 | 35 | io.confluent 36 | kafka-streams-avro-serde 37 | ${confluent.version} 38 | 39 | 40 | com.github.javafaker 41 | javafaker 42 | 1.0.2 43 | test 44 | 45 | 46 | org.apache.kafka 47 | kafka-streams-test-utils 48 | ${kafka.version} 49 | test 50 | 51 | 52 | junit 53 | junit 54 | 4.13.2 55 | test 56 | 57 | 58 | org.assertj 59 | assertj-core 60 | 3.20.2 61 | test 62 | 63 | 64 | 65 | 66 | chapter12 67 | 68 | 69 | maven-assembly-plugin 70 | 71 | 72 | org.apache.avro 73 | avro-maven-plugin 74 | ${avro.version} 75 | 76 | String 77 | true 78 | 79 | ${project.basedir}/src/main/avro/transaction.avsc 80 | ${project.basedir}/src/main/avro/funds.avsc 81 | ${project.basedir}/src/main/avro/transaction_result.avsc 82 | 83 | 84 | *.avsc 85 | 86 | 87 | 88 | 89 | generate-sources 90 | 91 | schema 92 | 93 | 94 | ${project.basedir}/src/main/avro/ 95 | ${project.build.directory}/generated-sources/avro 96 | 97 | 98 | 99 | 100 | 101 | 102 | org.codehaus.mojo 103 | build-helper-maven-plugin 104 | 3.5.0 105 | 106 | 107 | add-source 108 | generate-sources 109 | 110 | add-source 111 | 112 | 113 | 114 | ${project.build.directory}/generated-sources/avro 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter12/src/main/avro/account.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "org.kafkainaction", 3 | "type": "record", 4 | "name": "Account", 5 | "fields": [ 6 | { 7 | "name": "number", 8 | "type": "int", 9 | "doc": "doc" 10 | }, 11 | { 12 | "name": "firstName", 13 | "type": "string", 14 | "doc": "doc" 15 | }, 16 | { 17 | "name": "lastName", 18 | "type": "string", 19 | "doc": "doc" 20 | }, 21 | { 22 | "name": "streetAddress", 23 | "type": "string", 24 | "doc": "doc" 25 | }, 26 | { 27 | "name": "numberAddress", 28 | "type": "string", 29 | "doc": "doc" 30 | }, 31 | { 32 | "name": "cityAddress", 33 | "type": "string", 34 | "doc": "City" 35 | }, 36 | { 37 | "name": "countryAddress", 38 | "type": "string", 39 | "doc": "Country" 40 | }, 41 | { 42 | "name": "creationDate", 43 | "type": "long", 44 | "doc": "account creation date" 45 | }, 46 | { 47 | "name": "updateDate", 48 | "type": "long", 49 | "doc": "account creation date" 50 | } 51 | 52 | ] 53 | } -------------------------------------------------------------------------------- /KafkaInAction_Chapter12/src/main/avro/funds.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "org.kafkainaction", 3 | "type": "record", 4 | "name": "Funds", 5 | "fields": [ 6 | { 7 | "name": "account", 8 | "type": "string", 9 | "doc": "Account name" 10 | }, 11 | { 12 | "name": "balance", 13 | "type": { 14 | "type": "bytes", 15 | "logicalType": "decimal", 16 | "precision": 9, 17 | "scale": 2 18 | } 19 | } 20 | ] 21 | } -------------------------------------------------------------------------------- /KafkaInAction_Chapter12/src/main/avro/transaction.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "org.kafkainaction", 3 | "type": "record", 4 | "name": "Transaction", 5 | "fields": [ 6 | { 7 | "name": "guid", 8 | "type": "string", 9 | "doc": "The unique transaction guid" 10 | }, 11 | { 12 | "name": "account", 13 | "type": "string", 14 | "avro.java.string": "java.lang.String", 15 | "doc": "Account name" 16 | }, 17 | { 18 | "name": "amount", 19 | "type": { 20 | "type": "bytes", 21 | "logicalType": "decimal", 22 | "precision": 9, 23 | "scale": 2 24 | } 25 | }, 26 | { 27 | "name": "type", 28 | "type": { 29 | "type": "enum", 30 | "name": "TransactionType", 31 | "symbols": [ 32 | "DEPOSIT", 33 | "WITHDRAW" 34 | ] 35 | } 36 | }, 37 | { 38 | "name": "currency", 39 | "type": "string", 40 | "doc": "Transaction currency" 41 | }, 42 | { 43 | "name": "country", 44 | "type": "string", 45 | "doc": "Transaction country" 46 | } 47 | ] 48 | } -------------------------------------------------------------------------------- /KafkaInAction_Chapter12/src/main/avro/transaction_result.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "org.kafkainaction", 3 | "type": "record", 4 | "name": "TransactionResult", 5 | "includeSchemas": [ 6 | { 7 | "name": "Transaction" 8 | }, 9 | { 10 | "name": "Funds" 11 | } 12 | ], 13 | "fields": [ 14 | { 15 | "name": "transaction", 16 | "type": "Transaction" 17 | }, 18 | { 19 | "name": "funds", 20 | "type": "Funds" 21 | }, 22 | { 23 | "name": "success", 24 | "type": "boolean", 25 | "doc": "success" 26 | }, 27 | { 28 | "name": "errorType", 29 | "type": [ 30 | "null", 31 | { 32 | "type": "enum", 33 | "name": "ErrorType", 34 | "symbols": [ 35 | "INSUFFICIENT_FUNDS" 36 | ] 37 | } 38 | ] 39 | } 40 | ] 41 | } -------------------------------------------------------------------------------- /KafkaInAction_Chapter12/src/main/java/org/kafkainaction/kstreams2/SchemaSerdes.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.kstreams2; 2 | 3 | import org.apache.avro.specific.SpecificRecord; 4 | 5 | import java.util.Map; 6 | import java.util.Optional; 7 | import java.util.Properties; 8 | 9 | import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde; 10 | 11 | import static io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG; 12 | 13 | public class SchemaSerdes { 14 | 15 | static SpecificAvroSerde getSpecificAvroSerde(final Properties envProps) { 16 | final Map 17 | serdeConfig = 18 | Map.of(SCHEMA_REGISTRY_URL_CONFIG, 19 | Optional.ofNullable(envProps.getProperty("schema.registry.url")).orElse("") 20 | ); 21 | final SpecificAvroSerde specificAvroSerde = new SpecificAvroSerde<>(); 22 | 23 | specificAvroSerde.configure(serdeConfig, false); 24 | return specificAvroSerde; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter12/src/main/java/org/kafkainaction/kstreams2/TransactionTransformer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.kstreams2; 2 | 3 | import org.apache.kafka.streams.processor.api.Processor; 4 | import org.apache.kafka.streams.processor.api.ProcessorContext; 5 | import org.apache.kafka.streams.processor.api.Record; 6 | import org.apache.kafka.streams.state.KeyValueStore; 7 | import org.kafkainaction.Funds; 8 | import org.kafkainaction.Transaction; 9 | import org.kafkainaction.TransactionResult; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import java.math.BigDecimal; 14 | import java.util.Optional; 15 | 16 | import static org.kafkainaction.ErrorType.INSUFFICIENT_FUNDS; 17 | import static org.kafkainaction.TransactionType.DEPOSIT; 18 | 19 | public class TransactionTransformer implements Processor { 20 | 21 | private static final Logger log = LoggerFactory.getLogger(TransactionTransformer.class); 22 | 23 | private final String stateStoreName; 24 | private KeyValueStore store; 25 | private ProcessorContext context; 26 | 27 | public TransactionTransformer() { 28 | // default name for funds store 29 | this.stateStoreName = "fundsStore"; 30 | } 31 | 32 | public TransactionTransformer(final String stateStoreName) { 33 | this.stateStoreName = stateStoreName; 34 | } 35 | 36 | @Override 37 | public void close() { 38 | } 39 | 40 | private Funds createEmptyFunds(String account) { 41 | Funds funds = new Funds(account, BigDecimal.ZERO); 42 | store.put(account, funds); 43 | return funds; 44 | } 45 | 46 | private Funds depositFunds(Transaction transaction) { 47 | return updateFunds(transaction.getAccount(), transaction.getAmount()); 48 | } 49 | 50 | private Funds getFunds(String account) { 51 | return Optional.ofNullable(store.get(account)).orElseGet(() -> createEmptyFunds(account)); 52 | } 53 | 54 | private boolean hasEnoughFunds(Transaction transaction) { 55 | return getFunds(transaction.getAccount()).getBalance().compareTo(transaction.getAmount()) != -1; 56 | } 57 | 58 | @Override 59 | public void init(ProcessorContext context) { 60 | this.context = context; 61 | this.store = context.getStateStore(stateStoreName); 62 | } 63 | 64 | @Override 65 | public void process(Record record) { 66 | String key = record.key(); 67 | Transaction transaction = record.value(); 68 | TransactionResult result; 69 | 70 | if (transaction.getType().equals(DEPOSIT)) { 71 | result = new TransactionResult(transaction, 72 | depositFunds(transaction), 73 | true, 74 | null); 75 | } else if (hasEnoughFunds(transaction)) { 76 | result = new TransactionResult(transaction, withdrawFunds(transaction), true, null); 77 | } else { 78 | log.info("Not enough funds for account {}.", transaction.getAccount()); 79 | result = new TransactionResult(transaction, 80 | getFunds(transaction.getAccount()), 81 | false, 82 | INSUFFICIENT_FUNDS); 83 | } 84 | 85 | context.forward(new Record<>(key, result, record.timestamp())); 86 | } 87 | 88 | private Funds updateFunds(String account, BigDecimal amount) { 89 | Funds funds = new Funds(account, getFunds(account).getBalance().add(amount)); 90 | log.info("Updating funds for account {} with {}. Current balance is {}.", account, amount, funds.getBalance()); 91 | store.put(account, funds); 92 | return funds; 93 | } 94 | 95 | private Funds withdrawFunds(Transaction transaction) { 96 | return updateFunds(transaction.getAccount(), transaction.getAmount().negate()); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter12/src/main/resources/Ksql.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE ACCOUNT 2 | ( 3 | numkey string PRIMARY KEY, 4 | number INT, 5 | firstName STRING, 6 | lastName STRING, 7 | numberAddress STRING, 8 | streetAddress STRING, 9 | cityAddress STRING, 10 | countryAddress STRING, 11 | creationDate BIGINT, 12 | updateDate BIGINT 13 | ) WITH (KAFKA_TOPIC = 'account', VALUE_FORMAT = 'avro', PARTITIONS = 6, REPLICAS = 1); 14 | 15 | CREATE 16 | STREAM TRANSACTION_SUCCESS ( 17 | numkey string KEY, 18 | transaction STRUCT, 19 | funds STRUCT, 20 | success boolean, 21 | errorType STRING 22 | --errorType STRUCT 23 | ) WITH (KAFKA_TOPIC='transaction-success', VALUE_FORMAT='avro'); 24 | 25 | CREATE 26 | STREAM TRANSACTION_STATEMENT AS 27 | SELECT * 28 | FROM TRANSACTION_SUCCESS 29 | LEFT JOIN ACCOUNT ON TRANSACTION_SUCCESS.numkey = ACCOUNT.numkey EMIT CHANGES; 30 | 31 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter12/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %yellow(%d{yyyy-MM-dd HH:mm:ss}) %cyan(%logger{50}) %highlight([%p]) %green((%file:%line\)) - %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | %yellow(%d{yyyy-MM-dd HH:mm:ss}) %highlight([%p]) %magenta((%file:%line\)) - %msg%n 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter12/src/test/java/org/kafkainaction/kstreams2/AccountProducerTest.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.kstreams2; 2 | 3 | import com.github.javafaker.Faker; 4 | 5 | import org.apache.kafka.clients.admin.AdminClient; 6 | import org.apache.kafka.clients.admin.NewTopic; 7 | import org.apache.kafka.clients.producer.KafkaProducer; 8 | import org.apache.kafka.clients.producer.ProducerRecord; 9 | import org.apache.kafka.common.serialization.StringSerializer; 10 | import org.kafkainaction.Account; 11 | 12 | import java.time.LocalDateTime; 13 | import java.time.ZoneOffset; 14 | import java.util.Properties; 15 | import java.util.stream.IntStream; 16 | 17 | import io.confluent.kafka.serializers.KafkaAvroSerializer; 18 | 19 | import static io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG; 20 | import static java.lang.String.valueOf; 21 | import static java.util.Collections.singletonList; 22 | import static org.apache.kafka.clients.producer.ProducerConfig.BOOTSTRAP_SERVERS_CONFIG; 23 | import static org.apache.kafka.clients.producer.ProducerConfig.CLIENT_ID_CONFIG; 24 | import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; 25 | import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; 26 | 27 | public class AccountProducerTest { 28 | 29 | public static final String ACCOUNT_TOPIC_NAME = "account"; 30 | 31 | public static void main(String[] args) { 32 | 33 | var p = new Properties(); 34 | p.put(BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); 35 | p.put(SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081"); 36 | p.put(CLIENT_ID_CONFIG, "account-producer"); 37 | p.put(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); 38 | p.put(VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class); 39 | 40 | try (var client = AdminClient.create(p)) { 41 | var txRequest = new NewTopic(ACCOUNT_TOPIC_NAME, 6, (short) 1); 42 | client.createTopics(singletonList(txRequest)); 43 | } 44 | 45 | try (var producer = new KafkaProducer(p)) { 46 | 47 | final var faker = Faker.instance(); 48 | IntStream.range(1, 10).forEach(index -> { 49 | final var account = new Account(index, faker.name().firstName(), faker.name().lastName(), 50 | faker.address().streetName(), faker.address().buildingNumber(), 51 | faker.address().city(), 52 | faker.address().country(), 53 | LocalDateTime.now().toEpochSecond(ZoneOffset.UTC), 54 | LocalDateTime.now().toEpochSecond(ZoneOffset.UTC)); 55 | producer.send(new ProducerRecord<>(ACCOUNT_TOPIC_NAME, valueOf(account.getNumber()), account)); 56 | }); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter12/src/test/java/org/kafkainaction/kstreams2/TransactionProcessorTest.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.kstreams2; 2 | 3 | 4 | import org.apache.kafka.common.serialization.Serde; 5 | import org.apache.kafka.common.serialization.Serdes; 6 | import org.apache.kafka.streams.StreamsBuilder; 7 | import org.apache.kafka.streams.TestInputTopic; 8 | import org.apache.kafka.streams.TestOutputTopic; 9 | import org.apache.kafka.streams.Topology; 10 | import org.apache.kafka.streams.TopologyTestDriver; 11 | import org.apache.kafka.streams.state.KeyValueStore; 12 | import org.junit.Before; 13 | import org.junit.Test; 14 | import org.kafkainaction.ErrorType; 15 | import org.kafkainaction.Funds; 16 | import org.kafkainaction.Transaction; 17 | import org.kafkainaction.TransactionResult; 18 | 19 | import java.io.IOException; 20 | import java.math.BigDecimal; 21 | import java.util.List; 22 | import java.util.Map; 23 | import java.util.Properties; 24 | import java.util.UUID; 25 | 26 | import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde; 27 | 28 | import static io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG; 29 | import static java.nio.file.Files.createTempDirectory; 30 | import static org.apache.kafka.streams.StreamsConfig.STATE_DIR_CONFIG; 31 | import static org.assertj.core.api.Assertions.assertThat; 32 | import static org.kafkainaction.TransactionType.DEPOSIT; 33 | import static org.kafkainaction.TransactionType.WITHDRAW; 34 | 35 | public class TransactionProcessorTest { 36 | 37 | private Serde stringSerde; 38 | private Transaction deposit100; 39 | private Transaction withdraw100; 40 | private Transaction withdraw200; 41 | 42 | private Properties properties; 43 | private Topology topology; 44 | private SpecificAvroSerde transactionResultSerde; 45 | private SpecificAvroSerde transactionSerde; 46 | private SpecificAvroSerde fundsSerde; 47 | 48 | private static String transactionsInputTopicName = "transaction-request"; 49 | private static String transactionSuccessTopicName = "transaction-success"; 50 | private static String transactionFailedTopicName = "transaction-failed"; 51 | private static String fundsStoreName = "funds-store"; 52 | 53 | public TransactionProcessorTest() throws IOException { 54 | } 55 | 56 | @Before 57 | public void setUp() throws IOException { 58 | 59 | final StreamsBuilder streamsBuilder = new StreamsBuilder(); 60 | properties = new Properties(); 61 | properties.putAll( 62 | Map.of( 63 | SCHEMA_REGISTRY_URL_CONFIG, "mock://schema-registry.kafkainaction.org:8080", 64 | // workaround https://stackoverflow.com/a/50933452/27563 65 | STATE_DIR_CONFIG, createTempDirectory("kafka-streams").toAbsolutePath().toString() 66 | )); 67 | 68 | // serdes 69 | stringSerde = Serdes.String(); 70 | transactionResultSerde = SchemaSerdes.getSpecificAvroSerde(properties); 71 | transactionSerde = SchemaSerdes.getSpecificAvroSerde(properties); 72 | fundsSerde = SchemaSerdes.getSpecificAvroSerde(properties); 73 | 74 | topology = new TransactionProcessor(transactionsInputTopicName, 75 | transactionSuccessTopicName, 76 | transactionFailedTopicName, 77 | fundsStoreName) 78 | .topology(streamsBuilder, 79 | transactionSerde, 80 | transactionResultSerde, 81 | fundsSerde); 82 | 83 | deposit100 = new Transaction(UUID.randomUUID().toString(), 84 | "1", new BigDecimal(100), DEPOSIT, "USD", "USA"); 85 | 86 | withdraw100 = new Transaction(UUID.randomUUID().toString(), 87 | "1", new BigDecimal(100), WITHDRAW, "USD", "USA"); 88 | 89 | withdraw200 = new Transaction(UUID.randomUUID().toString(), 90 | "1", new BigDecimal(200), WITHDRAW, "USD", "USA"); 91 | } 92 | 93 | @Test 94 | public void testDriverShouldNotBeNull() { 95 | try (TopologyTestDriver testDriver = new TopologyTestDriver(topology, properties)) { 96 | assertThat(testDriver).isNotNull(); 97 | } 98 | } 99 | 100 | @Test 101 | public void shouldCreateSuccessfulTransaction() { 102 | 103 | try (TopologyTestDriver testDriver = new TopologyTestDriver(topology, properties)) { 104 | 105 | final TestInputTopic inputTopic = testDriver 106 | .createInputTopic(transactionsInputTopicName, 107 | stringSerde.serializer(), 108 | transactionSerde.serializer()); 109 | 110 | inputTopic.pipeInput(deposit100.getAccount(), deposit100); 111 | inputTopic.pipeInput(withdraw100.getAccount(), withdraw100); 112 | 113 | final TestOutputTopic 114 | outputTopic = 115 | testDriver.createOutputTopic(transactionSuccessTopicName, 116 | stringSerde.deserializer(), 117 | transactionResultSerde.deserializer()); 118 | 119 | final List successfulTransactions = outputTopic.readValuesToList(); 120 | // balance should be 0 121 | final TransactionResult transactionResult = successfulTransactions.get(1); 122 | 123 | assertThat(transactionResult.getFunds().getBalance()).isEqualByComparingTo(new BigDecimal(0)); 124 | } 125 | } 126 | 127 | @Test 128 | public void shouldBeInsufficientFunds() { 129 | 130 | try (TopologyTestDriver testDriver = new TopologyTestDriver(topology, properties)) { 131 | final TestInputTopic inputTopic = testDriver 132 | .createInputTopic(transactionsInputTopicName, 133 | stringSerde.serializer(), 134 | transactionSerde.serializer()); 135 | 136 | inputTopic.pipeInput(deposit100.getAccount(), deposit100); 137 | inputTopic.pipeInput(withdraw200.getAccount(), withdraw200); 138 | 139 | final TestOutputTopic 140 | failedResultOutputTopic = 141 | testDriver.createOutputTopic(transactionFailedTopicName, 142 | stringSerde.deserializer(), 143 | transactionResultSerde.deserializer()); 144 | 145 | final TestOutputTopic 146 | successResultOutputTopic = 147 | testDriver.createOutputTopic(transactionSuccessTopicName, 148 | stringSerde.deserializer(), 149 | transactionResultSerde.deserializer()); 150 | 151 | final TransactionResult successfulDeposit100Result = successResultOutputTopic.readValuesToList().get(0); 152 | 153 | assertThat(successfulDeposit100Result.getFunds().getBalance()).isEqualByComparingTo(new BigDecimal(100)); 154 | 155 | final List failedTransactions = failedResultOutputTopic.readValuesToList(); 156 | // balance should be 0 157 | final TransactionResult transactionResult = failedTransactions.get(0); 158 | assertThat(transactionResult.getErrorType()).isEqualTo(ErrorType.INSUFFICIENT_FUNDS); 159 | } 160 | } 161 | 162 | @Test 163 | public void balanceShouldBe300() { 164 | try (TopologyTestDriver testDriver = new TopologyTestDriver(topology, properties)) { 165 | final TestInputTopic inputTopic = testDriver 166 | .createInputTopic(transactionsInputTopicName, 167 | stringSerde.serializer(), 168 | transactionSerde.serializer()); 169 | 170 | inputTopic.pipeInput(deposit100.getAccount(), deposit100); 171 | inputTopic.pipeInput(deposit100.getAccount(), deposit100); 172 | inputTopic.pipeInput(deposit100.getAccount(), deposit100); 173 | 174 | final KeyValueStore store = testDriver.getKeyValueStore(fundsStoreName); 175 | 176 | assertThat(store.get("1").getBalance()).isEqualByComparingTo(new BigDecimal(300)); 177 | } 178 | } 179 | } -------------------------------------------------------------------------------- /KafkaInAction_Chapter12/src/test/java/org/kafkainaction/kstreams2/TransactionProducerTest.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.kstreams2; 2 | 3 | import org.apache.kafka.clients.admin.AdminClient; 4 | import org.apache.kafka.clients.admin.NewTopic; 5 | import org.apache.kafka.clients.producer.KafkaProducer; 6 | import org.apache.kafka.clients.producer.ProducerRecord; 7 | import org.apache.kafka.common.serialization.StringSerializer; 8 | import org.kafkainaction.Transaction; 9 | 10 | import java.math.BigDecimal; 11 | import java.util.Properties; 12 | import java.util.UUID; 13 | import java.util.stream.Stream; 14 | 15 | import io.confluent.kafka.serializers.KafkaAvroSerializer; 16 | 17 | import static io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG; 18 | import static java.lang.String.valueOf; 19 | import static java.util.Collections.singletonList; 20 | import static org.apache.kafka.clients.producer.ProducerConfig.BOOTSTRAP_SERVERS_CONFIG; 21 | import static org.apache.kafka.clients.producer.ProducerConfig.CLIENT_ID_CONFIG; 22 | import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; 23 | import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; 24 | import static org.kafkainaction.TransactionType.DEPOSIT; 25 | import static org.kafkainaction.TransactionType.WITHDRAW; 26 | 27 | public class TransactionProducerTest { 28 | 29 | public static void main(String[] args) { 30 | 31 | var p = new Properties(); 32 | p.put(BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); 33 | p.put(SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081"); 34 | p.put(CLIENT_ID_CONFIG, "transactions-producer"); 35 | p.put(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); 36 | p.put(VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class); 37 | 38 | try (var client = AdminClient.create(p)) { 39 | var txRequest = new NewTopic("transaction-request", 6, (short) 1); 40 | client.createTopics(singletonList(txRequest)); 41 | } 42 | 43 | try (var producer = new KafkaProducer(p)) { 44 | 45 | final var tx1 = 46 | new Transaction(UUID.randomUUID().toString(), valueOf(1), new BigDecimal(100), DEPOSIT, "CAD", 47 | "CA"); 48 | final var tx2 = 49 | new Transaction(UUID.randomUUID().toString(), valueOf(1), new BigDecimal(200), DEPOSIT, "CAD", 50 | "CA"); 51 | final var tx3 = 52 | new Transaction(UUID.randomUUID().toString(), valueOf(1), new BigDecimal(300), DEPOSIT, "CAD", 53 | "CA"); 54 | final var tx4 = 55 | new Transaction(UUID.randomUUID().toString(), valueOf(1), new BigDecimal(300), WITHDRAW, 56 | "CAD", "CA"); 57 | final var tx5 = 58 | new Transaction(UUID.randomUUID().toString(), valueOf(1), new BigDecimal(1000), WITHDRAW, 59 | "CAD", "CA"); 60 | 61 | final var tx6 = 62 | new Transaction(UUID.randomUUID().toString(), valueOf(2), new BigDecimal(100), DEPOSIT, 63 | "USD", "USA"); 64 | final var tx7 = 65 | new Transaction(UUID.randomUUID().toString(), valueOf(2), new BigDecimal(50), DEPOSIT, 66 | "USD", "USA"); 67 | final var tx8 = 68 | new Transaction(UUID.randomUUID().toString(), valueOf(2), new BigDecimal(300), DEPOSIT, 69 | "USD", "USA"); 70 | final var tx9 = 71 | new Transaction(UUID.randomUUID().toString(), valueOf(2), new BigDecimal(300), WITHDRAW, 72 | "USD", "USA"); 73 | 74 | Stream.of(tx1, tx2, tx3, tx4, tx5, tx6, tx7, tx8, tx9) 75 | .forEach(tx -> producer.send(new ProducerRecord<>("transaction-request", tx.getAccount(), tx))); 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter12/src/test/java/org/kafkainaction/kstreams2/TransactionTransformerExtendedTest.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.kstreams2; 2 | 3 | import org.apache.kafka.common.serialization.Serde; 4 | import org.apache.kafka.common.serialization.Serdes; 5 | import org.apache.kafka.streams.processor.api.MockProcessorContext; 6 | import org.apache.kafka.streams.processor.api.Record; 7 | import org.apache.kafka.streams.state.KeyValueStore; 8 | import org.apache.kafka.streams.state.Stores; 9 | import org.junit.Before; 10 | import org.junit.Test; 11 | import org.kafkainaction.Funds; 12 | import org.kafkainaction.Transaction; 13 | import org.kafkainaction.TransactionResult; 14 | 15 | import java.math.BigDecimal; 16 | import java.util.Map; 17 | import java.util.Properties; 18 | import java.util.UUID; 19 | 20 | import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde; 21 | 22 | import static io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG; 23 | import static org.apache.kafka.streams.StreamsConfig.APPLICATION_ID_CONFIG; 24 | import static org.apache.kafka.streams.StreamsConfig.BOOTSTRAP_SERVERS_CONFIG; 25 | import static org.assertj.core.api.Assertions.assertThat; 26 | import static org.kafkainaction.TransactionType.DEPOSIT; 27 | import static org.kafkainaction.TransactionType.WITHDRAW; 28 | 29 | /** 30 | * Extended tests for TransactionTransformer to improve test coverage. 31 | */ 32 | public class TransactionTransformerExtendedTest { 33 | 34 | private KeyValueStore fundsStore; 35 | private MockProcessorContext mockContext; 36 | private TransactionTransformer transactionTransformer; 37 | 38 | final static Map testConfig = Map.of( 39 | BOOTSTRAP_SERVERS_CONFIG, "localhost:8080", 40 | APPLICATION_ID_CONFIG, "mytest", 41 | SCHEMA_REGISTRY_URL_CONFIG, "mock://schema-registry.kafkainaction.org:8080" 42 | ); 43 | 44 | @Before 45 | public void setup() { 46 | final Properties properties = new Properties(); 47 | properties.putAll(testConfig); 48 | 49 | mockContext = new MockProcessorContext<>(properties); 50 | 51 | final SpecificAvroSerde 52 | fundsSpecificAvroSerde = 53 | SchemaSerdes.getSpecificAvroSerde(properties); 54 | 55 | final Serde stringSerde = Serdes.String(); 56 | final String fundsStoreName = "fundsStore"; 57 | this.fundsStore = Stores.keyValueStoreBuilder( 58 | Stores.inMemoryKeyValueStore(fundsStoreName), 59 | stringSerde, 60 | fundsSpecificAvroSerde) 61 | .withLoggingDisabled() // Changelog is not supported by MockProcessorContext. 62 | .build(); 63 | 64 | fundsStore.init(mockContext.getStateStoreContext(), fundsStore); 65 | mockContext.addStateStore(fundsStore); 66 | 67 | // Use the default constructor to test it 68 | transactionTransformer = new TransactionTransformer(); 69 | transactionTransformer.init(mockContext); 70 | } 71 | 72 | @Test 73 | public void shouldUseDefaultConstructor() { 74 | // The test setup uses the default constructor, so if we get here without errors, it works 75 | assertThat(transactionTransformer).isNotNull(); 76 | } 77 | 78 | @Test 79 | public void shouldProcessDeposit() { 80 | // Create a deposit transaction 81 | final Transaction deposit = 82 | new Transaction(UUID.randomUUID().toString(), "2", new BigDecimal("100"), DEPOSIT, "USD", "USA"); 83 | 84 | // Process the transaction 85 | transactionTransformer.process(new Record<>("tx1", deposit, 0L)); 86 | 87 | // If we get here without an exception, the test passes 88 | assertThat(true).isTrue(); 89 | } 90 | 91 | @Test 92 | public void shouldProcessWithdrawal() { 93 | // First deposit to have funds 94 | final Transaction deposit = 95 | new Transaction(UUID.randomUUID().toString(), "3", new BigDecimal("1000"), DEPOSIT, "USD", "USA"); 96 | 97 | // Then make a withdrawal 98 | final Transaction withdraw = 99 | new Transaction(UUID.randomUUID().toString(), "3", new BigDecimal("200"), WITHDRAW, "USD", "USA"); 100 | 101 | // Process the transactions 102 | transactionTransformer.process(new Record<>("deposit", deposit, 0L)); 103 | transactionTransformer.process(new Record<>("withdraw", withdraw, 0L)); 104 | 105 | // If we get here without an exception, the test passes 106 | assertThat(true).isTrue(); 107 | } 108 | 109 | @Test 110 | public void shouldCloseWithoutErrors() { 111 | // The close method is empty, but we should test it for coverage 112 | transactionTransformer.close(); 113 | // If we get here without errors, the test passes 114 | assertThat(true).isTrue(); 115 | } 116 | } -------------------------------------------------------------------------------- /KafkaInAction_Chapter12/src/test/java/org/kafkainaction/kstreams2/TransactionTransformerTest.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.kstreams2; 2 | 3 | import org.apache.kafka.common.serialization.Serde; 4 | import org.apache.kafka.common.serialization.Serdes; 5 | import org.apache.kafka.streams.processor.api.MockProcessorContext; 6 | import org.apache.kafka.streams.processor.api.Record; 7 | import org.apache.kafka.streams.state.KeyValueStore; 8 | import org.apache.kafka.streams.state.Stores; 9 | import org.junit.Before; 10 | import org.junit.Test; 11 | import org.kafkainaction.Funds; 12 | import org.kafkainaction.Transaction; 13 | import org.kafkainaction.TransactionResult; 14 | 15 | import java.math.BigDecimal; 16 | import java.util.Map; 17 | import java.util.Properties; 18 | import java.util.UUID; 19 | 20 | import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde; 21 | 22 | import static io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG; 23 | import static org.apache.kafka.streams.StreamsConfig.APPLICATION_ID_CONFIG; 24 | import static org.apache.kafka.streams.StreamsConfig.BOOTSTRAP_SERVERS_CONFIG; 25 | import static org.assertj.core.api.Assertions.assertThat; 26 | import static org.kafkainaction.TransactionType.DEPOSIT; 27 | 28 | 29 | public class TransactionTransformerTest { 30 | 31 | private KeyValueStore fundsStore; 32 | private MockProcessorContext mockContext; 33 | private TransactionTransformer transactionTransformer; 34 | 35 | final static Map testConfig = Map.of( 36 | BOOTSTRAP_SERVERS_CONFIG, "localhost:8080", 37 | APPLICATION_ID_CONFIG, "mytest", 38 | SCHEMA_REGISTRY_URL_CONFIG, "mock://schema-registry.kafkainaction.org:8080" 39 | ); 40 | 41 | @Before 42 | public void setup() { 43 | final Properties properties = new Properties(); 44 | properties.putAll(testConfig); 45 | mockContext = new MockProcessorContext<>(properties); 46 | 47 | final SpecificAvroSerde 48 | fundsSpecificAvroSerde = 49 | SchemaSerdes.getSpecificAvroSerde(properties); 50 | 51 | final Serde stringSerde = Serdes.String(); 52 | final String fundsStoreName = "fundsStore"; 53 | this.fundsStore = Stores.keyValueStoreBuilder( 54 | Stores.inMemoryKeyValueStore(fundsStoreName), 55 | stringSerde, 56 | fundsSpecificAvroSerde) 57 | .withLoggingDisabled() // Changelog is not supported by MockProcessorContext. 58 | .build(); 59 | 60 | fundsStore.init(mockContext.getStateStoreContext(), fundsStore); 61 | mockContext.addStateStore(fundsStore); 62 | 63 | transactionTransformer = new TransactionTransformer(fundsStoreName); 64 | transactionTransformer.init(mockContext); 65 | } 66 | 67 | @Test 68 | public void shouldInitializeTransformer() { 69 | // Just verify that the transformer can be initialized 70 | assertThat(transactionTransformer).isNotNull(); 71 | } 72 | 73 | @Test 74 | public void shouldProcessTransaction() { 75 | // Create a transaction 76 | final Transaction transaction = 77 | new Transaction(UUID.randomUUID().toString(), "1", new BigDecimal(100), DEPOSIT, "USD", "USA"); 78 | 79 | // Process the transaction (this should not throw an exception) 80 | transactionTransformer.process(new Record<>("key", transaction, 0L)); 81 | 82 | // If we get here without an exception, the test passes 83 | assertThat(true).isTrue(); 84 | } 85 | 86 | @Test 87 | public void shouldCloseWithoutErrors() { 88 | // The close method is empty, but we should test it for coverage 89 | transactionTransformer.close(); 90 | // If we get here without errors, the test passes 91 | assertThat(true).isTrue(); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter2/Commands.md: -------------------------------------------------------------------------------- 1 | # Commands used in Chapter 2 2 | 3 | ## Creating our Topic 4 | 5 | ```shell script 6 | export TOPIC=kinaction_helloworld 7 | ``` 8 | 9 | ```shell script 10 | bin/kafka-topics.sh --create --bootstrap-server localhost:9094 \ 11 | --topic $TOPIC --partitions 3 --replication-factor 3 12 | ``` 13 | 14 | ## Describe the Topic 15 | 16 | ```shell script 17 | bin/kafka-topics.sh --bootstrap-server localhost:9094 --describe --topic $TOPIC 18 | ``` 19 | 20 | ## Kafka Producer Console Command 21 | 22 | ```shell script 23 | bin/kafka-console-producer.sh --bootstrap-server localhost:9094 --topic $TOPIC 24 | ``` 25 | 26 | ## Kafka Consumer Console Command 27 | 28 | ```shell script 29 | bin/kafka-console-consumer.sh --bootstrap-server localhost:9094 \ 30 | --topic $TOPIC --from-beginning 31 | ``` 32 | 33 | ## Java Client POM entry 34 | 35 | ```xml 36 | 37 | org.apache.kafka 38 | kafka-clients 39 | 2.7.1 40 | 41 | ``` 42 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter2/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | Kafka-In-Action 6 | org.kafkainaction 7 | 1.0.0-SNAPSHOT 8 | 9 | 10 | chapter2 11 | Getting To Know Kafka 12 | 13 | 14 | chapter2 15 | 16 | 17 | maven-assembly-plugin 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter2/scripts/portInUse.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | lsof -t -i tcp:2181 | xargs kill -9 4 | lsof -t -i tcp:9092 | xargs kill -9 5 | lsof -t -i tcp:9093 | xargs kill -9 6 | lsof -t -i tcp:9094 | xargs kill -9 -------------------------------------------------------------------------------- /KafkaInAction_Chapter2/scripts/starteverything.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | bin/zookeeper-server-start.sh -daemon config/zookeeper.properties 4 | sleep 20 5 | bin/kafka-server-start.sh -daemon config/server0.properties 6 | bin/kafka-server-start.sh -daemon config/server1.properties 7 | bin/kafka-server-start.sh -daemon config/server2.properties 8 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter2/scripts/stopeverything.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | bin/kafka-server-stop.sh 4 | bin/zookeeper-server-stop.sh -------------------------------------------------------------------------------- /KafkaInAction_Chapter2/src/main/java/org/kafkainaction/consumer/HelloWorldConsumer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.consumer; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRecord; 4 | import org.apache.kafka.clients.consumer.ConsumerRecords; 5 | import org.apache.kafka.clients.consumer.KafkaConsumer; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.time.Duration; 10 | import java.util.List; 11 | import java.util.Properties; 12 | 13 | public class HelloWorldConsumer { 14 | 15 | final static Logger log = LoggerFactory.getLogger(HelloWorldConsumer.class); 16 | 17 | private volatile boolean keepConsuming = true; 18 | 19 | public static void main(String[] args) { 20 | Properties kaProperties = new Properties(); //<1> 21 | kaProperties.put("bootstrap.servers", 22 | "localhost:9092,localhost:9093,localhost:9094"); 23 | kaProperties.put("group.id", "kinaction_helloconsumer"); 24 | kaProperties.put("enable.auto.commit", "true"); 25 | kaProperties.put("auto.commit.interval.ms", "1000"); 26 | kaProperties.put("key.deserializer", 27 | "org.apache.kafka.common.serialization.StringDeserializer"); 28 | kaProperties.put("value.deserializer", 29 | "org.apache.kafka.common.serialization.StringDeserializer"); 30 | 31 | HelloWorldConsumer helloWorldConsumer = new HelloWorldConsumer(); 32 | helloWorldConsumer.consume(kaProperties); 33 | Runtime.getRuntime().addShutdownHook(new Thread(helloWorldConsumer::shutdown)); 34 | } 35 | 36 | private void consume(Properties kaProperties) { 37 | try (KafkaConsumer consumer = new KafkaConsumer<>(kaProperties)) { 38 | consumer.subscribe(List.of("kinaction_helloworld")); //<2> 39 | 40 | while (keepConsuming) { 41 | ConsumerRecords records = consumer.poll(Duration.ofMillis(250)); //<3> 42 | for (ConsumerRecord record : records) { //<4> 43 | log.info("kinaction_info offset = {}, kinaction_value = {}", 44 | record.offset(), record.value()); 45 | } 46 | } 47 | } 48 | } 49 | 50 | private void shutdown() { 51 | keepConsuming = false; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter2/src/main/java/org/kafkainaction/producer/HelloWorldProducer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.producer; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.Producer; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | 7 | import java.util.Properties; 8 | 9 | public class HelloWorldProducer { 10 | 11 | public static void main(String[] args) { 12 | 13 | Properties kaProperties = new Properties(); //<1> 14 | kaProperties.put("bootstrap.servers", 15 | "localhost:9092,localhost:9093,localhost:9094"); //<2> 16 | 17 | kaProperties.put( 18 | "key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); //<3> 19 | kaProperties.put("value.serializer", 20 | "org.apache.kafka.common.serialization.StringSerializer"); 21 | 22 | try (Producer producer = new KafkaProducer<>(kaProperties)) { //<4> 23 | 24 | ProducerRecord producerRecord = 25 | new ProducerRecord<>("kinaction_helloworld", null, "hello world again!"); //<5> 26 | 27 | producer.send(producerRecord); //<6> 28 | producer.close(); //<7> 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter2/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %yellow(%d{yyyy-MM-dd HH:mm:ss}) %highlight([%p]) %magenta((%file:%line\)) - %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter3/Commands.md: -------------------------------------------------------------------------------- 1 | # Source Code for Kafka in Action 2 | 3 | ## Chapter 3 Specific Notes 4 | 5 | * The commands and scripts are meant to be run from your Kafka root directory: i.e. you should see the folder `bin`, `config`, etc when you do an `ls` in this directory. 6 | 7 | * If you get a port in use exception: kill the process related to that port. 8 | For example, on a Mac, you can do the following command: `lsof -t -i tcp:8083 | xargs kill` if the `connect-standalone` process is still running, etc. 9 | 10 | ## sqlite3 database example 11 | 12 | * Close all zookeeper and kafka brokers you have open. 13 | We are going to start fresh. 14 | 15 | 16 | ### Download for Confluent Open Source 17 | * Note that the book text uses Confluent version 5.X and might not reflect the below instrucutions that have been updated to be newer versions* 18 | 19 | * https://www.confluent.io/previous-versions 20 | Select the «Download Community Tarball» version 6.1.1 or above and unzip that archive. 21 | Add location of unzipped Confluent Platform to environment variable `$CONFLUENT_HOME`. 22 | Add `$CONFLUENT_HOME/bin` to PATH. 23 | 24 | * install Confluent CLI https://docs.confluent.io/current/cli/installing.html 25 | * install JDBC connector using `confluent-hub` command 26 | 27 | confluent-hub install confluentinc/kafka-connect-jdbc:10.2.0 28 | 29 | and follow the prompts to install the connector 30 | 31 | ### Table create command in the root folder you just downloaded: 32 | 33 | * `sqlite3 kafkatest.db` 34 | 35 | ```sqlite 36 | CREATE TABLE invoices( 37 | id INT PRIMARY KEY NOT NULL, 38 | title TEXT NOT NULL, 39 | details CHAR(50), 40 | billedamt REAL, 41 | modified TIMESTAMP DEFAULT (STRFTIME('%s', 'now')) NOT NULL 42 | ); 43 | 44 | ``` 45 | 46 | ### Table insert command 47 | 48 | ```sqlite 49 | INSERT INTO invoices (id,title,details,billedamt) VALUES (1, 'book', 'Franz Kafka', 500.00 ); 50 | ``` 51 | 52 | ### Edit source jdbc connector file to the following: 53 | `> export CONFLUENT_HOME=~/confluent-6.2.0` 54 | 55 | 56 | `> mkdir -p $CONFLUENT_HOME/etc/kafka-connect-jdbc/` 57 | 58 | `> vi $CONFLUENT_HOME/etc/kafka-connect-jdbc/kafkatest-sqlite.properties` 59 | 60 | 61 | ### File Contents 62 | *NOTE: Update your database path below to be the full path for best results: ie. * 63 | connection.url=jdbc:sqlite://confluent-6.2.0/kafkatest.db 64 | 65 | ```properties 66 | name=kinaction-test-source-sqlite-jdbc-invoice 67 | connector.class=io.confluent.connect.jdbc.JdbcSourceConnector 68 | tasks.max=1 69 | # SQLite database stored in the file kafkatest.db, use and auto-incrementing column called 'id' to 70 | # detect new rows as they are added, and output to topics prefixed with 'kinaction-test-sqlite-jdbc-', e.g. 71 | # a table called 'invoices' will be written to the topic 'kinaction-test-sqlite-jdbc-invoices'. 72 | connection.url=jdbc:sqlite:kafkatest.db 73 | mode=incrementing 74 | incrementing.column.name=id 75 | topic.prefix=kinaction-test-sqlite-jdbc- 76 | ``` 77 | 78 | ### Start Confluent Kafka 79 | 80 | Run the following: 81 | 82 | ```bash 83 | > confluent local services connect start 84 | > confluent local services connect connector config jdbc-source --config $CONFLUENT_HOME/etc/kafka-connect-jdbc/kafkatest-sqlite.properties 85 | > confluent local services connect connector status 86 | > ./bin/kafka-avro-console-consumer --topic kinaction-test-sqlite-jdbc-invoices --bootstrap-server localhost:9092 --from-beginning 87 | ``` 88 | 89 | ## Avro Notes 90 | 91 | * You need to be running a schema registry to make this work. 92 | Make sure you start the `confluent local services connect start` before you try, and run the example. 93 | 94 | * You should see the producer message by running: 95 | 96 | ```bash 97 | confluent local services kafka consume kinaction_schematest --value-format avro --from-beginning 98 | ``` 99 | 100 | ## Java example commands 101 | 102 | NOTE: 103 | * Use JDK 11 104 | * Apache Kafka and Confluent Schema Registry should be up and running 105 | 106 | ```bash 107 | > ./mvnw verify # build a uber jar 108 | > java -cp target/chapter3-jar-with-dependencies.jar org.kafkainaction.consumer.HelloWorldConsumer # run a consumer application 109 | > java -cp target/chapter3-jar-with-dependencies.jar org.kafkainaction.producer.HelloWorldProducer # run a producer 110 | 111 | ``` 112 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter3/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | Kafka-In-Action 6 | org.kafkainaction 7 | 1.0.0-SNAPSHOT 8 | 9 | 10 | chapter3 11 | Designing a Kafka Project 12 | 13 | 14 | 15 | org.apache.avro 16 | avro 17 | ${avro.version} 18 | 19 | 20 | io.confluent 21 | kafka-avro-serializer 22 | ${confluent.version} 23 | 24 | 25 | 26 | 27 | chapter3 28 | 29 | 30 | maven-assembly-plugin 31 | 32 | 33 | org.apache.avro 34 | avro-maven-plugin 35 | ${avro.version} 36 | 37 | 38 | generate-sources 39 | 40 | schema 41 | 42 | 43 | ${project.basedir}/src/main/avro/ 44 | ${project.build.directory}/generated/src/main/java/ 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter3/src/main/avro/kinaction_alert.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "org.kafkainaction", 3 | "type": "record", 4 | "name": "Alert", 5 | "fields": [ 6 | { 7 | "name": "sensor_id", 8 | "type": "long", 9 | "doc": "The unique id that identifies the sensor" 10 | }, 11 | { 12 | "name": "time", 13 | "type": "long", 14 | "doc": "Time the alert was generated as UTC milliseconds from the epoch" 15 | }, 16 | { 17 | "name": "status", 18 | "type": { 19 | "type": "enum", 20 | "name": "AlertStatus", 21 | "symbols": [ 22 | "Critical", 23 | "Major", 24 | "Minor", 25 | "Warning" 26 | ] 27 | }, 28 | "doc": "The allowed values that our sensors will use to emit current status" 29 | } 30 | ] 31 | } 32 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter3/src/main/java/org/kafkainaction/consumer/HelloWorldConsumer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.consumer; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRecord; 4 | import org.apache.kafka.clients.consumer.ConsumerRecords; 5 | import org.apache.kafka.clients.consumer.KafkaConsumer; 6 | import org.kafkainaction.Alert; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.time.Duration; 11 | import java.util.List; 12 | import java.util.Properties; 13 | 14 | public class HelloWorldConsumer { 15 | 16 | final static Logger log = LoggerFactory.getLogger(HelloWorldConsumer.class); 17 | 18 | private volatile boolean keepConsuming = true; 19 | 20 | public static void main(String[] args) { 21 | Properties kaProperties = new Properties(); 22 | kaProperties.put("bootstrap.servers", "localhost:9094"); 23 | kaProperties.put("group.id", "kinaction_helloconsumer"); 24 | kaProperties.put("enable.auto.commit", "true"); 25 | kaProperties.put("auto.commit.interval.ms", "1000"); 26 | kaProperties.put("key.deserializer", 27 | "org.apache.kafka.common.serialization.LongDeserializer"); 28 | kaProperties.put("value.deserializer", 29 | "io.confluent.kafka.serializers.KafkaAvroDeserializer"); //<1> 30 | kaProperties.put("schema.registry.url", "http://localhost:8081"); 31 | 32 | HelloWorldConsumer helloWorldConsumer = new HelloWorldConsumer(); 33 | helloWorldConsumer.consume(kaProperties); 34 | 35 | Runtime.getRuntime().addShutdownHook(new Thread(helloWorldConsumer::shutdown)); 36 | } 37 | 38 | private void consume(Properties kaProperties) { 39 | try (KafkaConsumer consumer = new KafkaConsumer<>(kaProperties)) { 40 | consumer.subscribe(List.of("kinaction_schematest")); //<2> 41 | 42 | while (keepConsuming) { 43 | ConsumerRecords records = consumer.poll(Duration.ofMillis(250)); 44 | for (ConsumerRecord record : records) { //<3> 45 | log.info("kinaction_info offset = {}, kinaction_value = {}", 46 | record.offset(), 47 | record.value()); 48 | } 49 | } 50 | } 51 | } 52 | 53 | private void shutdown() { 54 | keepConsuming = false; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter3/src/main/java/org/kafkainaction/producer/HelloWorldProducer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.producer; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.Producer; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.kafkainaction.Alert; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.time.Instant; 11 | import java.util.Properties; 12 | 13 | import static org.kafkainaction.AlertStatus.Critical; 14 | 15 | public class HelloWorldProducer { 16 | 17 | static final Logger log = LoggerFactory.getLogger(HelloWorldProducer.class); 18 | 19 | public static void main(String[] args) { 20 | Properties kaProperties = new Properties(); 21 | kaProperties.put("bootstrap.servers", 22 | "localhost:9092,localhost:9093,localhost:9094"); 23 | kaProperties.put("key.serializer", 24 | "org.apache.kafka.common.serialization.LongSerializer"); 25 | kaProperties.put("value.serializer", 26 | "io.confluent.kafka.serializers.KafkaAvroSerializer"); //<1> 27 | kaProperties.put("schema.registry.url", "http://localhost:8081"); //<2> 28 | 29 | try (Producer producer = new KafkaProducer<>(kaProperties)) { 30 | Alert alert = new Alert(12345L, Instant.now().toEpochMilli(), Critical); //<3> 31 | 32 | log.info("kinaction_info Alert -> {}", alert); 33 | 34 | ProducerRecord producerRecord = 35 | new ProducerRecord<>("kinaction_schematest", 36 | alert.getSensorId(), 37 | alert); //<4> 38 | 39 | producer.send(producerRecord); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter3/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %yellow(%d{yyyy-MM-dd HH:mm:ss}) %highlight([%p]) %magenta((%file:%line\)) - %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter4/.mvn/wrapper/MavenWrapperDownloader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2007-present the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | import java.net.*; 17 | import java.io.*; 18 | import java.nio.channels.*; 19 | import java.util.Properties; 20 | 21 | public class MavenWrapperDownloader { 22 | 23 | private static final String WRAPPER_VERSION = "0.5.6"; 24 | /** 25 | * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. 26 | */ 27 | private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/" 28 | + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar"; 29 | 30 | /** 31 | * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to 32 | * use instead of the default one. 33 | */ 34 | private static final String MAVEN_WRAPPER_PROPERTIES_PATH = 35 | ".mvn/wrapper/maven-wrapper.properties"; 36 | 37 | /** 38 | * Path where the maven-wrapper.jar will be saved to. 39 | */ 40 | private static final String MAVEN_WRAPPER_JAR_PATH = 41 | ".mvn/wrapper/maven-wrapper.jar"; 42 | 43 | /** 44 | * Name of the property which should be used to override the default download url for the wrapper. 45 | */ 46 | private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; 47 | 48 | public static void main(String args[]) { 49 | System.out.println("- Downloader started"); 50 | File baseDirectory = new File(args[0]); 51 | System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); 52 | 53 | // If the maven-wrapper.properties exists, read it and check if it contains a custom 54 | // wrapperUrl parameter. 55 | File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); 56 | String url = DEFAULT_DOWNLOAD_URL; 57 | if(mavenWrapperPropertyFile.exists()) { 58 | FileInputStream mavenWrapperPropertyFileInputStream = null; 59 | try { 60 | mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); 61 | Properties mavenWrapperProperties = new Properties(); 62 | mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); 63 | url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); 64 | } catch (IOException e) { 65 | System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); 66 | } finally { 67 | try { 68 | if(mavenWrapperPropertyFileInputStream != null) { 69 | mavenWrapperPropertyFileInputStream.close(); 70 | } 71 | } catch (IOException e) { 72 | // Ignore ... 73 | } 74 | } 75 | } 76 | System.out.println("- Downloading from: " + url); 77 | 78 | File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); 79 | if(!outputFile.getParentFile().exists()) { 80 | if(!outputFile.getParentFile().mkdirs()) { 81 | System.out.println( 82 | "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'"); 83 | } 84 | } 85 | System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); 86 | try { 87 | downloadFileFromURL(url, outputFile); 88 | System.out.println("Done"); 89 | System.exit(0); 90 | } catch (Throwable e) { 91 | System.out.println("- Error downloading"); 92 | e.printStackTrace(); 93 | System.exit(1); 94 | } 95 | } 96 | 97 | private static void downloadFileFromURL(String urlString, File destination) throws Exception { 98 | if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) { 99 | String username = System.getenv("MVNW_USERNAME"); 100 | char[] password = System.getenv("MVNW_PASSWORD").toCharArray(); 101 | Authenticator.setDefault(new Authenticator() { 102 | @Override 103 | protected PasswordAuthentication getPasswordAuthentication() { 104 | return new PasswordAuthentication(username, password); 105 | } 106 | }); 107 | } 108 | URL website = new URL(urlString); 109 | ReadableByteChannel rbc; 110 | rbc = Channels.newChannel(website.openStream()); 111 | FileOutputStream fos = new FileOutputStream(destination); 112 | fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); 113 | fos.close(); 114 | rbc.close(); 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter4/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip 2 | wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar 3 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter4/Commands.md: -------------------------------------------------------------------------------- 1 | # Commands used in Chapter 4 2 | 3 | ## Creating the kinaction Topics 4 | 5 | `bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --topic kinaction_alert --partitions 3 --replication-factor 3` 6 | 7 | 8 | `bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --topic kinaction_alerttrend --partitions 3 --replication-factor 3` 9 | 10 | 11 | `bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --topic kinaction_audit --partitions 3 --replication-factor 3` 12 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter4/mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM http://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM set title of command window 39 | title %0 40 | @REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' 41 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 42 | 43 | @REM set %HOME% to equivalent of $HOME 44 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 45 | 46 | @REM Execute a user defined script before this one 47 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 48 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 49 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 50 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 51 | :skipRcPre 52 | 53 | @setlocal 54 | 55 | set ERROR_CODE=0 56 | 57 | @REM To isolate internal variables from possible post scripts, we use another setlocal 58 | @setlocal 59 | 60 | @REM ==== START VALIDATION ==== 61 | if not "%JAVA_HOME%" == "" goto OkJHome 62 | 63 | echo. 64 | echo Error: JAVA_HOME not found in your environment. >&2 65 | echo Please set the JAVA_HOME variable in your environment to match the >&2 66 | echo location of your Java installation. >&2 67 | echo. 68 | goto error 69 | 70 | :OkJHome 71 | if exist "%JAVA_HOME%\bin\java.exe" goto init 72 | 73 | echo. 74 | echo Error: JAVA_HOME is set to an invalid directory. >&2 75 | echo JAVA_HOME = "%JAVA_HOME%" >&2 76 | echo Please set the JAVA_HOME variable in your environment to match the >&2 77 | echo location of your Java installation. >&2 78 | echo. 79 | goto error 80 | 81 | @REM ==== END VALIDATION ==== 82 | 83 | :init 84 | 85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 86 | @REM Fallback to current working directory if not found. 87 | 88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 90 | 91 | set EXEC_DIR=%CD% 92 | set WDIR=%EXEC_DIR% 93 | :findBaseDir 94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 95 | cd .. 96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 97 | set WDIR=%CD% 98 | goto findBaseDir 99 | 100 | :baseDirFound 101 | set MAVEN_PROJECTBASEDIR=%WDIR% 102 | cd "%EXEC_DIR%" 103 | goto endDetectBaseDir 104 | 105 | :baseDirNotFound 106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 107 | cd "%EXEC_DIR%" 108 | 109 | :endDetectBaseDir 110 | 111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 112 | 113 | @setlocal EnableExtensions EnableDelayedExpansion 114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 116 | 117 | :endReadAdditionalConfig 118 | 119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 120 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 121 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 122 | 123 | set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" 124 | 125 | FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( 126 | IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B 127 | ) 128 | 129 | @REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 130 | @REM This allows using the maven wrapper in projects that prohibit checking in binary data. 131 | if exist %WRAPPER_JAR% ( 132 | if "%MVNW_VERBOSE%" == "true" ( 133 | echo Found %WRAPPER_JAR% 134 | ) 135 | ) else ( 136 | if not "%MVNW_REPOURL%" == "" ( 137 | SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" 138 | ) 139 | if "%MVNW_VERBOSE%" == "true" ( 140 | echo Couldn't find %WRAPPER_JAR%, downloading it ... 141 | echo Downloading from: %DOWNLOAD_URL% 142 | ) 143 | 144 | powershell -Command "&{"^ 145 | "$webclient = new-object System.Net.WebClient;"^ 146 | "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ 147 | "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ 148 | "}"^ 149 | "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ 150 | "}" 151 | if "%MVNW_VERBOSE%" == "true" ( 152 | echo Finished downloading %WRAPPER_JAR% 153 | ) 154 | ) 155 | @REM End of extension 156 | 157 | @REM Provide a "standardized" way to retrieve the CLI args that will 158 | @REM work with both Windows and non-Windows executions. 159 | set MAVEN_CMD_LINE_ARGS=%* 160 | 161 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 162 | if ERRORLEVEL 1 goto error 163 | goto end 164 | 165 | :error 166 | set ERROR_CODE=1 167 | 168 | :end 169 | @endlocal & set ERROR_CODE=%ERROR_CODE% 170 | 171 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 172 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 173 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 174 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 175 | :skipRcPost 176 | 177 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 178 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 179 | 180 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 181 | 182 | exit /B %ERROR_CODE% 183 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter4/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | Kafka-In-Action 6 | org.kafkainaction 7 | 1.0.0-SNAPSHOT 8 | 9 | 10 | chapter4 11 | Producers: Sourcing Data 12 | 13 | 14 | chapter4 15 | 16 | 17 | maven-assembly-plugin 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter4/src/main/java/org/kafkainaction/callback/AlertCallback.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.callback; 2 | 3 | import org.apache.kafka.clients.producer.Callback; 4 | import org.apache.kafka.clients.producer.RecordMetadata; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | public class AlertCallback implements Callback { //<1> 9 | 10 | private static final Logger log = LoggerFactory.getLogger(AlertCallback.class); 11 | 12 | public void onCompletion(RecordMetadata metadata, Exception exception) { //<2> 13 | if (exception != null) { 14 | log.error("kinaction_error", exception); 15 | } else { 16 | log.info("kinaction_info offset = {}, topic = {}, timestamp = {}", 17 | metadata.offset(), metadata.topic(), metadata.timestamp()); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter4/src/main/java/org/kafkainaction/consumer/AlertConsumer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.consumer; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRecord; 4 | import org.apache.kafka.clients.consumer.ConsumerRecords; 5 | import org.apache.kafka.clients.consumer.KafkaConsumer; 6 | import org.apache.kafka.clients.consumer.OffsetAndMetadata; 7 | import org.apache.kafka.common.TopicPartition; 8 | import org.kafkainaction.model.Alert; 9 | import org.kafkainaction.serde.AlertKeySerde; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import java.time.Duration; 14 | import java.util.HashMap; 15 | import java.util.List; 16 | import java.util.Map; 17 | import java.util.Properties; 18 | 19 | public class AlertConsumer { 20 | 21 | final static Logger log = LoggerFactory.getLogger(AlertConsumer.class); 22 | private volatile boolean keepConsuming = true; 23 | 24 | public static void main(String[] args) { 25 | Properties kaProperties = new Properties(); 26 | kaProperties.put("bootstrap.servers", "localhost:9092,localhost:9093"); 27 | kaProperties.put("enable.auto.commit", "false"); 28 | kaProperties.put("group.id", "kinaction_team0groupalert"); 29 | /** Deserialize key using {@link org.kafkainaction.serde.AlertKeySerde} */ 30 | kaProperties.put("key.deserializer", AlertKeySerde.class.getName()); 31 | kaProperties.put("value.deserializer", 32 | "org.apache.kafka.common.serialization.StringDeserializer"); 33 | 34 | AlertConsumer consumer = new AlertConsumer(); 35 | consumer.consume(kaProperties); 36 | 37 | Runtime.getRuntime().addShutdownHook(new Thread(consumer::shutdown)); 38 | } 39 | 40 | private void shutdown() { 41 | keepConsuming = false; 42 | } 43 | 44 | private void consume(final Properties kaProperties) { 45 | KafkaConsumer consumer = new KafkaConsumer<>(kaProperties); 46 | TopicPartition partitionZero = new TopicPartition("kinaction_alert", 0); 47 | consumer.assign(List.of(partitionZero)); 48 | 49 | while (keepConsuming) { 50 | ConsumerRecords records = consumer.poll(Duration.ofMillis(250)); 51 | for (ConsumerRecord record : records) { 52 | log.info("kinaction_info offset = {}, key = {}", 53 | record.offset(), 54 | record.key().getStageId()); 55 | commitOffset(record.offset(), record.partition(), "kinaction_alert", consumer); 56 | } 57 | } 58 | } 59 | 60 | public static void commitOffset(long offset, int part, String topic, KafkaConsumer consumer) { 61 | OffsetAndMetadata offsetMeta = new OffsetAndMetadata(++offset, ""); 62 | 63 | Map kaOffsetMap = new HashMap<>(); 64 | kaOffsetMap.put(new TopicPartition(topic, part), offsetMeta); 65 | 66 | consumer.commitAsync(kaOffsetMap, AlertConsumer::onComplete); 67 | } 68 | 69 | private static void onComplete(Map map, 70 | Exception e) { 71 | if (e != null) { 72 | for (TopicPartition key : map.keySet()) { 73 | log.info("kinaction_error topic {}, partition {}, offset {}", 74 | key.topic(), 75 | key.partition(), 76 | map.get(key).offset()); 77 | } 78 | } else { 79 | for (TopicPartition key : map.keySet()) { 80 | log.info("kinaction_info topic {}, partition {}, offset {}", 81 | key.topic(), 82 | key.partition(), 83 | map.get(key).offset()); 84 | } 85 | } 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter4/src/main/java/org/kafkainaction/model/Alert.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.model; 2 | 3 | import java.io.Serializable; 4 | 5 | public class Alert implements Serializable { 6 | 7 | private final int alertId; 8 | private String stageId; 9 | private final String alertLevel; 10 | private final String alertMessage; 11 | 12 | public Alert(int alertId, String stageId, 13 | String alertLevel, String alertMessage) { //<1> 14 | this.alertId = alertId; 15 | this.stageId = stageId; 16 | this.alertLevel = alertLevel; 17 | this.alertMessage = alertMessage; 18 | } 19 | 20 | public int getAlertId() { 21 | return alertId; 22 | } 23 | 24 | public String getStageId() { 25 | return stageId; 26 | } 27 | 28 | public void setStageId(String stageId) { 29 | this.stageId = stageId; 30 | } 31 | 32 | public String getAlertLevel() { 33 | return alertLevel; 34 | } 35 | 36 | public String getAlertMessage() { 37 | return alertMessage; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter4/src/main/java/org/kafkainaction/partitioner/AlertLevelPartitioner.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.partitioner; 2 | 3 | import org.apache.kafka.clients.producer.Partitioner; 4 | import org.apache.kafka.common.Cluster; 5 | import org.apache.kafka.common.PartitionInfo; 6 | import org.kafkainaction.model.Alert; 7 | 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.Random; 11 | 12 | public class AlertLevelPartitioner implements Partitioner { //<1> 13 | 14 | public int partition(final String topic, 15 | final Object objectKey, 16 | final byte[] keyBytes, 17 | final Object value, 18 | final byte[] valueBytes, 19 | final Cluster cluster) { 20 | 21 | 22 | int criticalLevelPartition = findCriticalPartitionNumber(cluster, topic); 23 | 24 | return isCriticalLevel(((Alert) objectKey).getAlertLevel()) ? 25 | criticalLevelPartition : 26 | findRandomPartition(cluster, topic, objectKey); 27 | } 28 | 29 | public int findCriticalPartitionNumber(Cluster cluster, String topic) { 30 | //not using parameters but could if needed for your logic 31 | return 0; 32 | } 33 | 34 | public int findRandomPartition(Cluster cluster, String topic, Object objectKey) { 35 | //not using parameter objectKey but could if needed for your logic 36 | List partitionMetaList = 37 | cluster.availablePartitionsForTopic(topic); 38 | 39 | Random randomPart = new Random(); 40 | return randomPart.nextInt(partitionMetaList.size()); 41 | } 42 | 43 | public boolean isCriticalLevel(String level) { 44 | if (level.toUpperCase().contains("CRITICAL")) { 45 | return true; 46 | } else { 47 | return false; 48 | } 49 | } 50 | 51 | @Override 52 | public void close() { 53 | 54 | } 55 | 56 | @Override 57 | public void configure(final Map map) { 58 | 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter4/src/main/java/org/kafkainaction/producer/AlertProducer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.producer; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.Producer; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.kafkainaction.callback.AlertCallback; 7 | import org.kafkainaction.model.Alert; 8 | import org.kafkainaction.partitioner.AlertLevelPartitioner; 9 | import org.kafkainaction.serde.AlertKeySerde; 10 | 11 | import java.util.Properties; 12 | 13 | public class AlertProducer { 14 | 15 | public static void main(String[] args) { 16 | 17 | Properties kaProperties = new Properties(); 18 | kaProperties.put("bootstrap.servers", "localhost:9092,localhost:9093"); 19 | 20 | kaProperties.put("key.serializer", AlertKeySerde.class.getName()); //<1> 21 | kaProperties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 22 | /** Use {@link org.kafkainaction.partitioner.AlertLevelPartitioner} to determine partition */ 23 | kaProperties.put("partitioner.class", AlertLevelPartitioner.class.getName()); //<2> 24 | 25 | try (Producer producer = new KafkaProducer<>(kaProperties)) { 26 | Alert alert = new Alert(1, "Stage 1", "CRITICAL", "Stage 1 stopped"); 27 | ProducerRecord 28 | producerRecord = new ProducerRecord<>("kinaction_alert", alert, alert.getAlertMessage()); //<3> 29 | 30 | producer.send(producerRecord, new AlertCallback()); 31 | } 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter4/src/main/java/org/kafkainaction/producer/AlertTrendingProducer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.producer; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.Producer; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.apache.kafka.clients.producer.RecordMetadata; 7 | import org.kafkainaction.model.Alert; 8 | import org.kafkainaction.serde.AlertKeySerde; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.util.Properties; 13 | import java.util.concurrent.ExecutionException; 14 | 15 | public class AlertTrendingProducer { 16 | 17 | private static final Logger log = 18 | LoggerFactory.getLogger(AlertTrendingProducer.class); 19 | 20 | public static void main(String[] args) 21 | throws InterruptedException, ExecutionException { 22 | 23 | Properties kaProperties = new Properties(); 24 | kaProperties.put("bootstrap.servers", 25 | "localhost:9092,localhost:9093,localhost:9094"); 26 | kaProperties.put("key.serializer", 27 | AlertKeySerde.class.getName()); //<1> 28 | kaProperties.put("value.serializer", 29 | "org.apache.kafka.common.serialization.StringSerializer"); 30 | 31 | try (Producer producer = new KafkaProducer<>(kaProperties)) { 32 | Alert alert = new Alert(0, "Stage 0", "CRITICAL", "Stage 0 stopped"); 33 | ProducerRecord producerRecord = 34 | new ProducerRecord<>("kinaction_alerttrend", alert, alert.getAlertMessage()); //<2> 35 | 36 | RecordMetadata result = producer.send(producerRecord).get(); 37 | log.info("kinaction_info offset = {}, topic = {}, timestamp = {}", 38 | result.offset(), result.topic(), result.timestamp()); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter4/src/main/java/org/kafkainaction/producer/AuditProducer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.producer; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.Producer; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.apache.kafka.clients.producer.RecordMetadata; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.util.Properties; 11 | import java.util.concurrent.ExecutionException; 12 | 13 | public class AuditProducer { 14 | 15 | private static final Logger log = LoggerFactory.getLogger(AuditProducer.class); 16 | 17 | public static void main(String[] args) throws InterruptedException, ExecutionException { 18 | 19 | Properties kaProperties = new Properties(); //<1> 20 | kaProperties.put("bootstrap.servers", "localhost:9092,localhost:9093"); 21 | kaProperties.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 22 | kaProperties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 23 | kaProperties.put("acks", "all"); //<2> 24 | kaProperties.put("retries", "3"); //<3> 25 | kaProperties.put("max.in.flight.requests.per.connection", "1"); 26 | 27 | try (Producer producer = new KafkaProducer<>(kaProperties)) { 28 | ProducerRecord producerRecord = new ProducerRecord<>("kinaction_audit", null, 29 | "audit event"); 30 | 31 | RecordMetadata result = producer.send(producerRecord).get(); 32 | log.info("kinaction_info offset = {}, topic = {}, timestamp = {}", result.offset(), result.topic(), result.timestamp()); 33 | 34 | } 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter4/src/main/java/org/kafkainaction/producer/FlumeSinkProducer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.producer; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.Producer; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.kafkainaction.callback.AlertCallback; 7 | 8 | import java.io.IOException; 9 | import java.nio.file.Files; 10 | import java.nio.file.Path; 11 | import java.nio.file.Paths; 12 | import java.util.Properties; 13 | import java.util.stream.Stream; 14 | 15 | 16 | @SuppressWarnings("unused") 17 | public class FlumeSinkProducer { 18 | 19 | public static void main(String[] args) { 20 | 21 | Properties kaProperties = readConfig(); 22 | 23 | String topic = kaProperties.getProperty("topic"); 24 | kaProperties.remove("topic"); 25 | 26 | try (Producer producer = new KafkaProducer<>(kaProperties)) { 27 | 28 | ProducerRecord producerRecord = new ProducerRecord<>(topic, null, "event"); 29 | producer.send(producerRecord, new AlertCallback()); 30 | } 31 | 32 | } 33 | 34 | private static Properties readConfig() { 35 | Path path = Paths.get("src/main/resources/kafkasink.conf"); 36 | 37 | Properties kaProperties = new Properties(); 38 | 39 | try (Stream lines = Files.lines(path)) { 40 | lines.forEachOrdered(line -> determineProperty(line, kaProperties)); 41 | } catch (IOException e) { 42 | System.out.println("kinaction_error " + e); 43 | } 44 | return kaProperties; 45 | } 46 | 47 | private static void determineProperty(String line, Properties kaProperties) { 48 | if (line.contains("bootstrap")) { 49 | kaProperties.put("bootstrap.servers", line.split("=")[1]); 50 | } else if (line.contains("acks")) { 51 | kaProperties.put("acks", line.split("=")[1]); 52 | } else if (line.contains("compression.type")) { 53 | kaProperties.put("compression.type", line.split("=")[1]); 54 | } else if (line.contains("topic")) { 55 | kaProperties.put("topic", line.split("=")[1]); 56 | } 57 | 58 | kaProperties.putIfAbsent("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 59 | kaProperties.putIfAbsent("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 60 | 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter4/src/main/java/org/kafkainaction/serde/AlertKeySerde.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.serde; 2 | 3 | import org.apache.kafka.common.serialization.Deserializer; 4 | import org.apache.kafka.common.serialization.Serializer; 5 | import org.kafkainaction.model.Alert; 6 | 7 | import java.nio.charset.StandardCharsets; 8 | import java.util.Map; 9 | 10 | public class AlertKeySerde implements Serializer, 11 | Deserializer { //<1> 12 | 13 | public byte[] serialize(String topic, Alert key) { //<2> 14 | if (key == null) { 15 | return null; 16 | } 17 | return key.getStageId().getBytes(StandardCharsets.UTF_8); //<3> 18 | } 19 | 20 | public Alert deserialize(String topic, byte[] value) { //<4> 21 | //could return Alert in future if needed 22 | return null; 23 | } 24 | 25 | @Override 26 | public void configure(final Map configs, final boolean isKey) { 27 | Serializer.super.configure(configs, isKey); 28 | } 29 | 30 | @Override 31 | public void close() { 32 | Serializer.super.close(); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter4/src/main/resources/kafkasink.conf: -------------------------------------------------------------------------------- 1 | a1.sinks.k1.kafka.topic=kinaction_helloworld 2 | a1.sinks.k1.kafka.bootstrap.servers=localhost:9092 3 | a1.sinks.k1.kafka.producer.acks=1 4 | a1.sinks.k1.kafka.producer.compression.type=snappy 5 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter4/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %yellow(%d{yyyy-MM-dd HH:mm:ss}) %highlight([%p]) %magenta((%file:%line\)) - %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter5/Commands.md: -------------------------------------------------------------------------------- 1 | # Source Code for Kafka in Action 2 | 3 | ## Notes 4 | 5 | * Please refer to Chapter 4 for Alert Consumer Example. 6 | 7 | 8 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter5/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | 6 | Kafka-In-Action 7 | org.kafkainaction 8 | 1.0.0-SNAPSHOT 9 | 10 | 11 | chapter5 12 | Consumers: Unlocking Data 13 | 14 | 15 | chapter5 16 | 17 | 18 | maven-assembly-plugin 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter5/src/main/java/org/kafkainaction/consumer/ASyncCommit.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.consumer; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRecord; 4 | import org.apache.kafka.clients.consumer.ConsumerRecords; 5 | import org.apache.kafka.clients.consumer.KafkaConsumer; 6 | import org.apache.kafka.clients.consumer.OffsetAndMetadata; 7 | import org.apache.kafka.common.TopicPartition; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.time.Duration; 12 | import java.util.HashMap; 13 | import java.util.List; 14 | import java.util.Map; 15 | import java.util.Properties; 16 | 17 | public class ASyncCommit { 18 | 19 | final static Logger log = LoggerFactory.getLogger(ASyncCommit.class); 20 | private volatile boolean keepConsuming = true; 21 | public static final String TOPIC_NAME = "kinaction_views"; 22 | 23 | public static void main(String[] args) { 24 | Properties kaProperties = new Properties(); 25 | kaProperties.put("bootstrap.servers", "localhost:9092,localhost:9093"); 26 | kaProperties.put("group.id", "kinaction_group_views"); 27 | kaProperties.put("key.deserializer", 28 | "org.apache.kafka.common.serialization.StringDeserializer"); 29 | kaProperties.put("value.deserializer", 30 | "org.apache.kafka.common.serialization.StringDeserializer"); 31 | 32 | final ASyncCommit aSyncCommit = new ASyncCommit(); 33 | aSyncCommit.consume(kaProperties); 34 | Runtime.getRuntime().addShutdownHook(new Thread(aSyncCommit::shutdown)); 35 | } 36 | 37 | private void consume(final Properties kaProperties) { 38 | try (KafkaConsumer consumer = new KafkaConsumer<>(kaProperties)) { 39 | 40 | consumer.assign(List.of(new TopicPartition(TOPIC_NAME, 1), 41 | new TopicPartition(TOPIC_NAME, 2))); 42 | 43 | while (keepConsuming) { 44 | ConsumerRecords records = consumer.poll(Duration.ofMillis(250)); 45 | for (ConsumerRecord record : records) { 46 | log.info("kinaction_info offset = {}, value = {}", record.offset(), record.value()); 47 | commitOffset(record.offset(), record.partition(), TOPIC_NAME, consumer); 48 | } 49 | } 50 | } 51 | } 52 | 53 | public static void commitOffset(long offset, 54 | int partition, 55 | String topic, 56 | KafkaConsumer consumer) { 57 | OffsetAndMetadata offsetMeta = new OffsetAndMetadata(++offset, ""); 58 | 59 | Map kaOffsetMap = new HashMap<>(); 60 | kaOffsetMap.put(new TopicPartition(topic, partition), offsetMeta); 61 | 62 | consumer.commitAsync(kaOffsetMap, (map, e) -> { 63 | if (e != null) { 64 | for (TopicPartition key : map.keySet()) { 65 | log.info("kinaction_error topic {}, offset {}", key.topic(), map.get(key).offset()); 66 | } 67 | } else { 68 | for (TopicPartition key : map.keySet()) { 69 | log.info("kinaction_info topic {}, offset {}", key.topic(), map.get(key).offset()); 70 | } 71 | } 72 | }); 73 | } 74 | 75 | private void shutdown() { 76 | keepConsuming = false; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter5/src/main/java/org/kafkainaction/consumer/AlertTrendConsumer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.consumer; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRecord; 4 | import org.apache.kafka.clients.consumer.KafkaConsumer; 5 | import org.kafkainaction.serde.AlertKeySerde; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.time.Duration; 10 | import java.util.List; 11 | import java.util.Properties; 12 | 13 | public class AlertTrendConsumer { 14 | 15 | final static Logger log = LoggerFactory.getLogger(AlertTrendConsumer.class); 16 | private volatile boolean keepConsuming = true; 17 | 18 | public static void main(String[] args) { 19 | Properties kaProperties = new Properties(); 20 | kaProperties.put("bootstrap.servers", "localhost:9092,localhost:9093"); 21 | kaProperties.put("enable.auto.commit", "true"); 22 | kaProperties.put("group.id", "kinaction_team0groupalerttrend"); 23 | kaProperties.put("key.deserializer", AlertKeySerde.class.getName()); 24 | kaProperties.put("value.deserializer", 25 | "org.apache.kafka.common.serialization.StringDeserializer"); 26 | 27 | AlertTrendConsumer consumer = new AlertTrendConsumer(); 28 | consumer.consume(kaProperties); 29 | 30 | Runtime.getRuntime().addShutdownHook(new Thread(consumer::shutdown)); 31 | } 32 | 33 | private void shutdown() { 34 | keepConsuming = false; 35 | } 36 | 37 | 38 | private void consume(final Properties kaProperties) { 39 | try (KafkaConsumer consumer = new KafkaConsumer<>(kaProperties)) { 40 | 41 | consumer.subscribe(List.of("kinaction_alerttrend")); 42 | 43 | while (keepConsuming) { 44 | var records = consumer.poll(Duration.ofMillis(250)); 45 | for (ConsumerRecord record : records) { 46 | log.info("kinaction_info offset = {}, value = {}", 47 | record.offset(), record.value()); 48 | } 49 | } 50 | } 51 | 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter5/src/main/java/org/kafkainaction/consumer/AuditConsumer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.consumer; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRecord; 4 | import org.apache.kafka.clients.consumer.KafkaConsumer; 5 | import org.apache.kafka.clients.consumer.OffsetAndMetadata; 6 | import org.apache.kafka.common.TopicPartition; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.time.Duration; 11 | import java.util.HashMap; 12 | import java.util.List; 13 | import java.util.Map; 14 | import java.util.Properties; 15 | 16 | public class AuditConsumer { 17 | 18 | final static Logger log = LoggerFactory.getLogger(AuditConsumer.class); 19 | private volatile boolean keepConsuming = true; 20 | 21 | public static void main(String[] args) { 22 | Properties kaProperties = new Properties(); 23 | kaProperties.put("bootstrap.servers", "localhost:9092,localhost:9093"); 24 | kaProperties.put("enable.auto.commit", "false"); 25 | kaProperties.put("group.id", "kinaction_group_audit"); 26 | kaProperties.put("key.deserializer", 27 | "org.apache.kafka.common.serialization.StringDeserializer"); 28 | kaProperties.put("value.deserializer", 29 | "org.apache.kafka.common.serialization.StringDeserializer"); 30 | 31 | final AuditConsumer auditConsumer = new AuditConsumer(); 32 | auditConsumer.consume(kaProperties); 33 | Runtime.getRuntime().addShutdownHook(new Thread(auditConsumer::shutdown)); 34 | } 35 | 36 | private void consume(final Properties kaProperties) { 37 | try (KafkaConsumer consumer = new KafkaConsumer<>(kaProperties)) { 38 | 39 | consumer.subscribe(List.of("kinaction_audit")); 40 | 41 | while (keepConsuming) { 42 | var records = consumer.poll(Duration.ofMillis(250)); 43 | for (ConsumerRecord record : records) { 44 | log.info("kinaction_info offset = {}, value = {}", 45 | record.offset(), record.value()); 46 | 47 | OffsetAndMetadata offsetMeta = new OffsetAndMetadata(record.offset() + 1, ""); 48 | 49 | Map kaOffsetMap = new HashMap<>(); 50 | kaOffsetMap.put(new TopicPartition("kinaction_audit", record.partition()), offsetMeta); 51 | 52 | consumer.commitSync(kaOffsetMap); 53 | } 54 | } 55 | } 56 | 57 | } 58 | 59 | private void shutdown() { 60 | keepConsuming = false; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter5/src/main/java/org/kafkainaction/consumer/KinactionStopConsumer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.consumer; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRecord; 4 | import org.apache.kafka.clients.consumer.ConsumerRecords; 5 | import org.apache.kafka.clients.consumer.KafkaConsumer; 6 | import org.apache.kafka.common.errors.WakeupException; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.time.Duration; 11 | import java.util.List; 12 | import java.util.concurrent.atomic.AtomicBoolean; 13 | 14 | public class KinactionStopConsumer implements Runnable { 15 | 16 | final static Logger log = LoggerFactory.getLogger(KinactionStopConsumer.class); 17 | 18 | private final KafkaConsumer consumer; 19 | private final AtomicBoolean stopping = new AtomicBoolean(false); 20 | 21 | public KinactionStopConsumer(KafkaConsumer consumer) { 22 | this.consumer = consumer; 23 | } 24 | 25 | public void run() { 26 | try { 27 | consumer.subscribe(List.of("kinaction_promos")); 28 | while (!stopping.get()) { 29 | ConsumerRecords records = consumer.poll(Duration.ofMillis(250)); 30 | for (ConsumerRecord record : records) { 31 | log.info("kinaction_info offset = {},value = {}", record.offset(), record.value()); 32 | } 33 | } 34 | } catch (WakeupException e) { 35 | if (!stopping.get()) { 36 | throw e; 37 | } 38 | } finally { 39 | consumer.close(); 40 | } 41 | } 42 | 43 | public void shutdown() { 44 | stopping.set(true); 45 | consumer.wakeup(); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter5/src/main/java/org/kafkainaction/consumer/WebClickConsumer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.consumer; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRecord; 4 | import org.apache.kafka.clients.consumer.KafkaConsumer; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.time.Duration; 9 | import java.util.List; 10 | import java.util.Properties; 11 | 12 | 13 | public class WebClickConsumer { 14 | 15 | final static Logger log = LoggerFactory.getLogger(WebClickConsumer.class); 16 | private volatile boolean keepConsuming = true; 17 | 18 | public static void main(String[] args) { 19 | Properties kaProperties = new Properties(); 20 | kaProperties.put("bootstrap.servers", 21 | "localhost:9092,localhost:9093,,localhost:9094"); 22 | kaProperties.put("group.id", "kinaction_webconsumer"); //<1> 23 | kaProperties.put("enable.auto.commit", "true"); 24 | kaProperties.put("auto.commit.interval.ms", "1000"); 25 | kaProperties.put("key.deserializer", 26 | "org.apache.kafka.common.serialization.StringDeserializer"); //<2> 27 | kaProperties.put("value.deserializer", 28 | "org.apache.kafka.common.serialization.StringDeserializer"); 29 | 30 | WebClickConsumer webClickConsumer = new WebClickConsumer(); 31 | webClickConsumer.consume(kaProperties); 32 | 33 | Runtime.getRuntime().addShutdownHook(new Thread(webClickConsumer::shutdown)); 34 | } 35 | 36 | private void consume(Properties kaProperties) { 37 | try (KafkaConsumer consumer = new KafkaConsumer<>(kaProperties)) { //<3> 38 | consumer.subscribe(List.of("kinaction_promos")); //<4> 39 | 40 | while (keepConsuming) { //<5> 41 | var records = consumer.poll(Duration.ofMillis(250)); 42 | for (ConsumerRecord record : records) { 43 | log.info("kinaction_info offset = {}, key = {}", 44 | record.offset(), record.key()); 45 | log.info("kinaction_info value = {}", Double.parseDouble(record.value()) * 1.543); 46 | } 47 | } 48 | } 49 | } 50 | 51 | private void shutdown() { 52 | keepConsuming = false; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter5/src/main/java/org/kafkainaction/model/Alert.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.model; 2 | 3 | import java.io.Serializable; 4 | 5 | public class Alert implements Serializable { 6 | 7 | private final int alertId; 8 | private String stageId; 9 | private final String alertLevel; 10 | private final String alertMessage; 11 | 12 | public Alert(int alertId, String stageId, 13 | String alertLevel, String alertMessage) { //<1> 14 | this.alertId = alertId; 15 | this.stageId = stageId; 16 | this.alertLevel = alertLevel; 17 | this.alertMessage = alertMessage; 18 | } 19 | 20 | public int getAlertId() { 21 | return alertId; 22 | } 23 | 24 | public String getStageId() { 25 | return stageId; 26 | } 27 | 28 | public void setStageId(String stageId) { 29 | this.stageId = stageId; 30 | } 31 | 32 | public String getAlertLevel() { 33 | return alertLevel; 34 | } 35 | 36 | public String getAlertMessage() { 37 | return alertMessage; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter5/src/main/java/org/kafkainaction/serde/AlertKeySerde.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.serde; 2 | 3 | import org.apache.kafka.common.serialization.Deserializer; 4 | import org.apache.kafka.common.serialization.Serializer; 5 | import org.kafkainaction.model.Alert; 6 | 7 | import java.nio.charset.StandardCharsets; 8 | import java.util.Map; 9 | 10 | public class AlertKeySerde implements Serializer, 11 | Deserializer { //<1> 12 | 13 | public byte[] serialize(String topic, Alert key) { //<2> 14 | if (key == null) { 15 | return null; 16 | } 17 | return key.getStageId().getBytes(StandardCharsets.UTF_8); //<3> 18 | } 19 | 20 | public Alert deserialize(String topic, byte[] value) { //<4> 21 | //could return Alert in future if needed 22 | return null; 23 | } 24 | 25 | @Override 26 | public void configure(final Map configs, final boolean isKey) { 27 | Serializer.super.configure(configs, isKey); 28 | } 29 | 30 | @Override 31 | public void close() { 32 | Serializer.super.close(); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter5/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %yellow(%d{yyyy-MM-dd HH:mm:ss}) %highlight([%p]) %magenta((%file:%line\)) - %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter6/Commands.md: -------------------------------------------------------------------------------- 1 | # Commands used in Chapter 6 2 | 3 | ## Listing topics from Zookeeper 4 | 5 | bin/zookeeper-shell.sh localhost:2181 6 | ls /brokers/topics 7 | 8 | ## Find Controller in ZooKeeper 9 | 10 | bin/zookeeper-shell.sh localhost:2181 11 | get /controller 12 | 13 | 14 | ## Starting a broker with a JMX Port 15 | 16 | JMX_PORT=$JMX_PORT bin/kafka-server-start.sh config/server0.properties 17 | 18 | 19 | ## Describe Topic 20 | 21 | $ bin/kafka-topics.sh --describe --bootstrap-server localhost:9094 --topic kinaction_replica_test 22 | 23 | # Sample output 24 | Topic:kinaction_replica_test PartitionCount:1 ReplicationFactor:3 Configs: 25 | Topic: kinaction_replica_test Partition: 0 Leader: 0 Replicas: 1,0,2 Isr: 0,2 26 | 27 | 28 | ## Under-replicated-partitions flag 29 | 30 | bin/kafka-topics.sh --describe --bootstrap-server localhost:9094 --under-replicated-partitions 31 | 32 | #Sample output 33 | Topic: kinaction_replica_test Partition: 0 Leader: 0 Replicas: 1,0,2 Isr: 0,2 34 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter6/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | Kafka-In-Action 6 | org.kafkainaction 7 | 1.0.0-SNAPSHOT 8 | 9 | 10 | chapter6-no-java 11 | Brokers 12 | 13 | 14 | chapter6 15 | 16 | 17 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter7/Commands.md: -------------------------------------------------------------------------------- 1 | # Commands used in Chapter 7 2 | 3 | ## Creating a topic for Chapter 7 4 | 5 | bin/kafka-topics.sh --create --bootstrap-server localhost:9094 \ 6 | --topic kinaction_topicandpart \ 7 | --replication-factor 2 \ 8 | --partitions 2 9 | 10 | 11 | ## Deleting a topic 12 | 13 | bin/kafka-topics.sh --delete --bootstrap-server localhost:9094 \ // #A <1> 14 | --topic kinaction_topicandpart 15 | 16 | 17 | ## Looking at a dump of a segment 18 | 19 | bin/kafka-dump-log.sh --print-data-log --files /tmp/kafkainaction/kafka-logs-0/kinaction_topicandpart-1/*.log| awk -F: '{print $NF}' | grep kinaction 20 | 21 | ## Creating a compacted topic 22 | 23 | bin/kafka-topics.sh --create --bootstrap-server localhost:9094 \ 24 | --topic kinaction_compact --partitions 3 --replication-factor 3 \ 25 | --config cleanup.policy=compact 26 | 27 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter7/pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 4.0.0 5 | 6 | Kafka-In-Action 7 | org.kafkainaction 8 | 1.0.0-SNAPSHOT 9 | 10 | 11 | chapter7 12 | Topics and Partitions 13 | 14 | 1.20.6 15 | 16 | 17 | 18 | 19 | org.apache.kafka 20 | kafka-streams 21 | ${kafka.version} 22 | 23 | 24 | org.apache.commons 25 | commons-lang3 26 | 3.0 27 | 28 | 29 | junit 30 | junit 31 | 4.13.2 32 | test 33 | 34 | 35 | org.apache.kafka 36 | kafka-clients 37 | ${kafka.version} 38 | test 39 | test 40 | 41 | 42 | org.apache.kafka 43 | kafka-streams 44 | ${kafka.version} 45 | test 46 | test 47 | 48 | 49 | org.apache.kafka 50 | kafka-streams-test-utils 51 | ${kafka.version} 52 | test 53 | 54 | 55 | 56 | org.apache.kafka 57 | kafka_2.12 58 | ${kafka.version} 59 | test 60 | test 61 | 62 | 63 | org.apache.kafka 64 | kafka_2.12 65 | ${kafka.version} 66 | test 67 | 68 | 69 | 70 | 71 | org.testcontainers 72 | testcontainers 73 | ${testcontainers.version} 74 | test 75 | 76 | 77 | org.testcontainers 78 | junit-jupiter 79 | ${testcontainers.version} 80 | test 81 | 82 | 83 | org.testcontainers 84 | kafka 85 | ${testcontainers.version} 86 | test 87 | 88 | 89 | 90 | 91 | chapter7 92 | 93 | 94 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter7/src/main/java/org/kafkainaction/consumer/AlertConsumer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.consumer; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRecords; 4 | import org.apache.kafka.clients.consumer.KafkaConsumer; 5 | import org.kafkainaction.model.Alert; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.time.Duration; 10 | import java.util.List; 11 | import java.util.Properties; 12 | 13 | public class AlertConsumer { 14 | 15 | final static Logger log = LoggerFactory.getLogger(AlertConsumer.class); 16 | 17 | @SuppressWarnings("resource") 18 | public ConsumerRecords getAlertMessages(Properties kaConsumerProperties) { 19 | KafkaConsumer consumer = new KafkaConsumer<>(kaConsumerProperties); 20 | consumer.subscribe(List.of("kinaction_alert")); 21 | return consumer.poll(Duration.ofMillis(2500)); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter7/src/main/java/org/kafkainaction/model/Alert.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.model; 2 | 3 | import java.io.Serializable; 4 | 5 | public class Alert implements Serializable { 6 | 7 | private final int alertId; 8 | private String stageId; 9 | private final String alertLevel; 10 | private final String alertMessage; 11 | 12 | public Alert(int alertId, String stageId, 13 | String alertLevel, String alertMessage) { //<1> 14 | this.alertId = alertId; 15 | this.stageId = stageId; 16 | this.alertLevel = alertLevel; 17 | this.alertMessage = alertMessage; 18 | } 19 | 20 | public int getAlertId() { 21 | return alertId; 22 | } 23 | 24 | public String getStageId() { 25 | return stageId; 26 | } 27 | 28 | public void setStageId(String stageId) { 29 | this.stageId = stageId; 30 | } 31 | 32 | public String getAlertLevel() { 33 | return alertLevel; 34 | } 35 | 36 | public String getAlertMessage() { 37 | return alertMessage; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter7/src/main/java/org/kafkainaction/partitioner/AlertLevelPartitioner.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.partitioner; 2 | 3 | import org.apache.kafka.clients.producer.Partitioner; 4 | import org.apache.kafka.common.Cluster; 5 | import org.apache.kafka.common.PartitionInfo; 6 | import org.kafkainaction.model.Alert; 7 | 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.Random; 11 | 12 | public class AlertLevelPartitioner implements Partitioner { //<1> 13 | 14 | public int partition(final String topic, 15 | final Object objectKey, 16 | final byte[] keyBytes, 17 | final Object value, 18 | final byte[] valueBytes, 19 | final Cluster cluster) { 20 | 21 | 22 | int criticalLevelPartition = findCriticalPartitionNumber(cluster, topic); 23 | 24 | return isCriticalLevel(((Alert) objectKey).getAlertLevel()) ? 25 | criticalLevelPartition : 26 | findRandomPartition(cluster, topic, objectKey); 27 | } 28 | 29 | public int findCriticalPartitionNumber(Cluster cluster, String topic) { 30 | //not using parameters but could if needed for your logic 31 | return 0; 32 | } 33 | 34 | public int findRandomPartition(Cluster cluster, String topic, Object objectKey) { 35 | //not using parameter objectKey but could if needed for your logic 36 | List partitionMetaList = 37 | cluster.availablePartitionsForTopic(topic); 38 | 39 | Random randomPart = new Random(); 40 | return randomPart.nextInt(partitionMetaList.size()); 41 | } 42 | 43 | public boolean isCriticalLevel(String level) { 44 | if (level.toUpperCase().contains("CRITICAL")) { 45 | return true; 46 | } else { 47 | return false; 48 | } 49 | } 50 | 51 | @Override 52 | public void close() { 53 | 54 | } 55 | 56 | @Override 57 | public void configure(final Map map) { 58 | 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter7/src/main/java/org/kafkainaction/producer/AlertProducer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.producer; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.Producer; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.kafkainaction.model.Alert; 7 | 8 | import java.util.Properties; 9 | import java.util.concurrent.ExecutionException; 10 | 11 | public class AlertProducer { 12 | 13 | public void sendMessage(Properties kaProperties) throws InterruptedException, ExecutionException { 14 | kaProperties.put("partitioner.class", "org.kafkainaction.partitioner.AlertLevelPartitioner"); // <2> 15 | 16 | try (Producer producer = new KafkaProducer<>(kaProperties)) { 17 | Alert alert = new Alert(1, "Stage 1", "CRITICAL", "Stage 1 stopped"); 18 | ProducerRecord producerRecord = new ProducerRecord<>("kinaction_alert", alert, 19 | alert.getAlertMessage()); // <3> 20 | 21 | producer.send(producerRecord).get(); 22 | } 23 | 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter7/src/main/java/org/kafkainaction/serde/AlertKeySerde.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.serde; 2 | 3 | import org.apache.kafka.common.serialization.Deserializer; 4 | import org.apache.kafka.common.serialization.Serializer; 5 | import org.kafkainaction.model.Alert; 6 | 7 | import java.nio.charset.StandardCharsets; 8 | import java.util.Map; 9 | 10 | public class AlertKeySerde implements Serializer, 11 | Deserializer { //<1> 12 | 13 | public byte[] serialize(String topic, Alert key) { //<2> 14 | if (key == null) { 15 | return null; 16 | } 17 | return key.getStageId().getBytes(StandardCharsets.UTF_8); //<3> 18 | } 19 | 20 | public Alert deserialize(String topic, byte[] value) { //<4> 21 | return null; 22 | } 23 | 24 | @Override 25 | public void configure(final Map configs, final boolean isKey) { 26 | Serializer.super.configure(configs, isKey); 27 | } 28 | 29 | @Override 30 | public void close() { 31 | Serializer.super.close(); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter7/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %yellow(%d{yyyy-MM-dd HH:mm:ss}) %cyan(%logger{50}) %highlight([%p]) %green((%file:%line\)) - %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | %yellow(%d{yyyy-MM-dd HH:mm:ss}) %highlight([%p]) %magenta((%file:%line\)) - %msg%n 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter7/src/test/java/org/kafkainaction/producer/AlertLevelPartitionerTest.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.producer; 2 | 3 | import org.apache.kafka.clients.admin.AdminClient; 4 | import org.apache.kafka.clients.admin.AdminClientConfig; 5 | import org.apache.kafka.clients.admin.NewTopic; 6 | import org.apache.kafka.clients.consumer.ConsumerConfig; 7 | import org.apache.kafka.clients.consumer.ConsumerRecord; 8 | import org.apache.kafka.clients.consumer.ConsumerRecords; 9 | import org.apache.kafka.clients.consumer.KafkaConsumer; 10 | import org.apache.kafka.clients.producer.ProducerConfig; 11 | import org.apache.kafka.common.serialization.StringDeserializer; 12 | import org.apache.kafka.common.serialization.StringSerializer; 13 | import org.junit.ClassRule; 14 | import org.junit.Test; 15 | import org.kafkainaction.model.Alert; 16 | import org.kafkainaction.partitioner.AlertLevelPartitioner; 17 | import org.kafkainaction.serde.AlertKeySerde; 18 | import org.testcontainers.kafka.KafkaContainer; 19 | import org.testcontainers.utility.DockerImageName; 20 | 21 | import java.time.Duration; 22 | import java.util.Collections; 23 | import java.util.List; 24 | import java.util.Properties; 25 | import java.util.concurrent.ExecutionException; 26 | 27 | import static org.junit.Assert.assertEquals; 28 | 29 | public class AlertLevelPartitionerTest { 30 | 31 | private static final String TOPIC = "kinaction_alert"; 32 | private static final int PARTITION_NUMBER = 3; 33 | 34 | @ClassRule 35 | public static KafkaContainer kafka = new KafkaContainer( 36 | DockerImageName.parse("apache/kafka-native:3.8.0")); 37 | 38 | @Test 39 | public void testAlertPartitioner() throws ExecutionException, InterruptedException { 40 | // Create topic with 3 partitions 41 | createTopic(TOPIC, PARTITION_NUMBER); 42 | 43 | // Create producer properties 44 | Properties producerProps = new Properties(); 45 | producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers()); 46 | producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, AlertKeySerde.class.getName()); 47 | producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 48 | producerProps.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, AlertLevelPartitioner.class.getName()); 49 | 50 | // Use AlertProducer to send a critical alert 51 | AlertProducer criticalProducer = new AlertProducer(); 52 | criticalProducer.sendMessage(producerProps); 53 | 54 | // Create a custom producer for the non-critical alert 55 | // We need to do this because the default AlertProducer only sends a critical alert 56 | Alert nonCriticalAlert = new Alert(2, "Stage 2", "WARNING", "Stage 2 warning"); 57 | CustomAlertProducer nonCriticalProducer = new CustomAlertProducer(); 58 | nonCriticalProducer.sendCustomAlert(producerProps, nonCriticalAlert); 59 | 60 | // Create a consumer to verify the partitioning 61 | // We use StringDeserializer for both key and value since AlertKeySerde.deserialize returns null 62 | Properties consumerProps = new Properties(); 63 | consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers()); 64 | consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, "test-group"); 65 | consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); 66 | consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 67 | consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 68 | 69 | int criticalPartition = -1; 70 | int nonCriticalPartition = -1; 71 | 72 | try (KafkaConsumer consumer = new KafkaConsumer<>(consumerProps)) { 73 | consumer.subscribe(List.of(TOPIC)); 74 | ConsumerRecords records = consumer.poll(Duration.ofSeconds(10)); 75 | 76 | for (ConsumerRecord record : records) { 77 | if (record.value().equals("Stage 1 stopped")) { 78 | criticalPartition = record.partition(); 79 | } else if (record.value().equals("Stage 2 warning")) { 80 | nonCriticalPartition = record.partition(); 81 | } 82 | } 83 | } 84 | 85 | // Critical alerts should go to partition 0 86 | assertEquals("Critical alerts should go to partition 0", 0, criticalPartition); 87 | 88 | // Non-critical alerts should not go to partition 0 89 | assert nonCriticalPartition > 0 && nonCriticalPartition < PARTITION_NUMBER : 90 | "Non-critical alert partition should be between 1 and " + (PARTITION_NUMBER - 1); 91 | } 92 | 93 | private void createTopic(String topicName, int partitions) throws ExecutionException, InterruptedException { 94 | Properties props = new Properties(); 95 | props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers()); 96 | 97 | try (AdminClient adminClient = AdminClient.create(props)) { 98 | NewTopic newTopic = new NewTopic(topicName, partitions, (short) 1); 99 | adminClient.createTopics(Collections.singleton(newTopic)).all().get(); 100 | } 101 | } 102 | 103 | /** 104 | * A custom producer that allows sending a specific alert 105 | * This is needed because the default AlertProducer only sends a critical alert 106 | */ 107 | static class CustomAlertProducer extends AlertProducer { 108 | public void sendCustomAlert(Properties props, Alert alert) throws InterruptedException, ExecutionException { 109 | props.put("partitioner.class", "org.kafkainaction.partitioner.AlertLevelPartitioner"); 110 | 111 | try (org.apache.kafka.clients.producer.Producer producer = 112 | new org.apache.kafka.clients.producer.KafkaProducer<>(props)) { 113 | org.apache.kafka.clients.producer.ProducerRecord producerRecord = 114 | new org.apache.kafka.clients.producer.ProducerRecord<>(TOPIC, alert, alert.getAlertMessage()); 115 | producer.send(producerRecord).get(); 116 | producer.flush(); 117 | } 118 | } 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter8/Commands.md: -------------------------------------------------------------------------------- 1 | # Commands used in Chapter 8 2 | 3 | ## Flume configuration for watching a directory 4 | 5 | ```` 6 | ag.sources = logdir 7 | ag.sinks = kafkasink 8 | ag.channels = c1 9 | 10 | #Configure the source directory to watch 11 | ag.sources.logdir.type = spooldir 12 | ag.sources.logdir.spoolDir = /var/log/kafkainactionlogs 13 | ag.sources.logdir.fileHeader = true 14 | 15 | # Describe the Kafka sink 16 | ag.sinks.kafkasink.channel = c1 17 | ag.sinks.kafkasink.type = org.apache.flume.sink.kafka.KafkaSink 18 | ag.sinks.kafkasink.kafka.topic = kinaction_flumetopic 19 | ag.sinks.kafkasink.kafka.bootstrap.servers = localhost:9092,localhost:9093,localhost:9094 20 | ag.sinks.kafkasink.kafka.flumeBatchSize = 10 21 | ag.sinks.kafkasink.kafka.producer.acks = 1 22 | ag.sinks.kafkasink.kafka.producer.linger.ms = 5 23 | ag.sinks.kafkasink.kafka.producer.compression.type = snappy 24 | 25 | # Memory channel configuration 26 | ag.channels.c1.type = memory 27 | ag.channels.c1.capacity = 1000 28 | ag.channels.c1.transactionCapacity = 100 29 | 30 | # Bind both the sink and source to the same channel 31 | ag.sources.logdir.channels = c1 ag.sinks.kafkasink.channel = c1 32 | ```` 33 | 34 | 35 | ## Flume Kafka Channel configuration 36 | ```` 37 | ag.channels.channel1.type = org.apache.flume.channel.kafka.KafkaChannel 38 | ag.channels.channel1.kafka.bootstrap.servers = localhost:9092,localhost:9093,localhost:9094 39 | ag.channels.channel1.kafka.topic = kinaction_channel1_ch 40 | ag.channels.channel1.kafka.consumer.group.id = kinaction_flume 41 | 42 | ```` 43 | 44 | 45 | ## Reference 46 | * https://flume.apache.org/releases/content/1.9.0/FlumeUserGuide.html#kafka-sink 47 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter8/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | Kafka-In-Action 6 | org.kafkainaction 7 | 1.0.0-SNAPSHOT 8 | 9 | 10 | chapter8-no-java 11 | Kafka Storage 12 | 13 | 14 | chapter8 15 | 16 | 17 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter9/Commands.md: -------------------------------------------------------------------------------- 1 | # Commands used in Chapter 9 2 | 3 | ## Creating the selfserviceTopic Topic 4 | 5 | bin/kafka-topics.sh --create --bootstrap-server localhost:9094 --topic kinaction_selfserviceTopic --partitions 2 --replication-factor 2 6 | 7 | You should see the ouput: `Created topic "kinaction_selfserviceTopic".` 8 | 9 | ## kcat Producer usage 10 | 11 | kcat -P -b localhost:9094 -t kinaction_selfserviceTopic 12 | 13 | ## kcat Consumer usage 14 | 15 | kcat -C -b localhost:9094 -t kinaction_selfserviceTopic 16 | 17 | ## REST Proxy startup 18 | 19 | bin/kafka-rest-start.sh etc/kafka-rest/kafka-rest.properties 20 | 21 | ## Curl call to REST Proxy for Topic list 22 | 23 | curl -X GET -H "Accept: application/vnd.kafka.v2+json" localhost:8082/topics 24 | 25 | ## Zookeeper Unit File 26 | 27 | [Unit] 28 | Requires=network.target remote-fs.target 29 | After=network.target remote-fs.target 30 | 31 | [Service] 32 | Type=simple 33 | ExecStart=/opt/kafkainaction/bin/zookeeper-server-start.sh /opt/kafkainaction/config/zookeeper.properties 34 | ExecStop=/opt/kafkainaction/bin/zookeeper-server-stop.sh 35 | Restart=on-abnormal 36 | 37 | [Install] 38 | WantedBy=multi-user.target 39 | 40 | ## Kafka Unit File 41 | 42 | [Unit] 43 | Requires=zookeeper.service 44 | After=zookeeper.service 45 | 46 | [Service] 47 | Type=simple 48 | ExecStart=/opt/kafkainaction/bin/kafka-server-start.sh \ 49 | /opt/kafkainaction/bin/config/server.properties > /opt/kafkainaction/broker.log 2>&1' 50 | ExecStop=/opt/kafkainaction/bin/kafka-server-stop.sh 51 | Restart=on-abnormal 52 | 53 | [Install] 54 | WantedBy=multi-user.target 55 | 56 | ## Kafka Startup with Systemctl 57 | 58 | sudo systemctl start zookeeper 59 | sudo systemctl start kafka 60 | 61 | ## Kafka server log retention 62 | 63 | log4j.appender.kafkaAppender.MaxFileSize=500KB 64 | log4j.appender.kafkaAppender.MaxBackupIndex=10 65 | 66 | ## Which Kafka Appender 67 | *Note: Make sure that you check for any recent log4j updates 68 | 69 | log4j.appender.kafkaAppender=org.apache.kafka.log4jappender.KafkaLog4jAppender 70 | 71 | 72 | 73 | org.apache.kafka 74 | kafka-log4j-appender 75 | LATEST 76 | 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter9/pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 4.0.0 5 | 6 | 7 | Kafka-In-Action 8 | org.kafkainaction 9 | 1.0.0-SNAPSHOT 10 | 11 | 12 | chapter9-no-java 13 | Management: Tools and logging 14 | 15 | 16 | chapter9 17 | 18 | 19 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter9/src/main/java/org/kafkainaction/callback/AlertCallback.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.callback; 2 | 3 | import org.apache.kafka.clients.producer.Callback; 4 | import org.apache.kafka.clients.producer.RecordMetadata; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | public class AlertCallback implements Callback { 9 | final static Logger log = LoggerFactory.getLogger(AlertCallback.class); 10 | 11 | public void onCompletion(RecordMetadata metadata, Exception exception) { 12 | if (exception != null) { 13 | log.info("kinaction_error " + exception.getMessage()); 14 | } 15 | 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter9/src/main/java/org/kafkainaction/consumer/AlertConsumer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.consumer; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRecord; 4 | import org.apache.kafka.clients.consumer.KafkaConsumer; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.time.Duration; 9 | import java.util.List; 10 | import java.util.Properties; 11 | 12 | public class AlertConsumer { 13 | 14 | final static Logger log = LoggerFactory.getLogger(AlertConsumer.class); 15 | 16 | public static void main(String[] args) { 17 | Properties kaProperties = new Properties(); 18 | kaProperties.put("bootstrap.servers", "localhost:9092,localhost:9093"); 19 | kaProperties.put("group.id", "kinaction_alertinterceptor"); 20 | kaProperties.put("enable.auto.commit", "true"); 21 | kaProperties.put("auto.commit.interval.ms", "1000"); 22 | kaProperties.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 23 | kaProperties.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 24 | kaProperties.put("interceptor.classes", AlertConsumerMetricsInterceptor.class.getName()); 25 | 26 | try (KafkaConsumer consumer = new KafkaConsumer<>(kaProperties)) { 27 | consumer.subscribe(List.of("kinaction_alert")); 28 | 29 | while (true) { 30 | var records = consumer.poll(Duration.ofMillis(250)); 31 | for (ConsumerRecord record : records) { 32 | log.info("kinaction_info offset = {}, value = {}", record.offset(), record.value()); 33 | } 34 | } 35 | } 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter9/src/main/java/org/kafkainaction/consumer/AlertConsumerMetricsInterceptor.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.consumer; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerInterceptor; 4 | import org.apache.kafka.clients.consumer.ConsumerRecord; 5 | import org.apache.kafka.clients.consumer.ConsumerRecords; 6 | import org.apache.kafka.clients.consumer.OffsetAndMetadata; 7 | import org.apache.kafka.common.TopicPartition; 8 | import org.apache.kafka.common.header.Header; 9 | import org.apache.kafka.common.header.Headers; 10 | import org.kafkainaction.model.Alert; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.util.Map; 15 | 16 | @SuppressWarnings("unused") 17 | public class AlertConsumerMetricsInterceptor implements ConsumerInterceptor { 18 | 19 | final static Logger log = LoggerFactory.getLogger(AlertConsumerMetricsInterceptor.class); 20 | 21 | public ConsumerRecords 22 | onConsume(ConsumerRecords records) { 23 | if (records.isEmpty()) { 24 | return records; 25 | } else { 26 | for (ConsumerRecord record : records) { 27 | Headers headers = record.headers(); // <2> 28 | for (Header header : headers) { 29 | if ("kinactionTraceId".equals(header.key())) { // <3> 30 | log.info("kinactionTraceId is: " + new String(header.value())); 31 | } 32 | } 33 | } 34 | } 35 | return records; // <4> 36 | } 37 | 38 | @Override 39 | public void onCommit(final Map map) { 40 | 41 | } 42 | 43 | @Override 44 | public void close() { 45 | 46 | } 47 | 48 | @Override 49 | public void configure(final Map map) { 50 | 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter9/src/main/java/org/kafkainaction/model/Alert.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.model; 2 | 3 | import java.io.Serializable; 4 | 5 | public class Alert implements Serializable { 6 | 7 | private static final long serialVersionUID = 1L; 8 | 9 | private int alertId; 10 | private String stageId; 11 | private String alertLevel; 12 | private String alertMessage; 13 | 14 | public Alert(int alertId, String stageId, String alertLevel, String alertMessage) { 15 | this.alertId = alertId; 16 | this.stageId = stageId; 17 | this.alertLevel = alertLevel; 18 | this.alertMessage = alertMessage; 19 | } 20 | 21 | public int getAlertId() { 22 | return alertId; 23 | } 24 | 25 | public String getStageId() { 26 | return stageId; 27 | } 28 | 29 | public void setStageId(String stageId) { 30 | this.stageId = stageId; 31 | } 32 | 33 | public String getAlertLevel() { 34 | return alertLevel; 35 | } 36 | 37 | public String getAlertMessage() { 38 | return alertMessage; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter9/src/main/java/org/kafkainaction/producer/AlertProducer.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.producer; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.Producer; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.kafkainaction.callback.AlertCallback; 7 | import org.kafkainaction.model.Alert; 8 | 9 | import java.util.Properties; 10 | 11 | public class AlertProducer { 12 | 13 | public static void main(String[] args) { 14 | 15 | Properties kaProperties = new Properties(); 16 | kaProperties.put("bootstrap.servers", "localhost:9092,localhost:9093"); 17 | kaProperties.put("key.serializer", "org.kafkainaction.serde.AlertKeySerde"); 18 | kaProperties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 19 | kaProperties.put("interceptor.classes", "org.kafkainaction.producer.AlertProducerMetricsInterceptor"); 20 | 21 | try (Producer producer = new KafkaProducer(kaProperties)) { 22 | 23 | Alert alert = new Alert(1, "Stage 1", "CRITICAL", "Stage 1 stopped"); 24 | var producerRecord = new ProducerRecord<>("kinaction_alert", alert, alert.getAlertMessage()); //<1> 25 | producer.send(producerRecord, new AlertCallback()); 26 | } 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter9/src/main/java/org/kafkainaction/producer/AlertProducerMetricsInterceptor.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.producer; 2 | 3 | import org.apache.kafka.clients.producer.ProducerInterceptor; 4 | import org.apache.kafka.clients.producer.ProducerRecord; 5 | import org.apache.kafka.clients.producer.RecordMetadata; 6 | import org.apache.kafka.common.header.Headers; 7 | import org.kafkainaction.model.Alert; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.util.Map; 12 | import java.util.UUID; 13 | 14 | @SuppressWarnings("unused") 15 | public class AlertProducerMetricsInterceptor implements ProducerInterceptor { //<1> 16 | final static Logger log = LoggerFactory.getLogger(AlertProducerMetricsInterceptor.class); 17 | 18 | public ProducerRecord onSend(ProducerRecord record) { //<2> 19 | Headers headers = record.headers(); 20 | String kinactionTraceId = UUID.randomUUID().toString(); 21 | headers.add("kinactionTraceId", kinactionTraceId.getBytes()); //<3> 22 | log.info("kinaction_info Created kinactionTraceId: {}", kinactionTraceId); 23 | return record; //<4> 24 | } 25 | 26 | public void onAcknowledgement(RecordMetadata metadata, Exception exception) { //<5> 27 | if (exception != null) { 28 | log.info("kinaction_error " + exception.getMessage()); 29 | } else { 30 | log.info("kinaction_info topic = {} offset = {}", 31 | metadata.topic(), metadata.offset()); 32 | } 33 | } 34 | 35 | @Override 36 | public void close() { 37 | 38 | } 39 | 40 | @Override 41 | public void configure(final Map map) { 42 | 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter9/src/main/java/org/kafkainaction/serde/AlertKeySerde.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.serde; 2 | 3 | import org.apache.kafka.common.serialization.Deserializer; 4 | import org.apache.kafka.common.serialization.Serializer; 5 | import org.kafkainaction.model.Alert; 6 | 7 | import java.nio.charset.StandardCharsets; 8 | import java.util.Map; 9 | 10 | public class AlertKeySerde implements Serializer, 11 | Deserializer { //<1> 12 | 13 | public byte[] serialize(String topic, Alert key) { //<2> 14 | if (key == null) { 15 | return null; 16 | } 17 | return key.getStageId().getBytes(StandardCharsets.UTF_8); //<3> 18 | } 19 | 20 | public Alert deserialize(String topic, byte[] value) { //<4> 21 | return null; 22 | } 23 | 24 | @Override 25 | public void configure(final Map configs, final boolean isKey) { 26 | Serializer.super.configure(configs, isKey); 27 | } 28 | 29 | @Override 30 | public void close() { 31 | Serializer.super.close(); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter9/src/main/java/org/kafkainaction/util/CreateTopic.java: -------------------------------------------------------------------------------- 1 | package org.kafkainaction.util; 2 | 3 | import org.apache.kafka.clients.admin.AdminClient; 4 | import org.apache.kafka.clients.admin.CreateTopicsResult; 5 | import org.apache.kafka.clients.admin.NewTopic; 6 | import org.apache.kafka.common.errors.TopicExistsException; 7 | 8 | import java.util.List; 9 | import java.util.Properties; 10 | import java.util.concurrent.ExecutionException; 11 | 12 | public class CreateTopic { 13 | 14 | public static void main(String[] args) { 15 | 16 | Properties kaProperties = new Properties(); 17 | kaProperties.put("bootstrap.servers", "localhost:9092,localhost:9093"); 18 | kaProperties.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 19 | kaProperties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 20 | NewTopic requestedTopic = new NewTopic("kinaction_selfserviceTopic", 2, (short) 2); 21 | AdminClient client = AdminClient.create(kaProperties); 22 | CreateTopicsResult topicResult = client.createTopics(List.of(requestedTopic)); 23 | try { 24 | topicResult.values().get("kinaction_selfserviceTopic").get(); 25 | } catch (InterruptedException e) { 26 | e.printStackTrace(); 27 | } catch (ExecutionException e) { 28 | if (!(e.getCause() instanceof TopicExistsException)) { 29 | throw new RuntimeException(e.getMessage(), e); 30 | } 31 | e.printStackTrace(); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /KafkaInAction_Chapter9/src/monitoring-interceptors-5.2.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kafka-In-Action-Book/Kafka-In-Action-Source-Code/1d588f0d76eef05b802dc442518b651cf12e80df/KafkaInAction_Chapter9/src/monitoring-interceptors-5.2.2.jar -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Confluent Platform 7.9.0 smoke test 2 | 3 | # Define colors 4 | GREEN := \033[0;32m 5 | YELLOW := \033[0;33m 6 | BLUE := \033[0;34m 7 | MAGENTA := \033[0;35m 8 | CYAN := \033[0;36m 9 | RED := \033[0;31m 10 | NC := \033[0m # No Color 11 | 12 | .PHONY: smoke-test setup test teardown 13 | 14 | # Run the full smoke test (setup, test, teardown) 15 | smoke-test: setup test teardown 16 | @echo "$(GREEN)✅ Smoke test completed successfully!$(NC)" 17 | @echo "$(MAGENTA)🎉 All Confluent Platform 7.9.0 components are working properly!$(NC)" 18 | 19 | # Start the Docker Compose environment 20 | setup: 21 | @echo "$(BLUE)🚀 Starting Docker Compose environment...$(NC)" 22 | @docker-compose up -d 23 | @echo "$(YELLOW)⏳ Waiting for services to be ready with exponential retry...$(NC)" 24 | @./wait-for-services.sh 25 | 26 | # Run the smoke test 27 | test: 28 | @echo "$(CYAN)🦓 Checking Zookeeper...$(NC)" 29 | @docker exec zookeeper bash -c "echo ruok | nc localhost 2181" 30 | 31 | @echo "$(CYAN)🔍 Checking Kafka brokers...$(NC)" 32 | @docker exec broker1 bash -c "kafka-topics --bootstrap-server broker1:29092 --list" 33 | 34 | @echo "$(CYAN)📝 Creating test topic...$(NC)" 35 | @docker exec broker1 bash -c "kafka-topics --bootstrap-server broker1:29092 --create --topic smoke-test --partitions 3 --replication-factor 3" 36 | 37 | @echo "$(CYAN)📤 Producing test message...$(NC)" 38 | @docker exec broker1 bash -c "echo 'Smoke test message' | kafka-console-producer --bootstrap-server broker1:29092 --topic smoke-test" 39 | 40 | @echo "$(CYAN)📥 Consuming test message...$(NC)" 41 | @docker exec broker1 bash -c "kafka-console-consumer --bootstrap-server broker1:29092 --topic smoke-test --from-beginning --max-messages 1 --timeout-ms 10000" 42 | 43 | @echo "$(CYAN)📋 Checking Schema Registry...$(NC)" 44 | @docker exec schema-registry bash -c "curl -s http://schema-registry:8081/subjects" 45 | 46 | @echo "$(CYAN)🔎 Checking ksqlDB...$(NC)" 47 | @docker exec ksqldb-server bash -c "curl -s http://ksqldb-server:8088/info" 48 | 49 | @echo "$(CYAN)🧹 Cleaning up...$(NC)" 50 | @docker exec broker1 bash -c "kafka-topics --bootstrap-server broker1:29092 --delete --topic smoke-test" 51 | 52 | # Stop and remove the Docker Compose environment 53 | teardown: 54 | @echo "$(RED)🛑 Shutting down Docker Compose environment...$(NC)" 55 | @docker-compose down 56 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Source Code for Kafka in Action 2 | 3 | ## Most up-to-date location 4 | * While the source code might be included as a zip file from the Manning website, the location that will likely be the most up-to-date will be located at https://github.com/Kafka-In-Action-Book/Kafka-In-Action-Source-Code. The authors recommend referring to that site rather than the Manning zip location if you have a choice. 5 | 6 | 7 | ### Errata 8 | 9 | * If you happen to find errata, one option is to look at: https://github.com/Kafka-In-Action-Book/Kafka-In-Action-Source-Code/blob/master/errata.md 10 | * This is not the official Manning site errata list for the book - but please feel free to create a pull request to share any errata that you wish to share to help others. 11 | 12 | ## Security Concerns 13 | * Please check out the following to stay up-to-date on any security issues. The code in this project is NOT production ready and dependencies might have VULNERABILITIES that evolve over time. 14 | * https://kafka.apache.org/cve-list 15 | 16 | 17 | ## Notes 18 | 19 | Here are some notes regarding the source code: 20 | 21 | 1. Select shell commands will be presented in a [Markdown](https://daringfireball.net/projects/markdown/syntax) format in a file called Commands.md or in [AsciiDoc](https://docs.asciidoctor.org/asciidoc/latest/) called Commands.adoc for each Chapter folder if there are any commands selected in that chapter. 22 | 2. Not all commands and snippets of code are included here that were in the book material. As a beginner book, some sections were meant only to give an idea of the general process and not complete examples. 23 | 24 | ### Requirements 25 | This project was built with the following versions: 26 | 27 | 1. Java 21 (also compatible with Java 11 and 17) 28 | 2. Apache Maven 3.6.x. 29 | We provide [Maven Wrapper](https://github.com/takari/maven-wrapper), so you don't need to install Maven yourself. 30 | 31 | ### Dependency Management 32 | This project uses [Renovate](https://docs.renovatebot.com/) to keep dependencies up to date. Renovate will automatically create pull requests to update dependencies according to the configuration in `renovate.json`. 33 | 34 | The following dependency groups are managed by Renovate: 35 | - Kafka dependencies 36 | - Avro dependencies 37 | - Confluent dependencies 38 | - Logback dependencies 39 | - JUnit dependencies 40 | 41 | ### Continuous Integration 42 | This project uses GitHub Actions to run tests on multiple Java versions (11, 17, and 21). The workflow is defined in `.github/workflows/maven.yml`. Each push to the `master` branch and each pull request will trigger the workflow to build and test the project with all supported Java versions. 43 | 44 | ### How to build 45 | 46 | Run following command in the root of this project to build all examples: 47 | 48 | ./mvnw verify 49 | 50 | Run the following command in the root of this project to build a specific example. 51 | For example, to build only example from Chapter 12, run: 52 | 53 | ./mvnw --projects KafkaInAction_Chapter12 verify 54 | 55 | ### IDE setup 56 | 57 | 1. We have used Eclipse for our IDE. To set up for eclipse run mvn eclipse:eclipse from the base directory of this repo. Or, you can Import->Existing Maven Projects. 58 | 59 | 60 | ### Installing Kafka 61 | Run the following in a directory (without spaces in the path) once you get the artifact downloaded. Refer to Appendix A if needed. 62 | 63 | tar -xzf kafka_2.13-2.7.1.tgz 64 | cd kafka_2.13-2.7.1 65 | 66 | ### Running Kafka 67 | #### Option 1: Manual Setup 68 | 1. To start Kafka go to /kafka_2.13-2.7.1/ 69 | 2. Run bin/zookeeper-server-start.sh config/zookeeper.properties 70 | 3. Modify the Kafka server configs 71 | 72 | ```` 73 | cp config/server.properties config/server0.properties 74 | cp config/server.properties config/server1.properties 75 | cp config/server.properties config/server2.properties 76 | ```` 77 | 78 | vi config/server0.properties 79 | ```` 80 | broker.id=0 81 | listeners=PLAINTEXT://localhost:9092 82 | log.dirs=/tmp/kafkainaction/kafka-logs-0 83 | ```` 84 | 85 | vi config/server1.properties 86 | 87 | ```` 88 | broker.id=1 89 | listeners=PLAINTEXT://localhost:9093 90 | log.dirs=/tmp/kafkainaction/kafka-logs-1 91 | ```` 92 | 93 | vi config/server2.properties 94 | 95 | ```` 96 | broker.id=2 97 | listeners=PLAINTEXT://localhost:9094 98 | log.dirs=/tmp/kafkainaction/kafka-logs-2 99 | ```` 100 | 101 | 4. Start the Kafka Brokers: 102 | 103 | ```` 104 | bin/kafka-server-start.sh config/server0.properties 105 | bin/kafka-server-start.sh config/server1.properties 106 | bin/kafka-server-start.sh config/server2.properties 107 | ```` 108 | 109 | #### Option 2: Docker Compose with Confluent Platform 7.9.0 110 | 111 | This project includes a Docker Compose file that sets up a complete Confluent Platform 7.9.0 environment, including: 112 | - Zookeeper 113 | - 3 Kafka brokers 114 | - Schema Registry 115 | - ksqlDB server and CLI 116 | 117 | To use Docker Compose: 118 | 119 | 1. Make sure you have Docker and Docker Compose installed 120 | 2. Run the following command from the project root: 121 | ``` 122 | docker-compose up -d 123 | ``` 124 | 3. To stop the environment: 125 | ``` 126 | docker-compose down 127 | ``` 128 | 129 | ##### Smoke Test 130 | 131 | A Makefile is included to validate the Docker Compose setup: 132 | 133 | ``` 134 | make smoke-test 135 | ``` 136 | 137 | The Makefile provides the following targets: 138 | - `smoke-test`: Run the full smoke test (setup, test, teardown) - **This is the default target** 139 | - `setup`: Start the Docker Compose environment and wait for services to be ready (with exponential retry) 140 | - `test`: Run the smoke test (check services, create topic, produce/consume messages) 141 | - `teardown`: Stop and remove the Docker Compose environment 142 | 143 | You can also run individual steps: 144 | 145 | ``` 146 | make setup # Start the environment 147 | make test # Run tests only 148 | make teardown # Stop and clean up 149 | ``` 150 | 151 | The smoke test is integrated into the GitHub Actions workflow and runs automatically on each push to the master branch and on pull requests. 152 | 153 | ### Stopping Kafka 154 | 155 | 1. To stop Kafka go to the Kafka directory install location 156 | 2. Run bin/kafka-server-stop.sh 157 | 3. Run bin/zookeeper-server-stop.sh 158 | 159 | ### Code by Chapter 160 | Most of the code from the book can be found in the project corresponding to the chapter. 161 | Some code has been moved to other chapters in order to reduce the number of replication of related classes. 162 | 163 | ### Running the examples 164 | 165 | Most of the example programs can be run from within an IDE or from the command line. Make sure that your ZooKeeper and Kafka Brokers are up and running before you can run any of the examples. 166 | 167 | The examples will usually write out to topics and print to the console. 168 | 169 | ### Shell Scripts 170 | 171 | In the Chapter 2 project, we have included a couple of scripts if you want to use them under `src/main/resources`. 172 | 173 | They include: 174 | * `starteverything.sh` //This will start your ZooKeeper and Kafka Brokers (you will still have to go through the first time setup with Appendix A before using this.) 175 | * stopeverything.sh // Will stop ZooKeeper and your brokers 176 | * portInUse.sh // If you get a port in use error on startup, this script will kill all of the processes using those ports (assuming you are using the same ports as in Appendix A setup). 177 | 178 | ## Disclaimer 179 | 180 | The author and publisher have made every effort to ensure that the information in this book was correct at press time. The author and publisher do not assume and hereby disclaim any 181 | liability to any party for any loss, damage, or disruption caused by errors or omissions, whether 182 | such errors or omissions result from negligence, accident, or any other cause, or from any usage 183 | of the information herein. Note: The information in this book also refers to and includes the source code found here. 184 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | zookeeper: 3 | image: confluentinc/cp-zookeeper:7.9.0 4 | hostname: zookeeper 5 | container_name: zookeeper 6 | ports: 7 | - "2181:2181" 8 | environment: 9 | ZOOKEEPER_CLIENT_PORT: 2181 10 | ZOOKEEPER_TICK_TIME: 2000 11 | KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=*" 12 | 13 | broker1: 14 | image: confluentinc/cp-kafka:7.9.0 15 | hostname: broker1 16 | container_name: broker1 17 | depends_on: 18 | - zookeeper 19 | ports: 20 | - "29092:29092" 21 | - "9092:9092" 22 | - "9101:9101" 23 | environment: 24 | KAFKA_BROKER_ID: 1 25 | KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' 26 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT 27 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker1:29092,PLAINTEXT_HOST://localhost:9092 28 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 29 | KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 3 30 | KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 31 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 32 | KAFKA_JMX_PORT: 9101 33 | KAFKA_JMX_HOSTNAME: localhost 34 | 35 | broker2: 36 | image: confluentinc/cp-kafka:7.9.0 37 | hostname: broker2 38 | container_name: broker2 39 | depends_on: 40 | - zookeeper 41 | ports: 42 | - "29093:29093" 43 | - "9093:9093" 44 | - "9201:9201" 45 | environment: 46 | KAFKA_BROKER_ID: 2 47 | KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' 48 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT 49 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker2:29093,PLAINTEXT_HOST://localhost:9093 50 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 51 | KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 3 52 | KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 53 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 54 | KAFKA_JMX_PORT: 9201 55 | KAFKA_JMX_HOSTNAME: localhost 56 | 57 | broker3: 58 | image: confluentinc/cp-kafka:7.9.0 59 | hostname: broker3 60 | container_name: broker3 61 | depends_on: 62 | - zookeeper 63 | ports: 64 | - "29094:29094" 65 | - "9094:9094" 66 | - "9301:9301" 67 | environment: 68 | KAFKA_BROKER_ID: 3 69 | KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' 70 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT 71 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker3:29094,PLAINTEXT_HOST://localhost:9094 72 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 73 | KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 3 74 | KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 75 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 76 | KAFKA_JMX_PORT: 9301 77 | KAFKA_JMX_HOSTNAME: localhost 78 | 79 | schema-registry: 80 | image: confluentinc/cp-schema-registry:7.9.0 81 | hostname: schema-registry 82 | container_name: schema-registry 83 | depends_on: 84 | - broker1 85 | - broker2 86 | - broker3 87 | ports: 88 | - "8081:8081" 89 | environment: 90 | SCHEMA_REGISTRY_HOST_NAME: schema-registry 91 | SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker1:29092' 92 | SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 93 | 94 | ksqldb-server: 95 | image: confluentinc/cp-ksqldb-server:7.9.0 96 | hostname: ksqldb-server 97 | container_name: ksqldb-server 98 | depends_on: 99 | - broker1 100 | - broker2 101 | - broker3 102 | - schema-registry 103 | ports: 104 | - "8088:8088" 105 | environment: 106 | KSQL_CONFIG_DIR: "/etc/ksqldb" 107 | KSQL_KSQL_EXTENSION_DIR: "/etc/ksqldb/ext/" 108 | KSQL_LOG4J_OPTS: "-Dlog4j.configuration=file:/etc/ksqldb/log4j.properties" 109 | KSQL_BOOTSTRAP_SERVERS: "broker1:29092" 110 | KSQL_HOST_NAME: ksqldb-server 111 | KSQL_LISTENERS: "http://0.0.0.0:8088" 112 | KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true" 113 | KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true" 114 | KSQL_CACHE_MAX_BYTES_BUFFERING: 0 115 | KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" 116 | 117 | ksqldb-cli: 118 | image: confluentinc/cp-ksqldb-cli:7.9.0 119 | container_name: ksqldb-cli 120 | depends_on: 121 | - broker1 122 | - ksqldb-server 123 | entrypoint: /bin/sh 124 | tty: true 125 | environment: 126 | KSQL_CONFIG_DIR: "/etc/ksqldb" 127 | -------------------------------------------------------------------------------- /errata.md: -------------------------------------------------------------------------------- 1 | ## Errata 2 | 3 | This is not the official Manning site errata list for the book - but please feel free to create a pull request to share any errata that you wish to share to help others. 4 | 5 | Example: 6 | 7 | ## Chapter X - This line had a typo, etc.... 8 | -------------------------------------------------------------------------------- /licenses/ApacheFlumeNotice.txt: -------------------------------------------------------------------------------- 1 | Apache Flume 2 | Copyright 2012 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | 7 | Portions of this software were developed at 8 | Cloudera, Inc. (http://www.cloudera.com/). 9 | -------------------------------------------------------------------------------- /licenses/ApacheKafkaNotice.txt: -------------------------------------------------------------------------------- 1 | Apache Kafka 2 | Copyright 2021 The Apache Software Foundation. 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (https://www.apache.org/). 6 | 7 | This distribution has a binary dependency on jersey, which is available under the CDDL 8 | License. The source code of jersey can be found at https://github.com/jersey/jersey/. 9 | 10 | This distribution has a binary test dependency on jqwik, which is available under 11 | the Eclipse Public License 2.0. The source code can be found at 12 | https://github.com/jlink/jqwik. 13 | 14 | The streams-scala (streams/streams-scala) module was donated by Lightbend and the original code was copyrighted by them: 15 | Copyright (C) 2018 Lightbend Inc. 16 | Copyright (C) 2017-2018 Alexis Seigneurin. 17 | 18 | This project contains the following code copied from Apache Hadoop: 19 | clients/src/main/java/org/apache/kafka/common/utils/PureJavaCrc32C.java 20 | Some portions of this file Copyright (c) 2004-2006 Intel Corporation and licensed under the BSD license. 21 | 22 | This project contains the following code copied from Apache Hive: 23 | streams/src/main/java/org/apache/kafka/streams/state/internals/Murmur3.java 24 | -------------------------------------------------------------------------------- /licenses/Links.md: -------------------------------------------------------------------------------- 1 | # Links to Licenses 2 | 3 | ## Apache Kafka 4 | * https://github.com/apache/kafka/blob/trunk/LICENSE 5 | * https://github.com/apache/kafka/tree/trunk/licenses 6 | * https://github.com/apache/kafka/blob/trunk/NOTICE 7 | 8 | ## Apache Flume 9 | * https://github.com/apache/flume/blob/trunk/LICENSE 10 | * https://github.com/apache/flume/blob/trunk/NOTICE 11 | -------------------------------------------------------------------------------- /licenses/README.md: -------------------------------------------------------------------------------- 1 | # Licenses 2 | 3 | This folder includes some of the licenses of project source code that was referenced in the material. 4 | 5 | Please submit a pull request if you feel like we did not include a license or a change needs to be made for these licenses. 6 | -------------------------------------------------------------------------------- /mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM http://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM set title of command window 39 | title %0 40 | @REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' 41 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 42 | 43 | @REM set %HOME% to equivalent of $HOME 44 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 45 | 46 | @REM Execute a user defined script before this one 47 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 48 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 49 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 50 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 51 | :skipRcPre 52 | 53 | @setlocal 54 | 55 | set ERROR_CODE=0 56 | 57 | @REM To isolate internal variables from possible post scripts, we use another setlocal 58 | @setlocal 59 | 60 | @REM ==== START VALIDATION ==== 61 | if not "%JAVA_HOME%" == "" goto OkJHome 62 | 63 | echo. 64 | echo Error: JAVA_HOME not found in your environment. >&2 65 | echo Please set the JAVA_HOME variable in your environment to match the >&2 66 | echo location of your Java installation. >&2 67 | echo. 68 | goto error 69 | 70 | :OkJHome 71 | if exist "%JAVA_HOME%\bin\java.exe" goto init 72 | 73 | echo. 74 | echo Error: JAVA_HOME is set to an invalid directory. >&2 75 | echo JAVA_HOME = "%JAVA_HOME%" >&2 76 | echo Please set the JAVA_HOME variable in your environment to match the >&2 77 | echo location of your Java installation. >&2 78 | echo. 79 | goto error 80 | 81 | @REM ==== END VALIDATION ==== 82 | 83 | :init 84 | 85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 86 | @REM Fallback to current working directory if not found. 87 | 88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 90 | 91 | set EXEC_DIR=%CD% 92 | set WDIR=%EXEC_DIR% 93 | :findBaseDir 94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 95 | cd .. 96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 97 | set WDIR=%CD% 98 | goto findBaseDir 99 | 100 | :baseDirFound 101 | set MAVEN_PROJECTBASEDIR=%WDIR% 102 | cd "%EXEC_DIR%" 103 | goto endDetectBaseDir 104 | 105 | :baseDirNotFound 106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 107 | cd "%EXEC_DIR%" 108 | 109 | :endDetectBaseDir 110 | 111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 112 | 113 | @setlocal EnableExtensions EnableDelayedExpansion 114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 116 | 117 | :endReadAdditionalConfig 118 | 119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 120 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 121 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 122 | 123 | set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" 124 | 125 | FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( 126 | IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B 127 | ) 128 | 129 | @REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 130 | @REM This allows using the maven wrapper in projects that prohibit checking in binary data. 131 | if exist %WRAPPER_JAR% ( 132 | if "%MVNW_VERBOSE%" == "true" ( 133 | echo Found %WRAPPER_JAR% 134 | ) 135 | ) else ( 136 | if not "%MVNW_REPOURL%" == "" ( 137 | SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" 138 | ) 139 | if "%MVNW_VERBOSE%" == "true" ( 140 | echo Couldn't find %WRAPPER_JAR%, downloading it ... 141 | echo Downloading from: %DOWNLOAD_URL% 142 | ) 143 | 144 | powershell -Command "&{"^ 145 | "$webclient = new-object System.Net.WebClient;"^ 146 | "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ 147 | "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ 148 | "}"^ 149 | "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ 150 | "}" 151 | if "%MVNW_VERBOSE%" == "true" ( 152 | echo Finished downloading %WRAPPER_JAR% 153 | ) 154 | ) 155 | @REM End of extension 156 | 157 | @REM Provide a "standardized" way to retrieve the CLI args that will 158 | @REM work with both Windows and non-Windows executions. 159 | set MAVEN_CMD_LINE_ARGS=%* 160 | 161 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 162 | if ERRORLEVEL 1 goto error 163 | goto end 164 | 165 | :error 166 | set ERROR_CODE=1 167 | 168 | :end 169 | @endlocal & set ERROR_CODE=%ERROR_CODE% 170 | 171 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 172 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 173 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 174 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 175 | :skipRcPost 176 | 177 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 178 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 179 | 180 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 181 | 182 | exit /B %ERROR_CODE% 183 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | org.kafkainaction 5 | Kafka-In-Action 6 | pom 7 | 1.0.0-SNAPSHOT 8 | 9 | 10 | 11 11 | 3.9.0 12 | 2.13 13 | 1.11.1 14 | 7.9.0 15 | UTF-8 16 | 17 | 18 | 19 | 20 | apache-repo 21 | Apache Repository 22 | https://repository.apache.org/content/repositories/releases 23 | 24 | true 25 | 26 | 27 | false 28 | 29 | 30 | 31 | confluent 32 | https://packages.confluent.io/maven/ 33 | 34 | 35 | 36 | 37 | 38 | 39 | org.apache.kafka 40 | kafka-clients 41 | ${kafka.version} 42 | 43 | 44 | ch.qos.logback 45 | logback-classic 46 | 1.2.10 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | org.apache.maven.plugins 55 | maven-compiler-plugin 56 | 3.12.1 57 | 58 | ${maven.compiler.release} 59 | true 60 | 61 | -Xlint:deprecation 62 | -Werror 63 | 64 | 65 | 66 | 67 | org.apache.maven.plugins 68 | maven-assembly-plugin 69 | 3.6.0 70 | 71 | 72 | jar-with-dependencies 73 | 74 | 75 | 76 | 77 | make-assembly 78 | package 79 | 80 | single 81 | 82 | 83 | 84 | 85 | 86 | org.apache.maven.plugins 87 | maven-surefire-plugin 88 | 3.2.5 89 | 90 | 91 | org.apache.maven.plugins 92 | maven-resources-plugin 93 | 3.3.1 94 | 95 | 96 | org.apache.maven.plugins 97 | maven-jar-plugin 98 | 3.3.0 99 | 100 | 101 | org.apache.maven.plugins 102 | maven-clean-plugin 103 | 3.3.2 104 | 105 | 106 | org.apache.maven.plugins 107 | maven-install-plugin 108 | 3.1.1 109 | 110 | 111 | org.apache.maven.plugins 112 | maven-deploy-plugin 113 | 3.1.1 114 | 115 | 116 | org.apache.maven.plugins 117 | maven-site-plugin 118 | 4.0.0-M13 119 | 120 | 121 | org.apache.maven.plugins 122 | maven-dependency-plugin 123 | 3.6.1 124 | 125 | 126 | org.apache.maven.plugins 127 | maven-enforcer-plugin 128 | 3.4.1 129 | 130 | 131 | org.apache.maven.plugins 132 | maven-source-plugin 133 | 3.3.0 134 | 135 | 136 | org.apache.maven.plugins 137 | maven-javadoc-plugin 138 | 3.6.3 139 | 140 | 141 | org.apache.maven.plugins 142 | maven-release-plugin 143 | 3.0.1 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | java11 153 | 154 | 11 155 | 156 | 157 | 11 158 | 159 | 160 | 161 | java17 162 | 163 | 17 164 | 165 | 166 | 17 167 | 168 | 169 | 170 | java21 171 | 172 | 21 173 | true 174 | 175 | 176 | 21 177 | 178 | 179 | 180 | 181 | 182 | KafkaInAction_Chapter2 183 | KafkaInAction_Chapter3 184 | KafkaInAction_Chapter4 185 | KafkaInAction_Chapter5 186 | KafkaInAction_Chapter6 187 | KafkaInAction_Chapter7 188 | KafkaInAction_Chapter8 189 | KafkaInAction_Chapter9 190 | KafkaInAction_Chapter10 191 | KafkaInAction_Chapter11 192 | KafkaInAction_Chapter12 193 | KafkaInAction_AppendixB 194 | 195 | 196 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:recommended", 5 | ":dependencyDashboard" 6 | ], 7 | "packageRules": [ 8 | { 9 | "matchCategories": [ 10 | "java" 11 | ], 12 | "enabled": true 13 | }, 14 | { 15 | "matchUpdateTypes": [ 16 | "minor", 17 | "patch" 18 | ], 19 | "groupName": "all non-major dependencies", 20 | "groupSlug": "all-minor-patch", 21 | "matchPackageNames": [ 22 | "*" 23 | ] 24 | }, 25 | { 26 | "groupName": "Kafka dependencies", 27 | "groupSlug": "kafka", 28 | "matchPackageNames": [ 29 | "/^org\\.apache\\.kafka/" 30 | ] 31 | }, 32 | { 33 | "groupName": "Avro dependencies", 34 | "groupSlug": "avro", 35 | "matchPackageNames": [ 36 | "/^org\\.apache\\.avro/" 37 | ] 38 | }, 39 | { 40 | "groupName": "Confluent dependencies", 41 | "groupSlug": "confluent", 42 | "matchPackageNames": [ 43 | "/^io\\.confluent/" 44 | ] 45 | }, 46 | { 47 | "groupName": "Logback dependencies", 48 | "groupSlug": "logback", 49 | "matchPackageNames": [ 50 | "/^ch\\.qos\\.logback/" 51 | ] 52 | }, 53 | { 54 | "groupName": "JUnit dependencies", 55 | "groupSlug": "junit", 56 | "matchPackageNames": [ 57 | "/^org\\.junit/" 58 | ] 59 | } 60 | ], 61 | "labels": [ 62 | "dependencies" 63 | ], 64 | "prHourlyLimit": 0, 65 | "prConcurrentLimit": 0, 66 | "rangeStrategy": "bump", 67 | "semanticCommits": "enabled", 68 | "dependencyDashboard": true, 69 | "dependencyDashboardTitle": "Dependency Dashboard", 70 | "schedule": [ 71 | "every weekend" 72 | ], 73 | "maven": { 74 | "enabled": true 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /wait-for-services.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | echo "🔄 Waiting for services to be ready..." 5 | 6 | # Function to check if Zookeeper is ready 7 | check_zookeeper() { 8 | echo "🦓 Checking Zookeeper..." 9 | docker exec zookeeper bash -c "echo ruok | nc localhost 2181" | grep -q "imok" 10 | return $? 11 | } 12 | 13 | # Function to check if Kafka brokers are ready 14 | check_kafka() { 15 | echo "🔍 Checking Kafka brokers..." 16 | docker exec broker1 bash -c "kafka-topics --bootstrap-server broker1:29092 --list" &> /dev/null 17 | return $? 18 | } 19 | 20 | # Function to wait for a service with exponential backoff 21 | wait_for_service() { 22 | local service_name=$1 23 | local check_function=$2 24 | local max_attempts=10 25 | local timeout=1 26 | local attempt=1 27 | local exitCode=1 28 | 29 | while [[ $attempt -le $max_attempts ]] 30 | do 31 | echo "⏳ Attempt $attempt/$max_attempts: Waiting for $service_name..." 32 | 33 | if $check_function; then 34 | echo "✅ $service_name is ready!" 35 | return 0 36 | fi 37 | 38 | echo "⏳ $service_name is not ready yet. Retrying in $timeout seconds..." 39 | sleep $timeout 40 | 41 | # Exponential backoff with a maximum of 30 seconds 42 | timeout=$(( timeout * 2 )) 43 | if [[ $timeout -gt 30 ]]; then 44 | timeout=30 45 | fi 46 | 47 | attempt=$(( attempt + 1 )) 48 | done 49 | 50 | echo "❌ Failed to connect to $service_name after $max_attempts attempts." 51 | return 1 52 | } 53 | 54 | # Wait for Zookeeper 55 | wait_for_service "Zookeeper" check_zookeeper 56 | 57 | # Wait for Kafka 58 | wait_for_service "Kafka" check_kafka 59 | 60 | echo "✅ All services are ready!" --------------------------------------------------------------------------------