├── .gitignore ├── Chapter2 ├── 2.1 local-broker-windows │ ├── server.properties │ └── zookeeper.properties └── 2.1 local-broker │ ├── server.properties │ └── zookeeper.properties ├── Chapter3 ├── .DS_Store ├── 3.4.1 kafka-producer │ ├── .DS_Store │ ├── kafka-producer-async-callback │ │ ├── build.gradle │ │ ├── gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ ├── ProducerCallback.java │ │ │ └── ProducerWithAsyncCallback.java │ ├── kafka-producer-custom-partitioner │ │ ├── build.gradle │ │ ├── gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ ├── CustomPartitioner.java │ │ │ └── ProducerWithCustomPartitioner.java │ ├── kafka-producer-exact-partition │ │ ├── build.gradle │ │ ├── gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ └── ProducerExactPartition.java │ ├── kafka-producer-key-value │ │ ├── build.gradle │ │ ├── gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ └── ProducerWithKeyValue.java │ ├── kafka-producer-sync-callback │ │ ├── build.gradle │ │ ├── gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ └── ProducerWithSyncCallback.java │ └── simple-kafka-producer │ │ ├── build.gradle │ │ ├── gradle │ │ └── wrapper │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── settings.gradle │ │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── example │ │ └── SimpleProducer.java ├── 3.4.2 kafka-consumer │ ├── .DS_Store │ ├── kafka-consumer-async-commit │ │ ├── build.gradle │ │ ├── gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ └── ConsumerWithASyncCommit.java │ ├── kafka-consumer-auto-commit │ │ ├── build.gradle │ │ ├── gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ └── ConsumerWithAutoCommit.java │ ├── kafka-consumer-exact-partition │ │ ├── build.gradle │ │ ├── gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ └── ConsumerWithExactPartition.java │ ├── kafka-consumer-rebalance-listener │ │ ├── build.gradle │ │ ├── gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ ├── ConsumerWithRebalanceListener.java │ │ │ └── RebalanceListener.java │ ├── kafka-consumer-sync-commit │ │ ├── build.gradle │ │ ├── gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ └── ConsumerWithSyncCommit.java │ ├── kafka-consumer-sync-offset-commit-shutdown-hook │ │ ├── build.gradle │ │ ├── gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ └── ConsumerWithSyncOffsetCommit.java │ ├── kafka-consumer-sync-offset-commit │ │ ├── build.gradle │ │ ├── gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ └── ConsumerWithSyncOffsetCommit.java │ └── simple-kafka-consumer │ │ ├── build.gradle │ │ ├── gradle │ │ └── wrapper │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── settings.gradle │ │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── example │ │ └── SimpleConsumer.java ├── 3.4.3 kafka-admin │ ├── .DS_Store │ └── kafka-admin-client │ │ ├── .gradle │ │ ├── 6.1.1 │ │ │ ├── executionHistory │ │ │ │ ├── executionHistory.bin │ │ │ │ └── executionHistory.lock │ │ │ ├── fileChanges │ │ │ │ └── last-build.bin │ │ │ ├── fileContent │ │ │ │ └── fileContent.lock │ │ │ ├── fileHashes │ │ │ │ ├── fileHashes.bin │ │ │ │ └── fileHashes.lock │ │ │ ├── gc.properties │ │ │ └── javaCompile │ │ │ │ ├── classAnalysis.bin │ │ │ │ ├── javaCompile.lock │ │ │ │ └── taskHistory.bin │ │ ├── buildOutputCleanup │ │ │ ├── buildOutputCleanup.lock │ │ │ ├── cache.properties │ │ │ └── outputFiles.bin │ │ ├── checksums │ │ │ ├── checksums.lock │ │ │ ├── md5-checksums.bin │ │ │ └── sha1-checksums.bin │ │ └── vcs-1 │ │ │ └── gc.properties │ │ ├── build.gradle │ │ ├── settings.gradle │ │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── example │ │ └── KafkaAdminClient.java ├── 3.5 kafka-streams │ ├── .DS_Store │ ├── kafka-streams-filter │ │ ├── .gradle │ │ │ ├── 6.1.1 │ │ │ │ ├── executionHistory │ │ │ │ │ ├── executionHistory.bin │ │ │ │ │ └── executionHistory.lock │ │ │ │ ├── fileChanges │ │ │ │ │ └── last-build.bin │ │ │ │ ├── fileHashes │ │ │ │ │ ├── fileHashes.bin │ │ │ │ │ └── fileHashes.lock │ │ │ │ └── gc.properties │ │ │ ├── buildOutputCleanup │ │ │ │ ├── buildOutputCleanup.lock │ │ │ │ ├── cache.properties │ │ │ │ └── outputFiles.bin │ │ │ ├── checksums │ │ │ │ ├── checksums.lock │ │ │ │ ├── md5-checksums.bin │ │ │ │ └── sha1-checksums.bin │ │ │ └── vcs-1 │ │ │ │ └── gc.properties │ │ ├── build.gradle │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ └── StreamsFilter.java │ ├── kstream-count │ │ ├── .DS_Store │ │ ├── .gradle │ │ │ ├── 6.1.1 │ │ │ │ ├── executionHistory │ │ │ │ │ ├── executionHistory.bin │ │ │ │ │ └── executionHistory.lock │ │ │ │ ├── fileChanges │ │ │ │ │ └── last-build.bin │ │ │ │ ├── fileHashes │ │ │ │ │ ├── fileHashes.bin │ │ │ │ │ └── fileHashes.lock │ │ │ │ └── gc.properties │ │ │ ├── buildOutputCleanup │ │ │ │ ├── buildOutputCleanup.lock │ │ │ │ ├── cache.properties │ │ │ │ └── outputFiles.bin │ │ │ ├── checksums │ │ │ │ ├── checksums.lock │ │ │ │ ├── md5-checksums.bin │ │ │ │ └── sha1-checksums.bin │ │ │ └── vcs-1 │ │ │ │ └── gc.properties │ │ ├── build.gradle │ │ ├── settings.gradle │ │ └── src │ │ │ ├── .DS_Store │ │ │ └── main │ │ │ ├── .DS_Store │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ └── KStreamCountApplication.java │ ├── kstream-globalktable-join │ │ ├── .gradle │ │ │ ├── 6.1.1 │ │ │ │ ├── executionHistory │ │ │ │ │ ├── executionHistory.bin │ │ │ │ │ └── executionHistory.lock │ │ │ │ ├── fileChanges │ │ │ │ │ └── last-build.bin │ │ │ │ ├── fileHashes │ │ │ │ │ ├── fileHashes.bin │ │ │ │ │ └── fileHashes.lock │ │ │ │ └── gc.properties │ │ │ ├── buildOutputCleanup │ │ │ │ ├── buildOutputCleanup.lock │ │ │ │ ├── cache.properties │ │ │ │ └── outputFiles.bin │ │ │ ├── checksums │ │ │ │ ├── checksums.lock │ │ │ │ ├── md5-checksums.bin │ │ │ │ └── sha1-checksums.bin │ │ │ └── vcs-1 │ │ │ │ └── gc.properties │ │ ├── build.gradle │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ └── KStreamJoinGlobalKTable.java │ ├── kstream-ktable-join │ │ ├── .gradle │ │ │ ├── 6.1.1 │ │ │ │ ├── executionHistory │ │ │ │ │ ├── executionHistory.bin │ │ │ │ │ └── executionHistory.lock │ │ │ │ ├── fileChanges │ │ │ │ │ └── last-build.bin │ │ │ │ ├── fileHashes │ │ │ │ │ ├── fileHashes.bin │ │ │ │ │ └── fileHashes.lock │ │ │ │ └── gc.properties │ │ │ ├── buildOutputCleanup │ │ │ │ ├── buildOutputCleanup.lock │ │ │ │ ├── cache.properties │ │ │ │ └── outputFiles.bin │ │ │ ├── checksums │ │ │ │ ├── checksums.lock │ │ │ │ ├── md5-checksums.bin │ │ │ │ └── sha1-checksums.bin │ │ │ └── vcs-1 │ │ │ │ └── gc.properties │ │ ├── build.gradle │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ └── KStreamJoinKTable.java │ ├── queryable-store │ │ ├── .gradle │ │ │ ├── 6.1.1 │ │ │ │ ├── executionHistory │ │ │ │ │ ├── executionHistory.bin │ │ │ │ │ └── executionHistory.lock │ │ │ │ ├── fileChanges │ │ │ │ │ └── last-build.bin │ │ │ │ ├── fileHashes │ │ │ │ │ ├── fileHashes.bin │ │ │ │ │ └── fileHashes.lock │ │ │ │ └── gc.properties │ │ │ ├── buildOutputCleanup │ │ │ │ ├── buildOutputCleanup.lock │ │ │ │ ├── cache.properties │ │ │ │ └── outputFiles.bin │ │ │ ├── checksums │ │ │ │ ├── checksums.lock │ │ │ │ ├── md5-checksums.bin │ │ │ │ └── sha1-checksums.bin │ │ │ └── vcs-1 │ │ │ │ └── gc.properties │ │ ├── build.gradle │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ └── QueryableStore.java │ ├── simple-kafka-processor │ │ ├── .gradle │ │ │ ├── 6.1.1 │ │ │ │ ├── executionHistory │ │ │ │ │ ├── executionHistory.bin │ │ │ │ │ └── executionHistory.lock │ │ │ │ ├── fileChanges │ │ │ │ │ └── last-build.bin │ │ │ │ ├── fileHashes │ │ │ │ │ ├── fileHashes.bin │ │ │ │ │ └── fileHashes.lock │ │ │ │ └── gc.properties │ │ │ ├── buildOutputCleanup │ │ │ │ ├── buildOutputCleanup.lock │ │ │ │ ├── cache.properties │ │ │ │ └── outputFiles.bin │ │ │ ├── checksums │ │ │ │ ├── checksums.lock │ │ │ │ ├── md5-checksums.bin │ │ │ │ └── sha1-checksums.bin │ │ │ └── vcs-1 │ │ │ │ └── gc.properties │ │ ├── build.gradle │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ ├── FilterProcessor.java │ │ │ └── SimpleKafkaProcessor.java │ └── simple-kafka-streams │ │ ├── .gradle │ │ ├── 6.1.1 │ │ │ ├── executionHistory │ │ │ │ ├── executionHistory.bin │ │ │ │ └── executionHistory.lock │ │ │ ├── fileChanges │ │ │ │ └── last-build.bin │ │ │ ├── fileHashes │ │ │ │ ├── fileHashes.bin │ │ │ │ └── fileHashes.lock │ │ │ └── gc.properties │ │ ├── buildOutputCleanup │ │ │ ├── buildOutputCleanup.lock │ │ │ ├── cache.properties │ │ │ └── outputFiles.bin │ │ ├── checksums │ │ │ ├── checksums.lock │ │ │ ├── md5-checksums.bin │ │ │ └── sha1-checksums.bin │ │ └── vcs-1 │ │ │ └── gc.properties │ │ ├── build.gradle │ │ ├── settings.gradle │ │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── example │ │ └── SimpleStreamApplication.java └── 3.6 kafka-connector │ ├── .DS_Store │ ├── connect-commands.txt │ ├── connect-distributed.properties │ ├── simple-sink-connector │ ├── .DS_Store │ ├── .gradle │ │ ├── 6.1.1 │ │ │ ├── executionHistory │ │ │ │ ├── executionHistory.bin │ │ │ │ └── executionHistory.lock │ │ │ ├── fileChanges │ │ │ │ └── last-build.bin │ │ │ ├── fileContent │ │ │ │ └── fileContent.lock │ │ │ ├── fileHashes │ │ │ │ ├── fileHashes.bin │ │ │ │ └── fileHashes.lock │ │ │ ├── gc.properties │ │ │ └── javaCompile │ │ │ │ ├── classAnalysis.bin │ │ │ │ ├── javaCompile.lock │ │ │ │ └── taskHistory.bin │ │ ├── buildOutputCleanup │ │ │ ├── buildOutputCleanup.lock │ │ │ ├── cache.properties │ │ │ └── outputFiles.bin │ │ ├── checksums │ │ │ ├── checksums.lock │ │ │ ├── md5-checksums.bin │ │ │ └── sha1-checksums.bin │ │ └── vcs-1 │ │ │ └── gc.properties │ ├── build.gradle │ ├── settings.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── example │ │ ├── SingleFileSinkConnector.java │ │ ├── SingleFileSinkConnectorConfig.java │ │ └── SingleFileSinkTask.java │ └── simple-source-connector │ ├── .DS_Store │ ├── .gradle │ ├── 6.1.1 │ │ ├── executionHistory │ │ │ ├── executionHistory.bin │ │ │ └── executionHistory.lock │ │ ├── fileChanges │ │ │ └── last-build.bin │ │ ├── fileContent │ │ │ └── fileContent.lock │ │ ├── fileHashes │ │ │ ├── fileHashes.bin │ │ │ └── fileHashes.lock │ │ ├── gc.properties │ │ └── javaCompile │ │ │ ├── classAnalysis.bin │ │ │ ├── javaCompile.lock │ │ │ └── taskHistory.bin │ ├── buildOutputCleanup │ │ ├── buildOutputCleanup.lock │ │ ├── cache.properties │ │ └── outputFiles.bin │ ├── checksums │ │ ├── checksums.lock │ │ └── sha1-checksums.bin │ └── vcs-1 │ │ └── gc.properties │ ├── build.gradle │ ├── settings.gradle │ └── src │ └── main │ └── java │ └── com │ └── example │ ├── SingleFileSourceConnector.java │ ├── SingleFileSourceConnectorConfig.java │ └── SingleFileSourceTask.java ├── Chapter4 ├── .DS_Store ├── 4.2.2 idempotence-producer │ ├── build.gradle │ ├── settings.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── example │ │ └── IdempotenceProducer.java ├── 4.2.3 transaction-consumer │ ├── build.gradle │ ├── settings.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── example │ │ └── TransactionConsumer.java ├── 4.2.3 transaction-producer │ ├── build.gradle │ ├── settings.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── example │ │ └── TransactionProducer.java ├── 4.3 kafka-consumer │ ├── .DS_Store │ ├── kafka-consumer-with-multi-worker-thread │ │ ├── build.gradle │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ ├── ConsumerWithMultiWorkerThread.java │ │ │ └── ConsumerWorker.java │ ├── kafka-multi-consumer-thread-by-partition │ │ ├── build.gradle │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ ├── ConsumerWorker.java │ │ │ └── MultiConsumerThreadByPartition.java │ └── kafka-multi-consumer-thread │ │ ├── build.gradle │ │ ├── settings.gradle │ │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── example │ │ ├── ConsumerWorker.java │ │ └── MultiConsumerThread.java └── 4.4 spring-kafka │ ├── .DS_Store │ ├── spring-kafka-batch-listener │ ├── build.gradle │ ├── settings.gradle │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── example │ │ │ └── SpringConsumerApplication.java │ │ └── resources │ │ └── application.yaml │ ├── spring-kafka-commit-listener │ ├── build.gradle │ ├── settings.gradle │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── example │ │ │ └── SpringConsumerApplication.java │ │ └── resources │ │ └── application.yaml │ ├── spring-kafka-listener-container │ ├── build.gradle │ ├── settings.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── example │ │ ├── ListenerContainerConfiguration.java │ │ └── SpringConsumerApplication.java │ ├── spring-kafka-producer │ ├── build.gradle │ ├── settings.gradle │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── example │ │ │ └── SpringProducerApplication.java │ │ └── resources │ │ └── application.yaml │ ├── spring-kafka-record-listener │ ├── build.gradle │ ├── settings.gradle │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── example │ │ │ └── SpringConsumerApplication.java │ │ └── resources │ │ └── application.yaml │ └── spring-kafka-template-producer │ ├── build.gradle │ ├── settings.gradle │ └── src │ └── main │ └── java │ └── com │ └── example │ ├── KafkaTemplateConfiguration.java │ └── SpringProducerApplication.java ├── Chapter5 ├── .DS_Store ├── 5.1 web-page-event-pipeline │ ├── .DS_Store │ ├── elasticsearch-kafka-connector │ │ ├── .DS_Store │ │ ├── .gradle │ │ │ ├── 5.2.1 │ │ │ │ ├── executionHistory │ │ │ │ │ ├── executionHistory.bin │ │ │ │ │ └── executionHistory.lock │ │ │ │ ├── fileChanges │ │ │ │ │ └── last-build.bin │ │ │ │ ├── fileContent │ │ │ │ │ └── fileContent.lock │ │ │ │ ├── fileHashes │ │ │ │ │ ├── fileHashes.bin │ │ │ │ │ └── fileHashes.lock │ │ │ │ ├── gc.properties │ │ │ │ └── javaCompile │ │ │ │ │ ├── classAnalysis.bin │ │ │ │ │ ├── javaCompile.lock │ │ │ │ │ └── taskHistory.bin │ │ │ ├── buildOutputCleanup │ │ │ │ ├── buildOutputCleanup.lock │ │ │ │ ├── cache.properties │ │ │ │ └── outputFiles.bin │ │ │ └── vcs-1 │ │ │ │ └── gc.properties │ │ ├── .idea │ │ │ ├── .gitignore │ │ │ ├── codeStyles │ │ │ │ ├── Project.xml │ │ │ │ └── codeStyleConfig.xml │ │ │ ├── compiler.xml │ │ │ ├── gradle.xml │ │ │ ├── jarRepositories.xml │ │ │ ├── misc.xml │ │ │ ├── uiDesigner.xml │ │ │ └── vcs.xml │ │ ├── build.gradle │ │ ├── build │ │ │ ├── .DS_Store │ │ │ ├── libs │ │ │ │ └── .DS_Store │ │ │ └── tmp │ │ │ │ └── jar │ │ │ │ └── MANIFEST.MF │ │ ├── gradle │ │ │ └── wrapper │ │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── pipeline │ │ │ ├── ElasticSearchSinkConnector.java │ │ │ ├── ElasticSearchSinkTask.java │ │ │ └── config │ │ │ └── ElasticSearchSinkConnectorConfig.java │ ├── favorite-color-webpage │ │ └── index.html │ ├── install-es-and-kibana.md │ ├── kafka-multi-consumer-thread-hdfs-save │ │ ├── .gradle │ │ │ ├── 6.1.1 │ │ │ │ ├── executionHistory │ │ │ │ │ └── executionHistory.lock │ │ │ │ ├── fileChanges │ │ │ │ │ └── last-build.bin │ │ │ │ ├── fileHashes │ │ │ │ │ └── fileHashes.lock │ │ │ │ └── gc.properties │ │ │ ├── buildOutputCleanup │ │ │ │ ├── buildOutputCleanup.lock │ │ │ │ └── cache.properties │ │ │ ├── checksums │ │ │ │ └── checksums.lock │ │ │ └── vcs-1 │ │ │ │ └── gc.properties │ │ ├── .idea │ │ │ ├── .gitignore │ │ │ ├── codeStyles │ │ │ │ └── codeStyleConfig.xml │ │ │ ├── compiler.xml │ │ │ ├── gradle.xml │ │ │ ├── jarRepositories.xml │ │ │ ├── kafka-consumer-multi-thread.iml │ │ │ ├── kafka-multi-consumer-thread-hdfs-save.iml │ │ │ ├── misc.xml │ │ │ ├── uiDesigner.xml │ │ │ └── vcs.xml │ │ ├── build.gradle │ │ ├── gradle │ │ │ └── wrapper │ │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── out │ │ │ └── production │ │ │ │ └── classes │ │ │ │ └── META-INF │ │ │ │ └── kafka-multi-consumer-thread-hdfs-save.main.kotlin_module │ │ ├── settings.gradle │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── pipeline │ │ │ ├── HdfsSinkApplication.java │ │ │ └── consumer │ │ │ └── ConsumerWorker.java │ └── kafka-spring-producer-with-rest-controller │ │ ├── .gradle │ │ ├── 5.2.1 │ │ │ ├── executionHistory │ │ │ │ ├── executionHistory.bin │ │ │ │ └── executionHistory.lock │ │ │ ├── fileChanges │ │ │ │ └── last-build.bin │ │ │ ├── fileContent │ │ │ │ └── fileContent.lock │ │ │ ├── fileHashes │ │ │ │ ├── fileHashes.bin │ │ │ │ └── fileHashes.lock │ │ │ ├── gc.properties │ │ │ └── javaCompile │ │ │ │ ├── classAnalysis.bin │ │ │ │ ├── javaCompile.lock │ │ │ │ └── taskHistory.bin │ │ ├── buildOutputCleanup │ │ │ ├── buildOutputCleanup.lock │ │ │ ├── cache.properties │ │ │ └── outputFiles.bin │ │ └── vcs-1 │ │ │ └── gc.properties │ │ ├── .idea │ │ ├── .gitignore │ │ ├── codeStyles │ │ │ └── codeStyleConfig.xml │ │ ├── compiler.xml │ │ ├── gradle.xml │ │ ├── misc.xml │ │ ├── uiDesigner.xml │ │ └── vcs.xml │ │ ├── build.gradle │ │ ├── build │ │ └── resources │ │ │ └── main │ │ │ └── applicaiton.yml │ │ ├── gradle │ │ └── wrapper │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── settings.gradle │ │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── pipeline │ │ │ ├── ProduceController.java │ │ │ ├── RestApiProducer.java │ │ │ └── UserEventVO.java │ │ └── resources │ │ └── applicaiton.yml └── 5.2 metric-log-pipeline-streams │ ├── .DS_Store │ └── metric-kafka-streams │ ├── .DS_Store │ ├── build.gradle │ ├── metricbeat │ └── metricbeat.yml │ ├── settings.gradle │ └── src │ ├── .DS_Store │ └── main │ ├── .DS_Store │ └── java │ └── com │ └── pipeline │ ├── MetricJsonUtils.java │ └── MetricStreams.java ├── Chapter6 ├── .DS_Store └── 6.1 confluent-kafka │ ├── .DS_Store │ ├── confluent-kafka-consumer │ ├── build.gradle │ ├── settings.gradle │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── example │ │ └── SimpleConsumer.java │ └── confluent-kafka-producer │ ├── build.gradle │ ├── settings.gradle │ └── src │ └── main │ └── java │ └── com │ └── example │ └── SimpleProducer.java └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | **/build/ 2 | **/out/ 3 | **/.gradle/ 4 | **/.idea/ -------------------------------------------------------------------------------- /Chapter2/2.1 local-broker-windows/server.properties: -------------------------------------------------------------------------------- 1 | # kafka server.properties for localhost broker test 2 | broker.id=0 3 | num.network.threads=3 4 | num.io.threads=8 5 | 6 | # Please modify directory 7 | log.dirs=/tmp/kafka-logs 8 | num.partitions=1 9 | listeners=PLAINTEXT://[::1]:9092 10 | #advertised.listeners=PLAINTEXT://localhost:9092 11 | socket.send.buffer.bytes=102400 12 | socket.receive.buffer.bytes=102400 13 | socket.request.max.bytes=104857600 14 | num.recovery.threads.per.data.dir=1 15 | offsets.topic.replication.factor=1 16 | transaction.state.log.replication.factor=1 17 | transaction.state.log.min.isr=1 18 | log.retention.hours=168 19 | log.segment.bytes=1073741824 20 | log.retention.check.interval.ms=300000 21 | zookeeper.connect=localhost:2181 22 | zookeeper.connection.timeout.ms=18000 23 | group.initial.rebalance.delay.ms=0 -------------------------------------------------------------------------------- /Chapter2/2.1 local-broker-windows/zookeeper.properties: -------------------------------------------------------------------------------- 1 | dataDir=/tmp/zookeeper 2 | clientPort=2181 3 | maxClientCnxns=0 4 | admin.enableServer=false -------------------------------------------------------------------------------- /Chapter2/2.1 local-broker/server.properties: -------------------------------------------------------------------------------- 1 | # kafka server.properties for localhost broker test 2 | broker.id=0 3 | num.network.threads=3 4 | num.io.threads=8 5 | 6 | # Please modify directory 7 | log.dirs=/Users/dvwy/Documents/test/kafka_2.12-2.5.0/data 8 | num.partitions=3 9 | listeners=PLAINTEXT://localhost:9092 10 | advertised.listeners=PLAINTEXT://localhost:9092 11 | socket.send.buffer.bytes=102400 12 | socket.receive.buffer.bytes=102400 13 | socket.request.max.bytes=104857600 14 | num.recovery.threads.per.data.dir=1 15 | offsets.topic.replication.factor=1 16 | transaction.state.log.replication.factor=1 17 | transaction.state.log.min.isr=1 18 | log.retention.hours=168 19 | log.segment.bytes=1073741824 20 | log.retention.check.interval.ms=300000 21 | zookeeper.connect=localhost:2181 22 | zookeeper.connection.timeout.ms=18000 23 | group.initial.rebalance.delay.ms=0 -------------------------------------------------------------------------------- /Chapter2/2.1 local-broker/zookeeper.properties: -------------------------------------------------------------------------------- 1 | dataDir=/tmp/zookeeper 2 | clientPort=2181 3 | maxClientCnxns=0 4 | admin.enableServer=false -------------------------------------------------------------------------------- /Chapter3/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/.DS_Store -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.1 kafka-producer/.DS_Store -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-async-callback/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-async-callback/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.1 kafka-producer/kafka-producer-async-callback/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-async-callback/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-async-callback/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-producer-async-callback' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-async-callback/src/main/java/com/example/ProducerCallback.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.producer.Callback; 4 | import org.apache.kafka.clients.producer.RecordMetadata; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | public class ProducerCallback implements Callback { 9 | private final static Logger logger = LoggerFactory.getLogger(ProducerCallback.class); 10 | 11 | @Override 12 | public void onCompletion(RecordMetadata recordMetadata, Exception e) { 13 | if (e != null) 14 | logger.error(e.getMessage(), e); 15 | else 16 | logger.info(recordMetadata.toString()); 17 | } 18 | } -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-async-callback/src/main/java/com/example/ProducerWithAsyncCallback.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.ProducerConfig; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.apache.kafka.common.serialization.StringSerializer; 7 | import java.util.Properties; 8 | 9 | public class ProducerWithAsyncCallback { 10 | private final static String TOPIC_NAME = "test"; 11 | private final static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 12 | 13 | public static void main(String[] args) { 14 | 15 | Properties configs = new Properties(); 16 | configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 17 | configs.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 18 | configs.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 19 | 20 | KafkaProducer producer = new KafkaProducer<>(configs); 21 | 22 | ProducerRecord record = new ProducerRecord<>(TOPIC_NAME, "Pangyo", "Pangyo"); 23 | producer.send(record, new ProducerCallback()); 24 | 25 | producer.flush(); 26 | producer.close(); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-custom-partitioner/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-custom-partitioner/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.1 kafka-producer/kafka-producer-custom-partitioner/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-custom-partitioner/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-custom-partitioner/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-producer-custom-partitioner' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-custom-partitioner/src/main/java/com/example/CustomPartitioner.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.producer.Partitioner; 4 | import org.apache.kafka.common.Cluster; 5 | import org.apache.kafka.common.InvalidRecordException; 6 | import org.apache.kafka.common.PartitionInfo; 7 | import org.apache.kafka.common.utils.Utils; 8 | 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | public class CustomPartitioner implements Partitioner { 13 | 14 | @Override 15 | public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, 16 | Cluster cluster) { 17 | 18 | if (keyBytes == null) { 19 | throw new InvalidRecordException("Need message key"); 20 | } 21 | if (((String)key).equals("Pangyo")) 22 | return 0; 23 | 24 | List partitions = cluster.partitionsForTopic(topic); 25 | int numPartitions = partitions.size(); 26 | return Utils.toPositive(Utils.murmur2(keyBytes)) % numPartitions; 27 | } 28 | 29 | 30 | @Override 31 | public void configure(Map configs) {} 32 | 33 | @Override 34 | public void close() {} 35 | } 36 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-custom-partitioner/src/main/java/com/example/ProducerWithCustomPartitioner.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.ProducerConfig; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.apache.kafka.common.serialization.StringSerializer; 7 | import java.util.Properties; 8 | 9 | public class ProducerWithCustomPartitioner { 10 | private final static String TOPIC_NAME = "test"; 11 | private final static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 12 | 13 | public static void main(String[] args) { 14 | 15 | Properties configs = new Properties(); 16 | configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 17 | configs.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 18 | configs.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 19 | configs.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, CustomPartitioner.class); 20 | 21 | KafkaProducer producer = new KafkaProducer<>(configs); 22 | 23 | ProducerRecord record = new ProducerRecord<>(TOPIC_NAME, "Pangyo", "Pangyo"); 24 | producer.send(record); 25 | producer.flush(); 26 | producer.close(); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-exact-partition/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-exact-partition/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.1 kafka-producer/kafka-producer-exact-partition/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-exact-partition/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-exact-partition/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-producer-exact-partition' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-exact-partition/src/main/java/com/example/ProducerExactPartition.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.ProducerConfig; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.apache.kafka.common.serialization.StringSerializer; 7 | import java.util.Properties; 8 | 9 | public class ProducerExactPartition { 10 | private final static String TOPIC_NAME = "test"; 11 | private final static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 12 | 13 | public static void main(String[] args) { 14 | 15 | Properties configs = new Properties(); 16 | configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 17 | configs.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 18 | configs.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 19 | 20 | KafkaProducer producer = new KafkaProducer<>(configs); 21 | 22 | int partitionNo = 0; 23 | ProducerRecord record = new ProducerRecord<>(TOPIC_NAME, partitionNo, "Pangyo", "Pangyo"); 24 | producer.send(record); 25 | 26 | producer.flush(); 27 | producer.close(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-key-value/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-key-value/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.1 kafka-producer/kafka-producer-key-value/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-key-value/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-key-value/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-producer-key-value' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-key-value/src/main/java/com/example/ProducerWithKeyValue.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.ProducerConfig; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.apache.kafka.common.serialization.StringSerializer; 7 | import java.util.Properties; 8 | 9 | public class ProducerWithKeyValue { 10 | private final static String TOPIC_NAME = "test"; 11 | private final static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 12 | 13 | public static void main(String[] args) { 14 | 15 | Properties configs = new Properties(); 16 | configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 17 | configs.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 18 | configs.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 19 | 20 | KafkaProducer producer = new KafkaProducer<>(configs); 21 | 22 | ProducerRecord record = new ProducerRecord<>(TOPIC_NAME, "Pangyo", "Pangyo"); 23 | producer.send(record); 24 | ProducerRecord record2 = new ProducerRecord<>(TOPIC_NAME, "Busan", "Busan"); 25 | producer.send(record2); 26 | producer.flush(); 27 | producer.close(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-sync-callback/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } 19 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-sync-callback/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.1 kafka-producer/kafka-producer-sync-callback/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-sync-callback/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-sync-callback/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-producer-sync-callback' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/kafka-producer-sync-callback/src/main/java/com/example/ProducerWithSyncCallback.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.ProducerConfig; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.apache.kafka.clients.producer.RecordMetadata; 7 | import org.apache.kafka.common.serialization.StringSerializer; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.util.Properties; 12 | 13 | public class ProducerWithSyncCallback { 14 | private final static Logger logger = LoggerFactory.getLogger(ProducerWithSyncCallback.class); 15 | private final static String TOPIC_NAME = "test"; 16 | private final static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 17 | 18 | public static void main(String[] args) { 19 | 20 | Properties configs = new Properties(); 21 | configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 22 | configs.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 23 | configs.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 24 | //configs.put(ProducerConfig.ACKS_CONFIG, "0"); 25 | 26 | KafkaProducer producer = new KafkaProducer<>(configs); 27 | 28 | ProducerRecord record = new ProducerRecord<>(TOPIC_NAME, "Pangyo", "Pangyo"); 29 | try { 30 | RecordMetadata metadata = producer.send(record).get(); 31 | logger.info(metadata.toString()); 32 | } catch (Exception e) { 33 | logger.error(e.getMessage(),e); 34 | } finally { 35 | producer.flush(); 36 | producer.close(); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/simple-kafka-producer/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/simple-kafka-producer/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.1 kafka-producer/simple-kafka-producer/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/simple-kafka-producer/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/simple-kafka-producer/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'simple-kafka-producer' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.4.1 kafka-producer/simple-kafka-producer/src/main/java/com/example/SimpleProducer.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.ProducerConfig; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.apache.kafka.common.serialization.StringSerializer; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.util.Properties; 11 | 12 | public class SimpleProducer { 13 | private final static Logger logger = LoggerFactory.getLogger(SimpleProducer.class); 14 | private final static String TOPIC_NAME = "test"; 15 | private final static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 16 | 17 | public static void main(String[] args) { 18 | 19 | Properties configs = new Properties(); 20 | configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 21 | configs.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 22 | configs.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 23 | 24 | KafkaProducer producer = new KafkaProducer<>(configs); 25 | 26 | String messageValue = "testMessage"; 27 | ProducerRecord record = new ProducerRecord<>(TOPIC_NAME, messageValue); 28 | producer.send(record); 29 | logger.info("{}", record); 30 | producer.flush(); 31 | producer.close(); 32 | } 33 | } -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.2 kafka-consumer/.DS_Store -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-async-commit/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-async-commit/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.2 kafka-consumer/kafka-consumer-async-commit/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-async-commit/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-async-commit/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-consumer-async-commit' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-auto-commit/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-auto-commit/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.2 kafka-consumer/kafka-consumer-auto-commit/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-auto-commit/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-auto-commit/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-consumer-auto-commit' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-auto-commit/src/main/java/com/example/ConsumerWithAutoCommit.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerConfig; 4 | import org.apache.kafka.clients.consumer.ConsumerRecord; 5 | import org.apache.kafka.clients.consumer.ConsumerRecords; 6 | import org.apache.kafka.clients.consumer.KafkaConsumer; 7 | import org.apache.kafka.common.serialization.StringDeserializer; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.time.Duration; 12 | import java.util.Arrays; 13 | import java.util.Properties; 14 | 15 | public class ConsumerWithAutoCommit { 16 | private final static Logger logger = LoggerFactory.getLogger(ConsumerWithAutoCommit.class); 17 | private final static String TOPIC_NAME = "test"; 18 | private final static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 19 | private final static String GROUP_ID = "test-group"; 20 | 21 | public static void main(String[] args) { 22 | Properties configs = new Properties(); 23 | configs.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 24 | configs.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID); 25 | configs.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 26 | configs.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 27 | configs.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true); 28 | configs.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, 60000); 29 | 30 | KafkaConsumer consumer = new KafkaConsumer<>(configs); 31 | consumer.subscribe(Arrays.asList(TOPIC_NAME)); 32 | 33 | while (true) { 34 | ConsumerRecords records = consumer.poll(Duration.ofSeconds(1)); 35 | for (ConsumerRecord record : records) { 36 | logger.info("record:{}", record); 37 | } 38 | } 39 | } 40 | } -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-exact-partition/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-exact-partition/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.2 kafka-consumer/kafka-consumer-exact-partition/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-exact-partition/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-exact-partition/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-consumer-exact-partition' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-exact-partition/src/main/java/com/example/ConsumerWithExactPartition.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.consumer.*; 4 | import org.apache.kafka.common.TopicPartition; 5 | import org.apache.kafka.common.serialization.StringDeserializer; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.time.Duration; 10 | import java.util.Collections; 11 | import java.util.Properties; 12 | 13 | public class ConsumerWithExactPartition { 14 | private final static Logger logger = LoggerFactory.getLogger(ConsumerWithExactPartition.class); 15 | private final static String TOPIC_NAME = "test"; 16 | private final static int PARTITION_NUMBER = 0; 17 | private final static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 18 | 19 | public static void main(String[] args) { 20 | Properties configs = new Properties(); 21 | configs.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 22 | configs.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 23 | configs.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 24 | 25 | KafkaConsumer consumer = new KafkaConsumer<>(configs); 26 | consumer.assign(Collections.singleton(new TopicPartition(TOPIC_NAME, PARTITION_NUMBER))); 27 | while (true) { 28 | ConsumerRecords records = consumer.poll(Duration.ofSeconds(1)); 29 | for (ConsumerRecord record : records) { 30 | logger.info("record:{}", record); 31 | } 32 | } 33 | } 34 | } -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-rebalance-listener/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-rebalance-listener/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.2 kafka-consumer/kafka-consumer-rebalance-listener/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-rebalance-listener/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-rebalance-listener/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-consumer-rebalance-listener' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-rebalance-listener/src/main/java/com/example/ConsumerWithRebalanceListener.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.consumer.*; 4 | import org.apache.kafka.common.serialization.StringDeserializer; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.time.Duration; 9 | import java.util.*; 10 | 11 | public class ConsumerWithRebalanceListener { 12 | private final static Logger logger = LoggerFactory.getLogger(ConsumerWithRebalanceListener.class); 13 | private final static String TOPIC_NAME = "test"; 14 | private final static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 15 | private final static String GROUP_ID = "test-group"; 16 | 17 | 18 | private static KafkaConsumer consumer; 19 | 20 | public static void main(String[] args) { 21 | Properties configs = new Properties(); 22 | configs.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 23 | configs.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID); 24 | configs.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 25 | configs.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 26 | 27 | consumer = new KafkaConsumer<>(configs); 28 | consumer.subscribe(Arrays.asList(TOPIC_NAME), new RebalanceListener()); 29 | while (true) { 30 | ConsumerRecords records = consumer.poll(Duration.ofSeconds(1)); 31 | for (ConsumerRecord record : records) { 32 | logger.info("{}", record); 33 | } 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-rebalance-listener/src/main/java/com/example/RebalanceListener.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; 4 | import org.apache.kafka.common.TopicPartition; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.util.Collection; 9 | 10 | public class RebalanceListener implements ConsumerRebalanceListener { 11 | private final static Logger logger = LoggerFactory.getLogger(RebalanceListener.class); 12 | 13 | public void onPartitionsAssigned(Collection partitions) { 14 | logger.warn("Partitions are assigned : " + partitions.toString()); 15 | 16 | } 17 | 18 | public void onPartitionsRevoked(Collection partitions) { 19 | logger.warn("Partitions are revoked : " + partitions.toString()); 20 | } 21 | } -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-sync-commit/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-sync-commit/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.2 kafka-consumer/kafka-consumer-sync-commit/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-sync-commit/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-sync-commit/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-consumer-sync-commit' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-sync-commit/src/main/java/com/example/ConsumerWithSyncCommit.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.consumer.*; 4 | import org.apache.kafka.common.serialization.StringDeserializer; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.time.Duration; 9 | import java.util.Arrays; 10 | import java.util.Properties; 11 | 12 | public class ConsumerWithSyncCommit { 13 | private final static Logger logger = LoggerFactory.getLogger(ConsumerWithSyncCommit.class); 14 | private final static String TOPIC_NAME = "test"; 15 | private final static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 16 | private final static String GROUP_ID = "test-group"; 17 | 18 | public static void main(String[] args) { 19 | Properties configs = new Properties(); 20 | configs.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 21 | configs.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID); 22 | configs.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 23 | configs.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 24 | configs.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); 25 | 26 | KafkaConsumer consumer = new KafkaConsumer<>(configs); 27 | consumer.subscribe(Arrays.asList(TOPIC_NAME)); 28 | 29 | while (true) { 30 | ConsumerRecords records = consumer.poll(Duration.ofSeconds(1)); 31 | for (ConsumerRecord record : records) { 32 | logger.info("record:{}", record); 33 | } 34 | consumer.commitSync(); 35 | } 36 | } 37 | } -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-sync-offset-commit-shutdown-hook/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-sync-offset-commit-shutdown-hook/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.2 kafka-consumer/kafka-consumer-sync-offset-commit-shutdown-hook/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-sync-offset-commit-shutdown-hook/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-sync-offset-commit-shutdown-hook/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-consumer-sync-offset-commit-shutdown-hook' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-sync-offset-commit/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-sync-offset-commit/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.2 kafka-consumer/kafka-consumer-sync-offset-commit/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-sync-offset-commit/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-sync-offset-commit/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-consumer-sync-offset-commit' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/kafka-consumer-sync-offset-commit/src/main/java/com/example/ConsumerWithSyncOffsetCommit.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.consumer.*; 4 | import org.apache.kafka.common.TopicPartition; 5 | import org.apache.kafka.common.serialization.StringDeserializer; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.time.Duration; 10 | import java.util.Arrays; 11 | import java.util.HashMap; 12 | import java.util.Map; 13 | import java.util.Properties; 14 | 15 | public class ConsumerWithSyncOffsetCommit { 16 | private final static Logger logger = LoggerFactory.getLogger(ConsumerWithSyncOffsetCommit.class); 17 | private final static String TOPIC_NAME = "test"; 18 | private final static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 19 | private final static String GROUP_ID = "test-group"; 20 | 21 | public static void main(String[] args) { 22 | Properties configs = new Properties(); 23 | configs.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 24 | configs.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID); 25 | configs.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 26 | configs.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 27 | configs.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); 28 | 29 | KafkaConsumer consumer = new KafkaConsumer<>(configs); 30 | consumer.subscribe(Arrays.asList(TOPIC_NAME)); 31 | 32 | while (true) { 33 | ConsumerRecords records = consumer.poll(Duration.ofSeconds(1)); 34 | Map currentOffset = new HashMap<>(); 35 | 36 | for (ConsumerRecord record : records) { 37 | logger.info("record:{}", record); 38 | currentOffset.put( 39 | new TopicPartition(record.topic(), record.partition()), 40 | new OffsetAndMetadata(record.offset() + 1, null)); 41 | consumer.commitSync(currentOffset); 42 | } 43 | } 44 | } 45 | } -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/simple-kafka-consumer/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/simple-kafka-consumer/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.2 kafka-consumer/simple-kafka-consumer/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/simple-kafka-consumer/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/simple-kafka-consumer/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'simple-kafka-consumer' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.4.2 kafka-consumer/simple-kafka-consumer/src/main/java/com/example/SimpleConsumer.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerConfig; 4 | import org.apache.kafka.clients.consumer.ConsumerRecord; 5 | import org.apache.kafka.clients.consumer.ConsumerRecords; 6 | import org.apache.kafka.clients.consumer.KafkaConsumer; 7 | import org.apache.kafka.common.serialization.StringDeserializer; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.time.Duration; 12 | import java.util.Arrays; 13 | import java.util.Properties; 14 | 15 | public class SimpleConsumer { 16 | private final static Logger logger = LoggerFactory.getLogger(SimpleConsumer.class); 17 | private final static String TOPIC_NAME = "test"; 18 | private final static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 19 | private final static String GROUP_ID = "test-group"; 20 | 21 | public static void main(String[] args) { 22 | Properties configs = new Properties(); 23 | configs.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 24 | configs.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID); 25 | configs.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 26 | configs.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 27 | 28 | KafkaConsumer consumer = new KafkaConsumer<>(configs); 29 | 30 | consumer.subscribe(Arrays.asList(TOPIC_NAME)); 31 | 32 | while (true) { 33 | ConsumerRecords records = consumer.poll(Duration.ofSeconds(1)); 34 | for (ConsumerRecord record : records) { 35 | logger.info("record:{}", record); 36 | } 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.3 kafka-admin/.DS_Store -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/executionHistory/executionHistory.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/executionHistory/executionHistory.bin -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/executionHistory/executionHistory.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/executionHistory/executionHistory.lock -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/fileChanges/last-build.bin: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/fileContent/fileContent.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/fileContent/fileContent.lock -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/fileHashes/fileHashes.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/fileHashes/fileHashes.bin -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/fileHashes/fileHashes.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/fileHashes/fileHashes.lock -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/javaCompile/classAnalysis.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/javaCompile/classAnalysis.bin -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/javaCompile/javaCompile.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/javaCompile/javaCompile.lock -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/javaCompile/taskHistory.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/6.1.1/javaCompile/taskHistory.bin -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/buildOutputCleanup/buildOutputCleanup.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/buildOutputCleanup/buildOutputCleanup.lock -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/buildOutputCleanup/cache.properties: -------------------------------------------------------------------------------- 1 | #Tue Jun 09 13:57:35 KST 2020 2 | gradle.version=6.1.1 3 | -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/buildOutputCleanup/outputFiles.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/buildOutputCleanup/outputFiles.bin -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/checksums/checksums.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/checksums/checksums.lock -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/checksums/md5-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/checksums/md5-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/checksums/sha1-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/checksums/sha1-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/vcs-1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.4.3 kafka-admin/kafka-admin-client/.gradle/vcs-1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | repositories { 9 | mavenCentral() 10 | } 11 | 12 | dependencies { 13 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 14 | implementation 'org.slf4j:slf4j-simple:1.7.30' 15 | } -------------------------------------------------------------------------------- /Chapter3/3.4.3 kafka-admin/kafka-admin-client/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-admin-client' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/.DS_Store -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/6.1.1/executionHistory/executionHistory.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/6.1.1/executionHistory/executionHistory.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/6.1.1/executionHistory/executionHistory.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/6.1.1/executionHistory/executionHistory.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/6.1.1/fileChanges/last-build.bin: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/6.1.1/fileHashes/fileHashes.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/6.1.1/fileHashes/fileHashes.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/6.1.1/fileHashes/fileHashes.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/6.1.1/fileHashes/fileHashes.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/6.1.1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/6.1.1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/buildOutputCleanup/buildOutputCleanup.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/buildOutputCleanup/buildOutputCleanup.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/buildOutputCleanup/cache.properties: -------------------------------------------------------------------------------- 1 | #Tue Jun 09 13:56:51 KST 2020 2 | gradle.version=6.1.1 3 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/buildOutputCleanup/outputFiles.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/buildOutputCleanup/outputFiles.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/checksums/checksums.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/checksums/checksums.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/checksums/md5-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/checksums/md5-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/checksums/sha1-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/checksums/sha1-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/vcs-1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kafka-streams-filter/.gradle/vcs-1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kafka-streams-filter/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-streams:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } 19 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kafka-streams-filter/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-streams-filter' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kafka-streams-filter/src/main/java/com/example/StreamsFilter.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.common.serialization.Serdes; 4 | import org.apache.kafka.streams.KafkaStreams; 5 | import org.apache.kafka.streams.StreamsBuilder; 6 | import org.apache.kafka.streams.StreamsConfig; 7 | import org.apache.kafka.streams.kstream.KStream; 8 | 9 | import java.util.Properties; 10 | 11 | public class StreamsFilter { 12 | 13 | private static String APPLICATION_NAME = "streams-filter-application"; 14 | private static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 15 | private static String STREAM_LOG = "stream_log"; 16 | private static String STREAM_LOG_FILTER = "stream_log_filter"; 17 | 18 | public static void main(String[] args) { 19 | 20 | Properties props = new Properties(); 21 | props.put(StreamsConfig.APPLICATION_ID_CONFIG, APPLICATION_NAME); 22 | props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 23 | props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 24 | props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 25 | 26 | StreamsBuilder builder = new StreamsBuilder(); 27 | KStream streamLog = builder.stream(STREAM_LOG); 28 | // KStream filteredStream = streamLog.filter( 29 | // (key, value) -> value.length() > 5); 30 | // filteredStream.to(STREAM_LOG_FILTER); 31 | 32 | streamLog.filter((key, value) -> value.length() > 5).to(STREAM_LOG_FILTER); 33 | 34 | 35 | KafkaStreams streams; 36 | streams = new KafkaStreams(builder.build(), props); 37 | streams.start(); 38 | 39 | } 40 | } -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-count/.DS_Store -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/.gradle/6.1.1/executionHistory/executionHistory.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-count/.gradle/6.1.1/executionHistory/executionHistory.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/.gradle/6.1.1/executionHistory/executionHistory.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-count/.gradle/6.1.1/executionHistory/executionHistory.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/.gradle/6.1.1/fileChanges/last-build.bin: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/.gradle/6.1.1/fileHashes/fileHashes.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-count/.gradle/6.1.1/fileHashes/fileHashes.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/.gradle/6.1.1/fileHashes/fileHashes.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-count/.gradle/6.1.1/fileHashes/fileHashes.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/.gradle/6.1.1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-count/.gradle/6.1.1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/.gradle/buildOutputCleanup/buildOutputCleanup.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-count/.gradle/buildOutputCleanup/buildOutputCleanup.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/.gradle/buildOutputCleanup/cache.properties: -------------------------------------------------------------------------------- 1 | #Tue Jun 09 13:56:51 KST 2020 2 | gradle.version=6.1.1 3 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/.gradle/buildOutputCleanup/outputFiles.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-count/.gradle/buildOutputCleanup/outputFiles.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/.gradle/checksums/checksums.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-count/.gradle/checksums/checksums.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/.gradle/checksums/md5-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-count/.gradle/checksums/md5-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/.gradle/checksums/sha1-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-count/.gradle/checksums/sha1-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/.gradle/vcs-1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-count/.gradle/vcs-1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } 19 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kstream-count' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/src/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-count/src/.DS_Store -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/src/main/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-count/src/main/.DS_Store -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-count/src/main/java/com/example/KStreamCountApplication.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.common.serialization.Serdes; 4 | import org.apache.kafka.streams.KafkaStreams; 5 | import org.apache.kafka.streams.StreamsBuilder; 6 | import org.apache.kafka.streams.StreamsConfig; 7 | import org.apache.kafka.streams.kstream.*; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.time.Duration; 12 | import java.util.Properties; 13 | 14 | public class KStreamCountApplication { 15 | 16 | private final static Logger log = LoggerFactory.getLogger(KStreamCountApplication.class); 17 | 18 | private static String APPLICATION_NAME = "stream-count-application"; 19 | private static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 20 | private static String TEST_LOG = "test"; 21 | 22 | public static void main(String[] args) { 23 | 24 | Properties props = new Properties(); 25 | props.put(StreamsConfig.APPLICATION_ID_CONFIG, APPLICATION_NAME); 26 | props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 27 | props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 28 | props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 29 | props.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 10000); 30 | 31 | StreamsBuilder builder = new StreamsBuilder(); 32 | KStream stream = builder.stream(TEST_LOG); 33 | KTable, Long> countTable = stream.groupByKey() 34 | .windowedBy(TimeWindows.of(Duration.ofSeconds(5))) 35 | .count(); 36 | countTable.toStream().foreach(((key, value) -> { 37 | log.info(key.key() + " is [" + key.window().startTime() + "~" + key.window().endTime() + "] count : " + value); 38 | })); 39 | 40 | KafkaStreams streams = new KafkaStreams(builder.build(), props); 41 | streams.start(); 42 | 43 | } 44 | } -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/6.1.1/executionHistory/executionHistory.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/6.1.1/executionHistory/executionHistory.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/6.1.1/executionHistory/executionHistory.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/6.1.1/executionHistory/executionHistory.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/6.1.1/fileChanges/last-build.bin: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/6.1.1/fileHashes/fileHashes.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/6.1.1/fileHashes/fileHashes.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/6.1.1/fileHashes/fileHashes.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/6.1.1/fileHashes/fileHashes.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/6.1.1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/6.1.1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/buildOutputCleanup/buildOutputCleanup.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/buildOutputCleanup/buildOutputCleanup.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/buildOutputCleanup/cache.properties: -------------------------------------------------------------------------------- 1 | #Tue Jun 09 13:56:51 KST 2020 2 | gradle.version=6.1.1 3 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/buildOutputCleanup/outputFiles.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/buildOutputCleanup/outputFiles.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/checksums/checksums.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/checksums/checksums.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/checksums/md5-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/checksums/md5-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/checksums/sha1-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/checksums/sha1-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/vcs-1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-globalktable-join/.gradle/vcs-1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-globalktable-join/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-streams:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } 19 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-globalktable-join/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kstream-globalktable-join' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/6.1.1/executionHistory/executionHistory.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/6.1.1/executionHistory/executionHistory.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/6.1.1/executionHistory/executionHistory.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/6.1.1/executionHistory/executionHistory.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/6.1.1/fileChanges/last-build.bin: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/6.1.1/fileHashes/fileHashes.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/6.1.1/fileHashes/fileHashes.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/6.1.1/fileHashes/fileHashes.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/6.1.1/fileHashes/fileHashes.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/6.1.1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/6.1.1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/buildOutputCleanup/buildOutputCleanup.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/buildOutputCleanup/buildOutputCleanup.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/buildOutputCleanup/cache.properties: -------------------------------------------------------------------------------- 1 | #Tue Jun 09 13:56:51 KST 2020 2 | gradle.version=6.1.1 3 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/buildOutputCleanup/outputFiles.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/buildOutputCleanup/outputFiles.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/checksums/checksums.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/checksums/checksums.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/checksums/md5-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/checksums/md5-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/checksums/sha1-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/checksums/sha1-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/vcs-1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/kstream-ktable-join/.gradle/vcs-1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-ktable-join/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-streams:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } 19 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/kstream-ktable-join/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kstream-ktable-join' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/queryable-store/.gradle/6.1.1/executionHistory/executionHistory.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/queryable-store/.gradle/6.1.1/executionHistory/executionHistory.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/queryable-store/.gradle/6.1.1/executionHistory/executionHistory.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/queryable-store/.gradle/6.1.1/executionHistory/executionHistory.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/queryable-store/.gradle/6.1.1/fileChanges/last-build.bin: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/queryable-store/.gradle/6.1.1/fileHashes/fileHashes.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/queryable-store/.gradle/6.1.1/fileHashes/fileHashes.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/queryable-store/.gradle/6.1.1/fileHashes/fileHashes.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/queryable-store/.gradle/6.1.1/fileHashes/fileHashes.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/queryable-store/.gradle/6.1.1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/queryable-store/.gradle/6.1.1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/queryable-store/.gradle/buildOutputCleanup/buildOutputCleanup.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/queryable-store/.gradle/buildOutputCleanup/buildOutputCleanup.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/queryable-store/.gradle/buildOutputCleanup/cache.properties: -------------------------------------------------------------------------------- 1 | #Tue Jun 09 13:56:51 KST 2020 2 | gradle.version=6.1.1 3 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/queryable-store/.gradle/buildOutputCleanup/outputFiles.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/queryable-store/.gradle/buildOutputCleanup/outputFiles.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/queryable-store/.gradle/checksums/checksums.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/queryable-store/.gradle/checksums/checksums.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/queryable-store/.gradle/checksums/md5-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/queryable-store/.gradle/checksums/md5-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/queryable-store/.gradle/checksums/sha1-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/queryable-store/.gradle/checksums/sha1-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/queryable-store/.gradle/vcs-1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/queryable-store/.gradle/vcs-1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/queryable-store/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-streams:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } 19 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/queryable-store/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'queryable-store' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/6.1.1/executionHistory/executionHistory.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/6.1.1/executionHistory/executionHistory.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/6.1.1/executionHistory/executionHistory.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/6.1.1/executionHistory/executionHistory.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/6.1.1/fileChanges/last-build.bin: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/6.1.1/fileHashes/fileHashes.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/6.1.1/fileHashes/fileHashes.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/6.1.1/fileHashes/fileHashes.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/6.1.1/fileHashes/fileHashes.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/6.1.1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/6.1.1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/buildOutputCleanup/buildOutputCleanup.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/buildOutputCleanup/buildOutputCleanup.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/buildOutputCleanup/cache.properties: -------------------------------------------------------------------------------- 1 | #Tue Jun 09 13:56:51 KST 2020 2 | gradle.version=6.1.1 3 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/buildOutputCleanup/outputFiles.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/buildOutputCleanup/outputFiles.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/checksums/checksums.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/checksums/checksums.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/checksums/md5-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/checksums/md5-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/checksums/sha1-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/checksums/sha1-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/vcs-1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-processor/.gradle/vcs-1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-processor/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-streams:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } 19 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-processor/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'simple-kafka-processor' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-processor/src/main/java/com/example/FilterProcessor.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.streams.processor.ProcessorContext; 4 | 5 | import org.apache.kafka.streams.processor.Processor; 6 | 7 | public class FilterProcessor implements Processor { 8 | 9 | private ProcessorContext context; 10 | 11 | @Override 12 | public void init(ProcessorContext context) { 13 | this.context = context; 14 | } 15 | 16 | @Override 17 | public void process(String key, String value) { 18 | if (value.length() > 5) { 19 | context.forward(key, value); 20 | } 21 | context.commit(); 22 | } 23 | 24 | @Override 25 | public void close() { 26 | } 27 | 28 | } -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-processor/src/main/java/com/example/SimpleKafkaProcessor.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.common.serialization.Serdes; 4 | import org.apache.kafka.streams.KafkaStreams; 5 | import org.apache.kafka.streams.StreamsConfig; 6 | import org.apache.kafka.streams.Topology; 7 | 8 | import java.util.Properties; 9 | 10 | public class SimpleKafkaProcessor { 11 | 12 | private static String APPLICATION_NAME = "processor-application"; 13 | private static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 14 | private static String STREAM_LOG = "stream_log"; 15 | private static String STREAM_LOG_FILTER = "stream_log_filter"; 16 | 17 | public static void main(String[] args) { 18 | 19 | Properties props = new Properties(); 20 | props.put(StreamsConfig.APPLICATION_ID_CONFIG, APPLICATION_NAME); 21 | props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 22 | props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 23 | props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 24 | 25 | Topology topology = new Topology(); 26 | topology.addSource("Source", 27 | STREAM_LOG) 28 | .addProcessor("Process", 29 | () -> new FilterProcessor(), 30 | "Source") 31 | .addSink("Sink", 32 | STREAM_LOG_FILTER, 33 | "Process"); 34 | 35 | KafkaStreams streaming = new KafkaStreams(topology, props); 36 | streaming.start(); 37 | } 38 | } 39 | 40 | //https://kafka.apache.org/documentation/streams/developer-guide/processor-api.html -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/6.1.1/executionHistory/executionHistory.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/6.1.1/executionHistory/executionHistory.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/6.1.1/executionHistory/executionHistory.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/6.1.1/executionHistory/executionHistory.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/6.1.1/fileChanges/last-build.bin: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/6.1.1/fileHashes/fileHashes.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/6.1.1/fileHashes/fileHashes.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/6.1.1/fileHashes/fileHashes.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/6.1.1/fileHashes/fileHashes.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/6.1.1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/6.1.1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/buildOutputCleanup/buildOutputCleanup.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/buildOutputCleanup/buildOutputCleanup.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/buildOutputCleanup/cache.properties: -------------------------------------------------------------------------------- 1 | #Tue Jun 09 13:56:51 KST 2020 2 | gradle.version=6.1.1 3 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/buildOutputCleanup/outputFiles.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/buildOutputCleanup/outputFiles.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/checksums/checksums.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/checksums/checksums.lock -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/checksums/md5-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/checksums/md5-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/checksums/sha1-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/checksums/sha1-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/vcs-1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.5 kafka-streams/simple-kafka-streams/.gradle/vcs-1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-streams/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } 19 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-streams/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'simple-kafka-streams' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.5 kafka-streams/simple-kafka-streams/src/main/java/com/example/SimpleStreamApplication.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.common.serialization.Serdes; 4 | import org.apache.kafka.streams.KafkaStreams; 5 | import org.apache.kafka.streams.StreamsBuilder; 6 | import org.apache.kafka.streams.StreamsConfig; 7 | import org.apache.kafka.streams.kstream.KStream; 8 | 9 | import java.util.Properties; 10 | 11 | public class SimpleStreamApplication { 12 | 13 | private static String APPLICATION_NAME = "streams-application"; 14 | private static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 15 | private static String STREAM_LOG = "stream_log"; 16 | private static String STREAM_LOG_COPY = "stream_log_copy"; 17 | 18 | public static void main(String[] args) { 19 | 20 | Properties props = new Properties(); 21 | props.put(StreamsConfig.APPLICATION_ID_CONFIG, APPLICATION_NAME); 22 | props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 23 | props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 24 | props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 25 | 26 | StreamsBuilder builder = new StreamsBuilder(); 27 | KStream stream = builder.stream(STREAM_LOG); 28 | 29 | stream.to(STREAM_LOG_COPY); 30 | 31 | KafkaStreams streams = new KafkaStreams(builder.build(), props); 32 | streams.start(); 33 | 34 | } 35 | } -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/.DS_Store -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/connect-commands.txt: -------------------------------------------------------------------------------- 1 | # 커넥터 플러그인 조회 2 | $ curl -X GET http://localhost:8083/connector-plugins 3 | 4 | # FileStreamSinkConnector 생성 5 | 6 | $ curl -X POST \ 7 | http://localhost:8083/connectors \ 8 | -H 'Content-Type: application/json' \ 9 | -d '{ 10 | "name": "file-sink-test", 11 | "config": 12 | { 13 | "topics":"test", 14 | "connector.class":"org.apache.kafka.connect.file.FileStreamSinkConnector", 15 | "tasks.max":1, 16 | "file":"/tmp/connect-test.txt" 17 | } 18 | }' 19 | 20 | # file-sink-test 커넥터 실행 상태 확인 21 | $ curl http://localhost:8083/connectors/file-sink-test/status 22 | 23 | # file-sink-test 커넥터의 태스크 확인 24 | $ curl http://localhost:8083/connectors/file-sink-test/tasks 25 | 26 | # file-sink-test 커넥터 특정 태스크 상태 확인 27 | $ curl http://localhost:8083/connectors/file-sink-test/tasks/0/status 28 | 29 | # file-sink-test 커넥터 특정 태스크 재시작 30 | $ curl -X POST http://localhost:8083/connectors/file-sink-test/tasks/0/restart 31 | 32 | # file-sink-test 커넥터 수정 33 | $ curl -X PUT http://localhost:8083/connectors/file-sink-test/config \ 34 | -H 'Content-Type: application/json' \ 35 | -d '{ 36 | "topics":"test", 37 | "connector.class":"org.apache.kafka.connect.file.FileStreamSinkConnector", 38 | "tasks.max":1, 39 | "file":"/tmp/connect-test2.txt" 40 | }' 41 | 42 | # file-sink-test 커넥터 중지 43 | $ curl -X PUT http://localhost:8083/connectors/file-sink-test/pause 44 | 45 | # file-sink-test 커넥터 시작 46 | $ curl -X PUT http://localhost:8083/connectors/file-sink-test/resume 47 | 48 | # file-sink-test 커넥터 재시작 49 | $ curl -X POST http://localhost:8083/connectors/file-sink-test/restart 50 | 51 | # file-sink-test 커넥터 삭제 52 | $ curl -X DELETE http://localhost:8083/connectors/file-sink-test 53 | 54 | # mac에서 내 ip 확인 55 | $ ifconfig | grep "inet " | awk '{ print $2}' 56 | 57 | # kafka-connect-ui 도커 실행 58 | $ docker run --rm -it -p 8000:8000 \ 59 | -e "CONNECT_URL=http://{{my-ip}}:8083" \ 60 | landoop/kafka-connect-ui -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/connect-distributed.properties: -------------------------------------------------------------------------------- 1 | bootstrap.servers=my-kafka:9092 2 | group.id=connect-cluster 3 | 4 | key.converter=org.apache.kafka.connect.storage.StringConverter 5 | value.converter=org.apache.kafka.connect.storage.StringConverter 6 | key.converter.schemas.enable=false 7 | value.converter.schemas.enable=false 8 | 9 | offset.storage.topic=connect-offsets 10 | offset.storage.replication.factor=1 11 | config.storage.topic=connect-configs 12 | config.storage.replication.factor=1 13 | status.storage.topic=connect-status 14 | status.storage.replication.factor=1 15 | offset.flush.interval.ms=10000 16 | -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-sink-connector/.DS_Store -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/executionHistory/executionHistory.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/executionHistory/executionHistory.bin -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/executionHistory/executionHistory.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/executionHistory/executionHistory.lock -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/fileChanges/last-build.bin: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/fileContent/fileContent.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/fileContent/fileContent.lock -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/fileHashes/fileHashes.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/fileHashes/fileHashes.bin -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/fileHashes/fileHashes.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/fileHashes/fileHashes.lock -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/javaCompile/classAnalysis.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/javaCompile/classAnalysis.bin -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/javaCompile/javaCompile.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/javaCompile/javaCompile.lock -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/javaCompile/taskHistory.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/6.1.1/javaCompile/taskHistory.bin -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/buildOutputCleanup/buildOutputCleanup.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/buildOutputCleanup/buildOutputCleanup.lock -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/buildOutputCleanup/cache.properties: -------------------------------------------------------------------------------- 1 | #Tue Jun 09 13:56:51 KST 2020 2 | gradle.version=6.1.1 3 | -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/buildOutputCleanup/outputFiles.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/buildOutputCleanup/outputFiles.bin -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/checksums/checksums.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/checksums/checksums.lock -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/checksums/md5-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/checksums/md5-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/checksums/sha1-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/checksums/sha1-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/vcs-1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-sink-connector/.gradle/vcs-1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:connect-api:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } 19 | 20 | jar { 21 | from { 22 | configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } 23 | } 24 | } -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'simple-sink-connector' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/src/main/java/com/example/SingleFileSinkConnector.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.common.config.ConfigDef; 4 | import org.apache.kafka.common.config.ConfigException; 5 | import org.apache.kafka.connect.connector.Task; 6 | import org.apache.kafka.connect.errors.ConnectException; 7 | import org.apache.kafka.connect.sink.SinkConnector; 8 | 9 | import java.util.ArrayList; 10 | import java.util.HashMap; 11 | import java.util.List; 12 | import java.util.Map; 13 | 14 | public class SingleFileSinkConnector extends SinkConnector { 15 | 16 | private Map configProperties; 17 | 18 | @Override 19 | public String version() { 20 | return "1.0"; 21 | } 22 | 23 | @Override 24 | public void start(Map props) { 25 | this.configProperties = props; 26 | try { 27 | new SingleFileSinkConnectorConfig(props); 28 | } catch (ConfigException e) { 29 | throw new ConnectException(e.getMessage(), e); 30 | } 31 | } 32 | 33 | @Override 34 | public Class taskClass() { 35 | return SingleFileSinkTask.class; 36 | } 37 | 38 | @Override 39 | public List> taskConfigs(int maxTasks) { 40 | List> taskConfigs = new ArrayList<>(); 41 | Map taskProps = new HashMap<>(); 42 | taskProps.putAll(configProperties); 43 | for (int i = 0; i < maxTasks; i++) { 44 | taskConfigs.add(taskProps); 45 | } 46 | return taskConfigs; 47 | } 48 | 49 | @Override 50 | public ConfigDef config() { 51 | return SingleFileSinkConnectorConfig.CONFIG; 52 | } 53 | 54 | @Override 55 | public void stop() { 56 | } 57 | } 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/src/main/java/com/example/SingleFileSinkConnectorConfig.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.common.config.AbstractConfig; 4 | import org.apache.kafka.common.config.ConfigDef; 5 | import org.apache.kafka.common.config.ConfigDef.Importance; 6 | import org.apache.kafka.common.config.ConfigDef.Type; 7 | import java.util.Map; 8 | 9 | public class SingleFileSinkConnectorConfig extends AbstractConfig { 10 | 11 | public static final String DIR_FILE_NAME = "file"; 12 | private static final String DIR_FILE_NAME_DEFAULT_VALUE = "/tmp/kafka.txt"; 13 | private static final String DIR_FILE_NAME_DOC = "저장할 디렉토리와 파일 이름"; 14 | 15 | public static ConfigDef CONFIG = new ConfigDef().define(DIR_FILE_NAME, 16 | Type.STRING, 17 | DIR_FILE_NAME_DEFAULT_VALUE, 18 | Importance.HIGH, 19 | DIR_FILE_NAME_DOC); 20 | 21 | public SingleFileSinkConnectorConfig(Map props) { 22 | super(CONFIG, props); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-sink-connector/src/main/java/com/example/SingleFileSinkTask.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.consumer.OffsetAndMetadata; 4 | import org.apache.kafka.common.TopicPartition; 5 | import org.apache.kafka.connect.errors.ConnectException; 6 | import org.apache.kafka.connect.sink.SinkRecord; 7 | import org.apache.kafka.connect.sink.SinkTask; 8 | 9 | import java.io.File; 10 | import java.io.FileWriter; 11 | import java.io.IOException; 12 | import java.util.Collection; 13 | import java.util.Map; 14 | 15 | public class SingleFileSinkTask extends SinkTask { 16 | private SingleFileSinkConnectorConfig config; 17 | private File file; 18 | private FileWriter fileWriter; 19 | 20 | @Override 21 | public String version() { 22 | return "1.0"; 23 | } 24 | 25 | @Override 26 | public void start(Map props) { 27 | try { 28 | config = new SingleFileSinkConnectorConfig(props); 29 | file = new File(config.getString(config.DIR_FILE_NAME)); 30 | fileWriter = new FileWriter(file, true); 31 | } catch (Exception e) { 32 | throw new ConnectException(e.getMessage(), e); 33 | } 34 | 35 | } 36 | 37 | @Override 38 | public void put(Collection records) { 39 | try { 40 | for (SinkRecord record : records) { 41 | fileWriter.write(record.value().toString() + "\n"); 42 | } 43 | } catch (IOException e) { 44 | throw new ConnectException(e.getMessage(), e); 45 | } 46 | } 47 | 48 | @Override 49 | public void flush(Map offsets) { 50 | try { 51 | fileWriter.flush(); 52 | } catch (IOException e) { 53 | throw new ConnectException(e.getMessage(), e); 54 | } 55 | } 56 | 57 | @Override 58 | public void stop() { 59 | try { 60 | fileWriter.close(); 61 | } catch (IOException e) { 62 | throw new ConnectException(e.getMessage(), e); 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-source-connector/.DS_Store -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/executionHistory/executionHistory.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/executionHistory/executionHistory.bin -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/executionHistory/executionHistory.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/executionHistory/executionHistory.lock -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/fileChanges/last-build.bin: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/fileContent/fileContent.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/fileContent/fileContent.lock -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/fileHashes/fileHashes.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/fileHashes/fileHashes.bin -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/fileHashes/fileHashes.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/fileHashes/fileHashes.lock -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/javaCompile/classAnalysis.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/javaCompile/classAnalysis.bin -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/javaCompile/javaCompile.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/javaCompile/javaCompile.lock -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/javaCompile/taskHistory.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/6.1.1/javaCompile/taskHistory.bin -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/buildOutputCleanup/buildOutputCleanup.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/buildOutputCleanup/buildOutputCleanup.lock -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/buildOutputCleanup/cache.properties: -------------------------------------------------------------------------------- 1 | #Tue Jun 09 13:56:51 KST 2020 2 | gradle.version=6.1.1 3 | -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/buildOutputCleanup/outputFiles.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/buildOutputCleanup/outputFiles.bin -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/checksums/checksums.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/checksums/checksums.lock -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/checksums/sha1-checksums.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/checksums/sha1-checksums.bin -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/vcs-1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter3/3.6 kafka-connector/simple-source-connector/.gradle/vcs-1/gc.properties -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:connect-api:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } 19 | 20 | jar { 21 | from { 22 | configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } 23 | } 24 | } -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'simple-source-connector' 2 | 3 | -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/src/main/java/com/example/SingleFileSourceConnector.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.common.config.ConfigDef; 4 | import org.apache.kafka.common.config.ConfigException; 5 | import org.apache.kafka.connect.connector.Task; 6 | import org.apache.kafka.connect.errors.ConnectException; 7 | import org.apache.kafka.connect.source.SourceConnector; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.util.ArrayList; 12 | import java.util.HashMap; 13 | import java.util.List; 14 | import java.util.Map; 15 | 16 | public class SingleFileSourceConnector extends SourceConnector { 17 | 18 | private final Logger logger = LoggerFactory.getLogger(SingleFileSourceConnector.class); 19 | 20 | private Map configProperties; 21 | 22 | @Override 23 | public String version() { 24 | return "1.0"; 25 | } 26 | 27 | @Override 28 | public void start(Map props) { 29 | this.configProperties = props; 30 | try { 31 | new SingleFileSourceConnectorConfig(props); 32 | } catch (ConfigException e) { 33 | throw new ConnectException(e.getMessage(), e); 34 | } 35 | } 36 | 37 | @Override 38 | public Class taskClass() { 39 | return SingleFileSourceTask.class; 40 | } 41 | 42 | @Override 43 | public List> taskConfigs(int maxTasks) { 44 | List> taskConfigs = new ArrayList<>(); 45 | Map taskProps = new HashMap<>(); 46 | taskProps.putAll(configProperties); 47 | for (int i = 0; i < maxTasks; i++) { 48 | taskConfigs.add(taskProps); 49 | } 50 | return taskConfigs; 51 | } 52 | 53 | @Override 54 | public ConfigDef config() { 55 | return SingleFileSourceConnectorConfig.CONFIG; 56 | } 57 | 58 | @Override 59 | public void stop() { 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /Chapter3/3.6 kafka-connector/simple-source-connector/src/main/java/com/example/SingleFileSourceConnectorConfig.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.common.config.AbstractConfig; 4 | import org.apache.kafka.common.config.ConfigDef; 5 | import org.apache.kafka.common.config.ConfigDef.Importance; 6 | import org.apache.kafka.common.config.ConfigDef.Type; 7 | 8 | import java.util.Map; 9 | 10 | public class SingleFileSourceConnectorConfig extends AbstractConfig { 11 | 12 | public static final String DIR_FILE_NAME = "file"; 13 | private static final String DIR_FILE_NAME_DEFAULT_VALUE = "/tmp/kafka.txt"; 14 | private static final String DIR_FILE_NAME_DOC = "읽을 파일 경로와 이름"; 15 | 16 | public static final String TOPIC_NAME = "topic"; 17 | private static final String TOPIC_DEFAULT_VALUE = "test"; 18 | private static final String TOPIC_DOC = "보낼 토픽명"; 19 | 20 | public static ConfigDef CONFIG = new ConfigDef().define(DIR_FILE_NAME, 21 | Type.STRING, 22 | DIR_FILE_NAME_DEFAULT_VALUE, 23 | Importance.HIGH, 24 | DIR_FILE_NAME_DOC) 25 | .define(TOPIC_NAME, 26 | Type.STRING, 27 | TOPIC_DEFAULT_VALUE, 28 | Importance.HIGH, 29 | TOPIC_DOC); 30 | 31 | public SingleFileSourceConnectorConfig(Map props) { 32 | super(CONFIG, props); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Chapter4/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter4/.DS_Store -------------------------------------------------------------------------------- /Chapter4/4.2.2 idempotence-producer/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } -------------------------------------------------------------------------------- /Chapter4/4.2.2 idempotence-producer/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'idempotence-producer' 2 | 3 | -------------------------------------------------------------------------------- /Chapter4/4.2.2 idempotence-producer/src/main/java/com/example/IdempotenceProducer.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.ProducerConfig; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.apache.kafka.common.serialization.StringSerializer; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.util.Properties; 11 | 12 | public class IdempotenceProducer { 13 | private final static Logger logger = LoggerFactory.getLogger(IdempotenceProducer.class); 14 | private final static String TOPIC_NAME = "test"; 15 | private final static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 16 | 17 | public static void main(String[] args) { 18 | 19 | Properties configs = new Properties(); 20 | configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 21 | configs.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 22 | configs.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 23 | configs.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, true); 24 | 25 | KafkaProducer producer = new KafkaProducer<>(configs); 26 | 27 | String messageValue = "testMessage"; 28 | ProducerRecord record = new ProducerRecord<>(TOPIC_NAME, messageValue); 29 | producer.send(record); 30 | logger.info("{}", record); 31 | producer.flush(); 32 | producer.close(); 33 | } 34 | } -------------------------------------------------------------------------------- /Chapter4/4.2.3 transaction-consumer/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } -------------------------------------------------------------------------------- /Chapter4/4.2.3 transaction-consumer/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'transaction-consumer' 2 | 3 | -------------------------------------------------------------------------------- /Chapter4/4.2.3 transaction-consumer/src/main/java/com/example/TransactionConsumer.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerConfig; 4 | import org.apache.kafka.clients.consumer.ConsumerRecord; 5 | import org.apache.kafka.clients.consumer.ConsumerRecords; 6 | import org.apache.kafka.clients.consumer.KafkaConsumer; 7 | import org.apache.kafka.common.serialization.StringDeserializer; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.time.Duration; 12 | import java.util.Arrays; 13 | import java.util.Properties; 14 | 15 | public class TransactionConsumer { 16 | private final static Logger logger = LoggerFactory.getLogger(TransactionConsumer.class); 17 | private final static String TOPIC_NAME = "test"; 18 | private final static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 19 | private final static String GROUP_ID = "test-group"; 20 | 21 | public static void main(String[] args) { 22 | Properties configs = new Properties(); 23 | configs.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 24 | configs.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID); 25 | configs.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 26 | configs.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 27 | configs.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed"); 28 | 29 | KafkaConsumer consumer = new KafkaConsumer<>(configs); 30 | 31 | consumer.subscribe(Arrays.asList(TOPIC_NAME)); 32 | 33 | while (true) { 34 | ConsumerRecords records = consumer.poll(Duration.ofSeconds(1)); 35 | for (ConsumerRecord record : records) { 36 | logger.info("record:{}", record); 37 | } 38 | } 39 | } 40 | } -------------------------------------------------------------------------------- /Chapter4/4.2.3 transaction-producer/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } -------------------------------------------------------------------------------- /Chapter4/4.2.3 transaction-producer/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'transaction-producer' 2 | 3 | -------------------------------------------------------------------------------- /Chapter4/4.2.3 transaction-producer/src/main/java/com/example/TransactionProducer.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.ProducerConfig; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | import org.apache.kafka.common.serialization.StringSerializer; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.util.Properties; 11 | 12 | public class TransactionProducer { 13 | private final static Logger logger = LoggerFactory.getLogger(TransactionProducer.class); 14 | private final static String TOPIC_NAME = "test"; 15 | private final static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 16 | 17 | public static void main(String[] args) { 18 | 19 | Properties configs = new Properties(); 20 | configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 21 | configs.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 22 | configs.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 23 | configs.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, "my-transaction-id"); 24 | 25 | KafkaProducer producer = new KafkaProducer<>(configs); 26 | producer.initTransactions(); 27 | producer.beginTransaction(); 28 | String messageValue = "testMessage"; 29 | ProducerRecord record = new ProducerRecord<>(TOPIC_NAME, messageValue); 30 | producer.send(record); 31 | logger.info("{}", record); 32 | producer.flush(); 33 | 34 | 35 | producer.commitTransaction(); 36 | //producer.abortTransaction(); 37 | 38 | producer.close(); 39 | } 40 | } -------------------------------------------------------------------------------- /Chapter4/4.3 kafka-consumer/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter4/4.3 kafka-consumer/.DS_Store -------------------------------------------------------------------------------- /Chapter4/4.3 kafka-consumer/kafka-consumer-with-multi-worker-thread/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } 19 | -------------------------------------------------------------------------------- /Chapter4/4.3 kafka-consumer/kafka-consumer-with-multi-worker-thread/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-consumer-with-worker-thread' 2 | 3 | -------------------------------------------------------------------------------- /Chapter4/4.3 kafka-consumer/kafka-consumer-with-multi-worker-thread/src/main/java/com/example/ConsumerWorker.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | public class ConsumerWorker implements Runnable { 7 | 8 | private final static Logger logger = LoggerFactory.getLogger(ConsumerWorker.class); 9 | private String recordValue; 10 | 11 | ConsumerWorker(String recordValue) { 12 | this.recordValue = recordValue; 13 | } 14 | 15 | @Override 16 | public void run() { 17 | logger.info("thread:{}\trecord:{}", Thread.currentThread().getName(), recordValue); 18 | } 19 | } -------------------------------------------------------------------------------- /Chapter4/4.3 kafka-consumer/kafka-multi-consumer-thread-by-partition/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } 19 | -------------------------------------------------------------------------------- /Chapter4/4.3 kafka-consumer/kafka-multi-consumer-thread-by-partition/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-consumer-multi-thread-by-partition' 2 | 3 | -------------------------------------------------------------------------------- /Chapter4/4.3 kafka-consumer/kafka-multi-consumer-thread-by-partition/src/main/java/com/example/ConsumerWorker.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRecord; 4 | import org.apache.kafka.clients.consumer.ConsumerRecords; 5 | import org.apache.kafka.clients.consumer.KafkaConsumer; 6 | import org.apache.kafka.common.errors.WakeupException; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.time.Duration; 11 | import java.util.Arrays; 12 | import java.util.Properties; 13 | 14 | public class ConsumerWorker implements Runnable { 15 | 16 | private final static Logger logger = LoggerFactory.getLogger(ConsumerWorker.class); 17 | 18 | private final Properties prop; 19 | private final String topic; 20 | private final String threadName; 21 | private KafkaConsumer consumer; 22 | 23 | ConsumerWorker(Properties prop, String topic, int number) { 24 | this.prop = prop; 25 | this.topic = topic; 26 | this.threadName = "consumer-thread-" + number; 27 | } 28 | 29 | @Override 30 | public void run() { 31 | consumer = new KafkaConsumer<>(prop); 32 | consumer.subscribe(Arrays.asList(topic)); 33 | try { 34 | while (true) { 35 | ConsumerRecords records = consumer.poll(Duration.ofSeconds(1)); 36 | for (ConsumerRecord record : records) { 37 | logger.info("{}", record); 38 | } 39 | consumer.commitSync(); 40 | } 41 | } catch (WakeupException e) { 42 | System.out.println(threadName + " trigger WakeupException"); 43 | } finally { 44 | consumer.commitSync(); 45 | consumer.close(); 46 | } 47 | } 48 | 49 | public void shutdown() { 50 | consumer.wakeup(); 51 | } 52 | } -------------------------------------------------------------------------------- /Chapter4/4.3 kafka-consumer/kafka-multi-consumer-thread/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } 19 | -------------------------------------------------------------------------------- /Chapter4/4.3 kafka-consumer/kafka-multi-consumer-thread/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-consumer-multi-thread' 2 | 3 | -------------------------------------------------------------------------------- /Chapter4/4.3 kafka-consumer/kafka-multi-consumer-thread/src/main/java/com/example/ConsumerWorker.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRecord; 4 | import org.apache.kafka.clients.consumer.ConsumerRecords; 5 | import org.apache.kafka.clients.consumer.KafkaConsumer; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.time.Duration; 10 | import java.util.Arrays; 11 | import java.util.Properties; 12 | 13 | public class ConsumerWorker implements Runnable { 14 | private final static Logger logger = LoggerFactory.getLogger(ConsumerWorker.class); 15 | private Properties prop; 16 | private String topic; 17 | private String threadName; 18 | private KafkaConsumer consumer; 19 | 20 | ConsumerWorker(Properties prop, String topic, int number) { 21 | this.prop = prop; 22 | this.topic = topic; 23 | this.threadName = "consumer-thread-" + number; 24 | } 25 | 26 | @Override 27 | public void run() { 28 | consumer = new KafkaConsumer<>(prop); 29 | consumer.subscribe(Arrays.asList(topic)); 30 | while (true) { 31 | ConsumerRecords records = consumer.poll(Duration.ofSeconds(1)); 32 | for (ConsumerRecord record : records) { 33 | logger.info("{}", record); 34 | } 35 | consumer.commitSync(); 36 | } 37 | } 38 | } -------------------------------------------------------------------------------- /Chapter4/4.3 kafka-consumer/kafka-multi-consumer-thread/src/main/java/com/example/MultiConsumerThread.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerConfig; 4 | import org.apache.kafka.common.serialization.StringDeserializer; 5 | 6 | import java.util.Properties; 7 | import java.util.concurrent.ExecutorService; 8 | import java.util.concurrent.Executors; 9 | 10 | public class MultiConsumerThread { 11 | 12 | private final static String TOPIC_NAME = "test"; 13 | private final static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 14 | private final static String GROUP_ID = "test-group"; 15 | private final static int CONSUMER_COUNT = 3; 16 | 17 | public static void main(String[] args) { 18 | Properties configs = new Properties(); 19 | configs.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 20 | configs.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID); 21 | configs.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 22 | configs.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 23 | 24 | ExecutorService executorService = Executors.newCachedThreadPool(); 25 | for (int i = 0; i < CONSUMER_COUNT; i++) { 26 | ConsumerWorker worker = new ConsumerWorker(configs, TOPIC_NAME, i); 27 | executorService.execute(worker); 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter4/4.4 spring-kafka/.DS_Store -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-batch-listener/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0-SNAPSHOT' 7 | 8 | repositories { 9 | mavenCentral() 10 | } 11 | 12 | dependencies { 13 | implementation 'org.springframework.kafka:spring-kafka:2.5.10.RELEASE' 14 | implementation 'org.springframework.boot:spring-boot-starter:2.4.0' 15 | 16 | } 17 | -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-batch-listener/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'spring-kafka-batch-listener' 2 | 3 | -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-batch-listener/src/main/java/com/example/SpringConsumerApplication.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRecords; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.boot.SpringApplication; 7 | import org.springframework.boot.autoconfigure.SpringBootApplication; 8 | import org.springframework.kafka.annotation.KafkaListener; 9 | 10 | import java.util.List; 11 | 12 | @SpringBootApplication 13 | public class SpringConsumerApplication { 14 | public static Logger logger = LoggerFactory.getLogger(SpringConsumerApplication.class); 15 | 16 | public static void main(String[] args) { 17 | SpringApplication application = new SpringApplication(SpringConsumerApplication.class); 18 | application.run(args); 19 | } 20 | 21 | @KafkaListener(topics = "test", 22 | groupId = "test-group-01") 23 | public void batchListener(ConsumerRecords records) { 24 | records.forEach(record -> logger.info(record.toString())); 25 | } 26 | 27 | @KafkaListener(topics = "test", 28 | groupId = "test-group-02") 29 | public void batchListener(List list) { 30 | list.forEach(recordValue -> logger.info(recordValue)); 31 | } 32 | 33 | @KafkaListener(topics = "test", 34 | groupId = "test-group-03", 35 | concurrency = "3") 36 | public void concurrentBatchListener(ConsumerRecords records) { 37 | records.forEach(record -> logger.info(record.toString())); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-batch-listener/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | kafka: 3 | consumer: 4 | bootstrap-servers: my-kafka:9092 5 | listener: 6 | type: BATCH -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-commit-listener/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0-SNAPSHOT' 7 | 8 | repositories { 9 | mavenCentral() 10 | } 11 | 12 | dependencies { 13 | implementation 'org.springframework.kafka:spring-kafka:2.5.10.RELEASE' 14 | implementation 'org.springframework.boot:spring-boot-starter:2.4.0' 15 | 16 | } 17 | -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-commit-listener/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'spring-kafka-commit-listener' 2 | 3 | -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-commit-listener/src/main/java/com/example/SpringConsumerApplication.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.consumer.Consumer; 4 | import org.apache.kafka.clients.consumer.ConsumerRecords; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.boot.SpringApplication; 8 | import org.springframework.boot.autoconfigure.SpringBootApplication; 9 | import org.springframework.kafka.annotation.KafkaListener; 10 | import org.springframework.kafka.support.Acknowledgment; 11 | 12 | @SpringBootApplication 13 | public class SpringConsumerApplication { 14 | public static Logger logger = LoggerFactory.getLogger(SpringConsumerApplication.class); 15 | 16 | 17 | public static void main(String[] args) { 18 | SpringApplication application = new SpringApplication(SpringConsumerApplication.class); 19 | application.run(args); 20 | } 21 | 22 | @KafkaListener(topics = "test", groupId = "test-group-01") 23 | public void commitListener(ConsumerRecords records, Acknowledgment ack) { 24 | records.forEach(record -> logger.info(record.toString())); 25 | ack.acknowledge(); 26 | } 27 | 28 | @KafkaListener(topics = "test", groupId = "test-group-02") 29 | public void consumerCommitListener(ConsumerRecords records, Consumer consumer) { 30 | records.forEach(record -> logger.info(record.toString())); 31 | consumer.commitSync(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-commit-listener/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | kafka: 3 | consumer: 4 | bootstrap-servers: my-kafka:9092 5 | listener: 6 | type: BATCH 7 | ack-mode: MANUAL_IMMEDIATE -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-listener-container/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0-SNAPSHOT' 7 | 8 | repositories { 9 | mavenCentral() 10 | } 11 | 12 | dependencies { 13 | implementation 'org.springframework.kafka:spring-kafka:2.5.10.RELEASE' 14 | implementation 'org.springframework.boot:spring-boot-starter:2.4.0' 15 | } 16 | -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-listener-container/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'spring-kafka-listener-container' 2 | 3 | -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-listener-container/src/main/java/com/example/SpringConsumerApplication.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.boot.SpringApplication; 6 | import org.springframework.boot.autoconfigure.SpringBootApplication; 7 | import org.springframework.kafka.annotation.KafkaListener; 8 | 9 | @SpringBootApplication 10 | public class SpringConsumerApplication { 11 | public static Logger logger = LoggerFactory.getLogger(SpringConsumerApplication.class); 12 | 13 | public static void main(String[] args) { 14 | SpringApplication application = new SpringApplication(SpringConsumerApplication.class); 15 | application.run(args); 16 | } 17 | 18 | @KafkaListener(topics = "test", 19 | groupId = "test-group", 20 | containerFactory = "customContainerFactory") 21 | public void customListener(String data) { 22 | logger.info(data); 23 | } 24 | } 25 | 26 | -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-producer/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0-SNAPSHOT' 7 | 8 | repositories { 9 | mavenCentral() 10 | } 11 | 12 | dependencies { 13 | implementation 'org.springframework.kafka:spring-kafka:2.5.10.RELEASE' 14 | implementation 'org.springframework.boot:spring-boot-starter:2.4.0' 15 | } -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-producer/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'spring-kafka-producer' 2 | 3 | -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-producer/src/main/java/com/example/SpringProducerApplication.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.boot.CommandLineRunner; 5 | import org.springframework.boot.SpringApplication; 6 | import org.springframework.boot.autoconfigure.SpringBootApplication; 7 | import org.springframework.kafka.core.KafkaTemplate; 8 | 9 | @SpringBootApplication 10 | public class SpringProducerApplication implements CommandLineRunner { 11 | 12 | private static String TOPIC_NAME = "test"; 13 | 14 | @Autowired 15 | private KafkaTemplate template; 16 | 17 | public static void main(String[] args) { 18 | SpringApplication application = new SpringApplication(SpringProducerApplication.class); 19 | application.run(args); 20 | } 21 | 22 | @Override 23 | public void run(String... args) { 24 | for (int i = 0; i < 10; i++) { 25 | template.send(TOPIC_NAME, "test" + i); 26 | } 27 | System.exit(0); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-producer/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | kafka: 3 | producer: 4 | bootstrap-servers: my-kafka:9092 5 | acks: all -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-record-listener/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0-SNAPSHOT' 7 | 8 | repositories { 9 | mavenCentral() 10 | } 11 | 12 | dependencies { 13 | implementation 'org.springframework.kafka:spring-kafka:2.5.10.RELEASE' 14 | implementation 'org.springframework.boot:spring-boot-starter:2.4.0' 15 | 16 | } 17 | -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-record-listener/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'spring-kafka-record-listener' 2 | 3 | -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-record-listener/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | kafka: 3 | consumer: 4 | bootstrap-servers: my-kafka:9092 -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-template-producer/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0-SNAPSHOT' 7 | 8 | repositories { 9 | mavenCentral() 10 | } 11 | dependencies { 12 | implementation 'org.springframework.kafka:spring-kafka:2.5.10.RELEASE' 13 | implementation 'org.springframework.boot:spring-boot-starter:2.4.0' 14 | 15 | } -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-template-producer/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'spring-kafka-template-producer' 2 | 3 | -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-template-producer/src/main/java/com/example/KafkaTemplateConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.kafka.clients.producer.ProducerConfig; 4 | import org.apache.kafka.common.serialization.StringSerializer; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | import org.springframework.kafka.core.*; 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | @Configuration 12 | public class KafkaTemplateConfiguration { 13 | 14 | @Bean 15 | public KafkaTemplate customKafkaTemplate() { 16 | 17 | Map props = new HashMap<>(); 18 | props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "my-kafka:9092"); 19 | props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); 20 | props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); 21 | props.put(ProducerConfig.ACKS_CONFIG, "all"); 22 | 23 | ProducerFactory pf = new DefaultKafkaProducerFactory<>(props); 24 | 25 | return new KafkaTemplate<>(pf); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /Chapter4/4.4 spring-kafka/spring-kafka-template-producer/src/main/java/com/example/SpringProducerApplication.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.boot.CommandLineRunner; 5 | import org.springframework.boot.SpringApplication; 6 | import org.springframework.boot.WebApplicationType; 7 | import org.springframework.boot.autoconfigure.SpringBootApplication; 8 | import org.springframework.kafka.core.KafkaProducerException; 9 | import org.springframework.kafka.core.KafkaSendCallback; 10 | import org.springframework.kafka.core.KafkaTemplate; 11 | import org.springframework.kafka.support.SendResult; 12 | import org.springframework.util.concurrent.ListenableFuture; 13 | 14 | @SpringBootApplication 15 | public class SpringProducerApplication implements CommandLineRunner { 16 | 17 | private static String TOPIC_NAME = "test"; 18 | 19 | @Autowired 20 | private KafkaTemplate customKafkaTemplate; 21 | 22 | public static void main(String[] args) { 23 | SpringApplication application = new SpringApplication(SpringProducerApplication.class); 24 | application.run(args); 25 | } 26 | 27 | @Override 28 | public void run(String... args) { 29 | ListenableFuture> future = customKafkaTemplate.send(TOPIC_NAME, "test"); 30 | future.addCallback(new KafkaSendCallback() { 31 | @Override 32 | public void onSuccess(SendResult result) { 33 | 34 | } 35 | 36 | @Override 37 | public void onFailure(KafkaProducerException ex) { 38 | 39 | } 40 | }); 41 | System.exit(0); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /Chapter5/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/.DS_Store -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/.DS_Store -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.DS_Store -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/executionHistory/executionHistory.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/executionHistory/executionHistory.bin -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/executionHistory/executionHistory.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/executionHistory/executionHistory.lock -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/fileChanges/last-build.bin: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/fileContent/fileContent.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/fileContent/fileContent.lock -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/fileHashes/fileHashes.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/fileHashes/fileHashes.bin -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/fileHashes/fileHashes.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/fileHashes/fileHashes.lock -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/gc.properties -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/javaCompile/classAnalysis.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/javaCompile/classAnalysis.bin -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/javaCompile/javaCompile.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/javaCompile/javaCompile.lock -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/javaCompile/taskHistory.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/5.2.1/javaCompile/taskHistory.bin -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/buildOutputCleanup/buildOutputCleanup.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/buildOutputCleanup/buildOutputCleanup.lock -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/buildOutputCleanup/cache.properties: -------------------------------------------------------------------------------- 1 | #Wed Sep 30 20:52:09 KST 2020 2 | gradle.version=5.2.1 3 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/buildOutputCleanup/outputFiles.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/buildOutputCleanup/outputFiles.bin -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/vcs-1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.gradle/vcs-1/gc.properties -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /workspace.xml -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.idea/codeStyles/Project.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.idea/codeStyles/codeStyleConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.idea/gradle.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 18 | 19 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.idea/jarRepositories.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 9 | 10 | 14 | 15 | 19 | 20 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.pipeline' 6 | version '1.0' 7 | 8 | repositories { 9 | mavenCentral() 10 | } 11 | 12 | dependencies { 13 | implementation 'org.slf4j:slf4j-simple:1.7.30' 14 | implementation 'com.google.code.gson:gson:2.8.6' 15 | implementation 'org.apache.kafka:connect-api:2.5.0' 16 | implementation 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.9.2' 17 | } 18 | 19 | jar { 20 | from { 21 | configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } 22 | } 23 | } -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/build/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/build/.DS_Store -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/build/libs/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/build/libs/.DS_Store -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/build/tmp/jar/MANIFEST.MF: -------------------------------------------------------------------------------- 1 | Manifest-Version: 1.0 2 | 3 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Wed Sep 30 20:52:24 KST 2020 2 | distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-all.zip 3 | distributionBase=GRADLE_USER_HOME 4 | distributionPath=wrapper/dists 5 | zipStorePath=wrapper/dists 6 | zipStoreBase=GRADLE_USER_HOME 7 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'elasticsearch-kafka-connector' 2 | 3 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/src/main/java/com/pipeline/ElasticSearchSinkConnector.java: -------------------------------------------------------------------------------- 1 | package com.pipeline; 2 | 3 | import com.pipeline.config.ElasticSearchSinkConnectorConfig; 4 | import org.apache.kafka.common.config.ConfigDef; 5 | import org.apache.kafka.common.config.ConfigException; 6 | import org.apache.kafka.connect.connector.Task; 7 | import org.apache.kafka.connect.errors.ConnectException; 8 | import org.apache.kafka.connect.sink.SinkConnector; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.util.ArrayList; 13 | import java.util.HashMap; 14 | import java.util.List; 15 | import java.util.Map; 16 | 17 | public class ElasticSearchSinkConnector extends SinkConnector { 18 | 19 | private final Logger logger = LoggerFactory.getLogger(ElasticSearchSinkConnector.class); 20 | 21 | private Map configProperties; 22 | 23 | @Override 24 | public String version() { 25 | return "1.0"; 26 | } 27 | 28 | @Override 29 | public void start(Map props) { 30 | this.configProperties = props; 31 | try { 32 | new ElasticSearchSinkConnectorConfig(props); 33 | } catch (ConfigException e) { 34 | throw new ConnectException(e.getMessage(), e); 35 | } 36 | } 37 | 38 | @Override 39 | public Class taskClass() { 40 | return ElasticSearchSinkTask.class; 41 | } 42 | 43 | @Override 44 | public List> taskConfigs(int maxTasks) { 45 | List> taskConfigs = new ArrayList<>(); 46 | Map taskProps = new HashMap<>(); 47 | taskProps.putAll(configProperties); 48 | for (int i = 0; i < maxTasks; i++) { 49 | taskConfigs.add(taskProps); 50 | } 51 | return taskConfigs; 52 | } 53 | 54 | @Override 55 | public ConfigDef config() { 56 | return ElasticSearchSinkConnectorConfig.CONFIG; 57 | } 58 | 59 | @Override 60 | public void stop() { 61 | logger.info("Stop elasticsearch connector"); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/elasticsearch-kafka-connector/src/main/java/com/pipeline/config/ElasticSearchSinkConnectorConfig.java: -------------------------------------------------------------------------------- 1 | package com.pipeline.config; 2 | 3 | import org.apache.kafka.common.config.AbstractConfig; 4 | import org.apache.kafka.common.config.ConfigDef; 5 | import org.apache.kafka.common.config.ConfigDef.Importance; 6 | import org.apache.kafka.common.config.ConfigDef.Type; 7 | import java.util.Map; 8 | 9 | public class ElasticSearchSinkConnectorConfig extends AbstractConfig { 10 | 11 | public static final String ES_CLUSTER_HOST = "es.host"; 12 | private static final String ES_CLUSTER_HOST_DEFAULT_VALUE = "localhost"; 13 | private static final String ES_CLUSTER_HOST_DOC = "엘라스틱서치 호스트를 입력"; 14 | 15 | public static final String ES_CLUSTER_PORT = "es.port"; 16 | private static final String ES_CLUSTER_PORT_DEFAULT_VALUE = "9200"; 17 | private static final String ES_CLUSTER_PORT_DOC = "엘라스틱서치 포트를 입력"; 18 | 19 | public static final String ES_INDEX = "es.index"; 20 | private static final String ES_INDEX_DEFAULT_VALUE = "kafka-connector-index"; 21 | private static final String ES_INDEX_DOC = "엘라스틱서치 인덱스를 입력"; 22 | 23 | public static ConfigDef CONFIG = new ConfigDef().define(ES_CLUSTER_HOST, Type.STRING, ES_CLUSTER_HOST_DEFAULT_VALUE, Importance.HIGH, ES_CLUSTER_HOST_DOC) 24 | .define(ES_CLUSTER_PORT, Type.INT, ES_CLUSTER_PORT_DEFAULT_VALUE, Importance.HIGH, ES_CLUSTER_PORT_DOC) 25 | .define(ES_INDEX, Type.STRING, ES_INDEX_DEFAULT_VALUE, Importance.HIGH, ES_INDEX_DOC); 26 | 27 | public ElasticSearchSinkConnectorConfig(Map props) { 28 | super(CONFIG, props); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/favorite-color-webpage/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Favorite color select webpage 6 | 7 | 17 | 18 | 19 |
20 |

이름을 입력하세요.

21 | 22 | 23 |

마음에 드는 색상을 고르세요.

24 |
    25 |
    • 26 |
    • 27 |
    • 28 |
    • 29 |
30 |
31 | 32 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/install-es-and-kibana.md: -------------------------------------------------------------------------------- 1 | ## Install elasticsearch 2 | - Download binary : https://www.elastic.co/kr/downloads/elasticsearch 3 | ``` 4 | $ bin/elasticsearch 5 | ``` 6 | 7 | - Brew install : https://www.elastic.co/guide/en/elasticsearch/reference/current/brew.html 8 | ``` 9 | $ brew tap elastic/tap 10 | $ brew install elastic/tap/elasticsearch-full 11 | $ elasticsearch 12 | ``` 13 | 14 | ## Install kibana 15 | - Download binary : https://www.elastic.co/kr/downloads/kibana 16 | ``` 17 | $ bin/kibana 18 | ``` 19 | 20 | - Brew install : https://www.elastic.co/guide/en/kibana/current/brew.html 21 | ``` 22 | $ brew tap elastic/tap 23 | $ brew install elastic/tap/kibana-full 24 | $ kibana 25 | ``` 26 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.gradle/6.1.1/executionHistory/executionHistory.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.gradle/6.1.1/executionHistory/executionHistory.lock -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.gradle/6.1.1/fileChanges/last-build.bin: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.gradle/6.1.1/fileHashes/fileHashes.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.gradle/6.1.1/fileHashes/fileHashes.lock -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.gradle/6.1.1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.gradle/6.1.1/gc.properties -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.gradle/buildOutputCleanup/buildOutputCleanup.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.gradle/buildOutputCleanup/buildOutputCleanup.lock -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.gradle/buildOutputCleanup/cache.properties: -------------------------------------------------------------------------------- 1 | #Thu Oct 01 12:38:09 KST 2020 2 | gradle.version=6.1.1 3 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.gradle/checksums/checksums.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.gradle/checksums/checksums.lock -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.gradle/vcs-1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.gradle/vcs-1/gc.properties -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.idea/codeStyles/codeStyleConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.idea/gradle.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 20 | 21 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.idea/jarRepositories.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 9 | 10 | 14 | 15 | 19 | 20 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.idea/kafka-consumer-multi-thread.iml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.idea/kafka-multi-consumer-thread-hdfs-save.iml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.pipeline' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.slf4j:slf4j-simple:1.7.30' 17 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 18 | implementation 'org.apache.hadoop:hadoop-client:3.3.0' 19 | } 20 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Mon Aug 24 16:54:48 KST 2020 2 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.1.1-all.zip 3 | distributionBase=GRADLE_USER_HOME 4 | distributionPath=wrapper/dists 5 | zipStorePath=wrapper/dists 6 | zipStoreBase=GRADLE_USER_HOME 7 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/out/production/classes/META-INF/kafka-multi-consumer-thread-hdfs-save.main.kotlin_module: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-multi-consumer-thread-hdfs-save' 2 | 3 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-multi-consumer-thread-hdfs-save/src/main/java/com/pipeline/HdfsSinkApplication.java: -------------------------------------------------------------------------------- 1 | package com.pipeline; 2 | 3 | import com.pipeline.consumer.ConsumerWorker; 4 | import org.apache.kafka.clients.consumer.ConsumerConfig; 5 | import org.apache.kafka.common.serialization.StringDeserializer; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | import java.util.Properties; 11 | import java.util.concurrent.ExecutorService; 12 | import java.util.concurrent.Executors; 13 | 14 | public class HdfsSinkApplication { 15 | private final static Logger logger = LoggerFactory.getLogger(HdfsSinkApplication.class); 16 | 17 | private final static String BOOTSTRAP_SERVERS = "my-kafka:9092"; 18 | private final static String TOPIC_NAME = "select-color"; 19 | private final static String GROUP_ID = "color-hdfs-save-consumer-group"; 20 | private final static int CONSUMER_COUNT = 2; 21 | private final static List workers = new ArrayList<>(); 22 | 23 | public static void main(String[] args) { 24 | Runtime.getRuntime().addShutdownHook(new ShutdownThread()); 25 | 26 | Properties configs = new Properties(); 27 | configs.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 28 | configs.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID); 29 | configs.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 30 | configs.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 31 | 32 | ExecutorService executorService = Executors.newCachedThreadPool(); 33 | for (int i = 0; i < CONSUMER_COUNT; i++) { 34 | workers.add(new ConsumerWorker(configs, TOPIC_NAME, i)); 35 | } 36 | workers.forEach(executorService::execute); 37 | } 38 | 39 | static class ShutdownThread extends Thread { 40 | public void run() { 41 | logger.info("Shutdown hook"); 42 | workers.forEach(ConsumerWorker::stopAndWakeup); 43 | } 44 | } 45 | } -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/executionHistory/executionHistory.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/executionHistory/executionHistory.bin -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/executionHistory/executionHistory.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/executionHistory/executionHistory.lock -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/fileChanges/last-build.bin: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/fileContent/fileContent.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/fileContent/fileContent.lock -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/fileHashes/fileHashes.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/fileHashes/fileHashes.bin -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/fileHashes/fileHashes.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/fileHashes/fileHashes.lock -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/gc.properties -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/javaCompile/classAnalysis.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/javaCompile/classAnalysis.bin -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/javaCompile/javaCompile.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/javaCompile/javaCompile.lock -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/javaCompile/taskHistory.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/5.2.1/javaCompile/taskHistory.bin -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/buildOutputCleanup/buildOutputCleanup.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/buildOutputCleanup/buildOutputCleanup.lock -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/buildOutputCleanup/cache.properties: -------------------------------------------------------------------------------- 1 | #Tue Sep 29 23:02:41 KST 2020 2 | gradle.version=5.2.1 3 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/buildOutputCleanup/outputFiles.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/buildOutputCleanup/outputFiles.bin -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/vcs-1/gc.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.gradle/vcs-1/gc.properties -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /workspace.xml -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.idea/codeStyles/codeStyleConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.idea/gradle.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 18 | 19 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.pipeline' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.springframework.kafka:spring-kafka:2.5.10.RELEASE' 17 | implementation 'org.springframework.boot:spring-boot-starter-web:2.3.4.RELEASE' 18 | implementation 'com.google.code.gson:gson:2.8.6' 19 | } 20 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/build/resources/main/applicaiton.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | kafka: 3 | bootstrap-servers: my-kafka:9092 4 | producer: 5 | acks: 1 6 | key-serializer: org.apache.kafka.common.serialization.StringSerializer 7 | value-serializer: org.apache.kafka.common.serialization.StringSerializer -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Tue Sep 29 23:05:53 KST 2020 2 | distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-all.zip 3 | distributionBase=GRADLE_USER_HOME 4 | distributionPath=wrapper/dists 5 | zipStorePath=wrapper/dists 6 | zipStoreBase=GRADLE_USER_HOME 7 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-spring-producer-with-rest-controller' 2 | 3 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/src/main/java/com/pipeline/ProduceController.java: -------------------------------------------------------------------------------- 1 | package com.pipeline; 2 | 3 | import com.google.gson.Gson; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.kafka.core.KafkaTemplate; 7 | import org.springframework.kafka.support.SendResult; 8 | import org.springframework.util.concurrent.ListenableFutureCallback; 9 | import org.springframework.web.bind.annotation.*; 10 | 11 | import java.text.SimpleDateFormat; 12 | import java.util.Date; 13 | 14 | 15 | @RestController 16 | @CrossOrigin(origins = "*", allowedHeaders = "*") 17 | public class ProduceController { 18 | 19 | private final Logger logger = LoggerFactory.getLogger(ProduceController.class); 20 | 21 | private final KafkaTemplate kafkaTemplate; 22 | 23 | public ProduceController(KafkaTemplate kafkaTemplate) { 24 | this.kafkaTemplate = kafkaTemplate; 25 | } 26 | 27 | @GetMapping("/api/select") 28 | public void selectColor( 29 | @RequestHeader("user-agent") String userAgentName, 30 | @RequestParam(value = "color")String colorName, 31 | @RequestParam(value = "user")String userName) { 32 | SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZZ"); 33 | Date now = new Date(); 34 | Gson gson = new Gson(); 35 | UserEventVO userEventVO = new UserEventVO(sdfDate.format(now), userAgentName, colorName,userName); 36 | String jsonColorLog = gson.toJson(userEventVO); 37 | kafkaTemplate.send("select-color", jsonColorLog).addCallback(new ListenableFutureCallback>() { 38 | @Override 39 | public void onSuccess(SendResult result) { 40 | logger.info(result.toString()); 41 | } 42 | 43 | @Override 44 | public void onFailure(Throwable ex) { 45 | logger.error(ex.getMessage(), ex); 46 | } 47 | }); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/src/main/java/com/pipeline/RestApiProducer.java: -------------------------------------------------------------------------------- 1 | package com.pipeline; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class RestApiProducer { 8 | public static void main(String[] args) { 9 | SpringApplication.run(RestApiProducer.class, args); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/src/main/java/com/pipeline/UserEventVO.java: -------------------------------------------------------------------------------- 1 | package com.pipeline; 2 | 3 | public class UserEventVO { 4 | 5 | public UserEventVO(String timestamp, String userAgent, String colorName, String userName) { 6 | this.timestamp = timestamp; 7 | this.userAgent = userAgent; 8 | this.colorName = colorName; 9 | this.userName = userName; 10 | } 11 | private String timestamp; 12 | private String userAgent; 13 | private String colorName; 14 | private String userName; 15 | } 16 | -------------------------------------------------------------------------------- /Chapter5/5.1 web-page-event-pipeline/kafka-spring-producer-with-rest-controller/src/main/resources/applicaiton.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | kafka: 3 | bootstrap-servers: my-kafka:9092 4 | producer: 5 | acks: 1 6 | key-serializer: org.apache.kafka.common.serialization.StringSerializer 7 | value-serializer: org.apache.kafka.common.serialization.StringSerializer -------------------------------------------------------------------------------- /Chapter5/5.2 metric-log-pipeline-streams/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.2 metric-log-pipeline-streams/.DS_Store -------------------------------------------------------------------------------- /Chapter5/5.2 metric-log-pipeline-streams/metric-kafka-streams/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.2 metric-log-pipeline-streams/metric-kafka-streams/.DS_Store -------------------------------------------------------------------------------- /Chapter5/5.2 metric-log-pipeline-streams/metric-kafka-streams/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'org.example' 6 | version '1.0-SNAPSHOT' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-streams:2.5.0' 17 | implementation 'com.google.code.gson:gson:2.8.0' 18 | } 19 | -------------------------------------------------------------------------------- /Chapter5/5.2 metric-log-pipeline-streams/metric-kafka-streams/metricbeat/metricbeat.yml: -------------------------------------------------------------------------------- 1 | metricbeat.modules: 2 | - module: system 3 | metricsets: 4 | - cpu 5 | - memory 6 | enabled: true 7 | period: 10s 8 | processes: ['.*'] 9 | 10 | output.kafka: 11 | hosts: ["my-kafka:9092"] 12 | topic: 'test' 13 | -------------------------------------------------------------------------------- /Chapter5/5.2 metric-log-pipeline-streams/metric-kafka-streams/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'metric-kafka-streams' 2 | 3 | -------------------------------------------------------------------------------- /Chapter5/5.2 metric-log-pipeline-streams/metric-kafka-streams/src/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.2 metric-log-pipeline-streams/metric-kafka-streams/src/.DS_Store -------------------------------------------------------------------------------- /Chapter5/5.2 metric-log-pipeline-streams/metric-kafka-streams/src/main/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter5/5.2 metric-log-pipeline-streams/metric-kafka-streams/src/main/.DS_Store -------------------------------------------------------------------------------- /Chapter5/5.2 metric-log-pipeline-streams/metric-kafka-streams/src/main/java/com/pipeline/MetricJsonUtils.java: -------------------------------------------------------------------------------- 1 | package com.pipeline; 2 | 3 | import com.google.gson.JsonObject; 4 | import com.google.gson.JsonParser; 5 | 6 | public class MetricJsonUtils { 7 | 8 | public static double getTotalCpuPercent(String value) { 9 | return new JsonParser().parse(value).getAsJsonObject().get("system").getAsJsonObject().get("cpu") 10 | .getAsJsonObject().get("total").getAsJsonObject().get("norm").getAsJsonObject().get("pct").getAsDouble(); 11 | } 12 | 13 | public static String getMetricName(String value) { 14 | return new JsonParser().parse(value).getAsJsonObject().get("metricset").getAsJsonObject().get("name") 15 | .getAsString(); 16 | } 17 | 18 | public static String getHostTimestamp(String value) { 19 | JsonObject objectValue = new JsonParser().parse(value).getAsJsonObject(); 20 | JsonObject result = objectValue.getAsJsonObject("host"); 21 | result.add("timestamp", objectValue.get("@timestamp")); 22 | return result.toString(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Chapter5/5.2 metric-log-pipeline-streams/metric-kafka-streams/src/main/java/com/pipeline/MetricStreams.java: -------------------------------------------------------------------------------- 1 | package com.pipeline; 2 | 3 | import org.apache.kafka.streams.StreamsConfig; 4 | import org.apache.kafka.common.serialization.Serdes; 5 | import org.apache.kafka.streams.KafkaStreams; 6 | import org.apache.kafka.streams.StreamsBuilder; 7 | import org.apache.kafka.streams.kstream.*; 8 | import java.util.Properties; 9 | 10 | public class MetricStreams { 11 | 12 | private static KafkaStreams streams; 13 | 14 | public static void main(final String[] args) { 15 | 16 | Runtime.getRuntime().addShutdownHook(new ShutdownThread()); 17 | 18 | Properties props = new Properties(); 19 | props.put(StreamsConfig.APPLICATION_ID_CONFIG, "metric-streams-application"); 20 | props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "my-kafka:9092"); 21 | props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 22 | props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 23 | 24 | StreamsBuilder builder = new StreamsBuilder(); 25 | KStream metrics = builder.stream("metric.all"); 26 | KStream[] metricBranch = metrics.branch( 27 | (key, value) -> MetricJsonUtils.getMetricName(value).equals("cpu"), 28 | (key, value) -> MetricJsonUtils.getMetricName(value).equals("memory") 29 | ); 30 | 31 | metricBranch[0].to("metric.cpu"); 32 | metricBranch[1].to("metric.memory"); 33 | 34 | KStream filteredCpuMetric = metricBranch[0] 35 | .filter((key, value) -> MetricJsonUtils.getTotalCpuPercent(value) > 0.5); 36 | 37 | filteredCpuMetric.mapValues(value -> MetricJsonUtils.getHostTimestamp(value)).to("metric.cpu.alert"); 38 | 39 | streams = new KafkaStreams(builder.build(), props); 40 | streams.start(); 41 | } 42 | 43 | static class ShutdownThread extends Thread { 44 | public void run() { 45 | streams.close(); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /Chapter6/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter6/.DS_Store -------------------------------------------------------------------------------- /Chapter6/6.1 confluent-kafka/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bjpublic/apache-kafka-with-java/1bb8eee4b158eaaf12a6541004b302540e81d45a/Chapter6/6.1 confluent-kafka/.DS_Store -------------------------------------------------------------------------------- /Chapter6/6.1 confluent-kafka/confluent-kafka-consumer/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } 19 | -------------------------------------------------------------------------------- /Chapter6/6.1 confluent-kafka/confluent-kafka-consumer/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'confluent-kafka-consumer' 2 | 3 | -------------------------------------------------------------------------------- /Chapter6/6.1 confluent-kafka/confluent-kafka-producer/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | } 4 | 5 | group 'com.example' 6 | version '1.0' 7 | 8 | sourceCompatibility = 1.8 9 | targetCompatibility = 1.8 10 | 11 | repositories { 12 | mavenCentral() 13 | } 14 | 15 | dependencies { 16 | implementation 'org.apache.kafka:kafka-clients:2.5.0' 17 | implementation 'org.slf4j:slf4j-simple:1.7.30' 18 | } -------------------------------------------------------------------------------- /Chapter6/6.1 confluent-kafka/confluent-kafka-producer/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'confluent-kafka-producer' 2 | 3 | --------------------------------------------------------------------------------