├── Code Files ├── .gitattributes ├── .gitignore ├── Commands-used-in-section-1.txt ├── LICENSE ├── advertclicks │ ├── .gitignore │ ├── .mvn │ │ └── wrapper │ │ │ ├── MavenWrapperDownloader.java │ │ │ ├── maven-wrapper.jar │ │ │ └── maven-wrapper.properties │ ├── mvnw │ ├── mvnw.cmd │ ├── pom.xml │ ├── practice-resources │ │ ├── kafka-scripts.sh │ │ └── samples.txt │ └── src │ │ └── main │ │ └── resources │ │ └── application.yaml ├── avroposfanout │ ├── .gitignore │ ├── .mvn │ │ └── wrapper │ │ │ ├── MavenWrapperDownloader.java │ │ │ ├── maven-wrapper.jar │ │ │ └── maven-wrapper.properties │ ├── mvnw │ ├── mvnw.cmd │ ├── pom.xml │ ├── practice-resources │ │ └── kafka-scripts.sh │ └── src │ │ └── main │ │ ├── avro │ │ ├── DeliveryAddress.avsc │ │ ├── LineItem.avsc │ │ └── PosInvoice.avsc │ │ └── resources │ │ └── application.yaml ├── avroposgen │ ├── .gitignore │ ├── .mvn │ │ └── wrapper │ │ │ ├── MavenWrapperDownloader.java │ │ │ ├── maven-wrapper.jar │ │ │ └── maven-wrapper.properties │ ├── mvnw │ ├── mvnw.cmd │ ├── pom.xml │ ├── practice-resources │ │ └── kafka-scripts.sh │ └── src │ │ └── main │ │ ├── avro │ │ ├── DeliveryAddress.avsc │ │ ├── LineItem.avsc │ │ └── PosInvoice.avsc │ │ └── resources │ │ └── application.yaml └── exactlyoncefanout │ ├── .gitignore │ ├── .mvn │ └── wrapper │ │ ├── MavenWrapperDownloader.java │ │ ├── maven-wrapper.jar │ │ └── maven-wrapper.properties │ ├── mvnw │ ├── mvnw.cmd │ ├── pom.xml │ ├── practice-resources │ └── kafka-scripts.sh │ └── src │ └── main │ ├── avro │ ├── DeliveryAddress.avsc │ ├── HadoopRecord.avsc │ ├── LineItem.avsc │ ├── Notification.avsc │ └── PosInvoice.avsc │ └── resources │ └── application.yaml ├── LICENSE ├── README.md ├── advertclicks ├── mvnw ├── mvnw.cmd ├── pom.xml ├── practice-resources │ ├── kafka-scripts.sh │ └── samples.txt └── src │ ├── main │ ├── java │ │ └── guru │ │ │ └── learningjournal │ │ │ └── examples │ │ │ └── kafka │ │ │ └── advertclicks │ │ │ ├── AdvertClicksApplication.java │ │ │ ├── Services │ │ │ └── ClickListenerService.java │ │ │ ├── bindings │ │ │ └── ClicksListenerBinding.java │ │ │ └── models │ │ │ ├── AdClick.java │ │ │ └── AdInventories.java │ └── resources │ │ └── application.yaml │ └── test │ └── java │ └── guru │ └── learningjournal │ └── examples │ └── kafka │ └── advertclicks │ └── AdvertClicksApplicationTests.java ├── avroposfanout ├── mvnw ├── mvnw.cmd ├── pom.xml ├── practice-resources │ └── kafka-scripts.sh └── src │ ├── main │ ├── avro │ │ ├── DeliveryAddress.avsc │ │ ├── LineItem.avsc │ │ └── PosInvoice.avsc │ ├── java │ │ └── guru │ │ │ └── learningjournal │ │ │ └── examples │ │ │ └── kafka │ │ │ └── avroposfanout │ │ │ ├── AvroPosFanoutApplication.java │ │ │ ├── bindings │ │ │ └── PosListenerBinding.java │ │ │ ├── model │ │ │ ├── HadoopRecord.java │ │ │ └── Notification.java │ │ │ └── services │ │ │ ├── HadoopRecordProcessorService.java │ │ │ ├── NotificationProcessorService.java │ │ │ └── RecordBuilder.java │ └── resources │ │ └── application.yaml │ └── test │ └── java │ └── guru │ └── learningjournal │ └── examples │ └── kafka │ └── avroposfanout │ └── AvroPosFanoutApplicationTests.java ├── avroposgen ├── mvnw ├── mvnw.cmd ├── pom.xml ├── practice-resources │ └── kafka-scripts.sh └── src │ ├── main │ ├── avro │ │ ├── DeliveryAddress.avsc │ │ ├── LineItem.avsc │ │ └── PosInvoice.avsc │ ├── java │ │ └── guru │ │ │ └── learningjournal │ │ │ └── examples │ │ │ └── kafka │ │ │ └── avroposgen │ │ │ ├── AvroPosGeneratorApplication.java │ │ │ └── services │ │ │ ├── KafkaProducerService.java │ │ │ └── datagenerator │ │ │ ├── AddressGenerator.java │ │ │ ├── InvoiceGenerator.java │ │ │ └── ProductGenerator.java │ └── resources │ │ ├── application.yaml │ │ └── data │ │ ├── Invoice.json │ │ ├── address.json │ │ └── products.json │ └── test │ └── java │ └── guru │ └── learningjournal │ └── examples │ └── kafka │ └── avroposgen │ └── AvroPosGeneratorApplicationTests.java ├── exactlyoncefanout ├── mvnw ├── mvnw.cmd ├── pom.xml ├── practice-resources │ └── kafka-scripts.sh └── src │ ├── main │ ├── avro │ │ ├── DeliveryAddress.avsc │ │ ├── HadoopRecord.avsc │ │ ├── LineItem.avsc │ │ ├── Notification.avsc │ │ └── PosInvoice.avsc │ ├── java │ │ └── guru │ │ │ └── learningjournal │ │ │ └── examples │ │ │ └── kafka │ │ │ └── exactlyoncefanout │ │ │ ├── ExactlyOnceFanoutApplication.java │ │ │ ├── bindings │ │ │ └── PosListenerBinding.java │ │ │ └── services │ │ │ ├── PosListenerService.java │ │ │ └── RecordBuilder.java │ └── resources │ │ └── application.yaml │ └── test │ └── java │ └── guru │ └── learningjournal │ └── examples │ └── kafka │ └── exactlyoncefanout │ └── ExactlyOnceFanoutApplicationTests.java ├── hello-streams ├── build.gradle ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── practice-resources │ ├── data-samples.txt │ └── kafka-scripts.sh ├── settings.gradle └── src │ ├── main │ ├── java │ │ └── guru │ │ │ └── learningjournal │ │ │ └── examples │ │ │ └── kafka │ │ │ └── hellostreams │ │ │ ├── HelloStreamsApplication.java │ │ │ ├── bindings │ │ │ └── KafkaListenerBinding.java │ │ │ └── services │ │ │ └── KafkaListenerService.java │ └── resources │ │ └── application.yaml │ └── test │ └── java │ └── guru │ └── learningjournal │ └── examples │ └── kafka │ └── hellostreams │ └── HelloStreamsApplicationTests.java ├── jsonposgen ├── build.gradle ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── practice-resources │ └── kafka-scripts.sh ├── settings.gradle └── src │ ├── main │ ├── java │ │ └── guru │ │ │ └── learningjournal │ │ │ └── examples │ │ │ └── kafka │ │ │ └── jsonposgen │ │ │ ├── JsonPosGeneratorApplication.java │ │ │ ├── model │ │ │ ├── DeliveryAddress.java │ │ │ ├── LineItem.java │ │ │ └── PosInvoice.java │ │ │ └── services │ │ │ ├── KafkaProducerService.java │ │ │ └── datagenerator │ │ │ ├── AddressGenerator.java │ │ │ ├── InvoiceGenerator.java │ │ │ └── ProductGenerator.java │ └── resources │ │ ├── application.yaml │ │ └── data │ │ ├── Invoice.json │ │ ├── address.json │ │ └── products.json │ └── test │ └── java │ └── guru │ └── learningjournal │ └── examples │ └── kafka │ └── jsonposgen │ └── JsonPosGeneratorApplicationTests.java ├── kafkaproducer ├── build.gradle ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── practice-resources │ └── kafka-scripts.sh └── scratches │ └── scratch.rest ├── kstreamaggregate ├── mvnw ├── mvnw.cmd ├── pom.xml ├── practice-resources │ ├── kafka-scripts.sh │ └── samples.txt └── src │ ├── main │ ├── avro │ │ ├── DepartmentAggregate.avsc │ │ └── Employee.avsc │ ├── java │ │ └── guru │ │ │ └── learningjournal │ │ │ └── examples │ │ │ └── kafka │ │ │ └── kstreamaggregate │ │ │ ├── KStreamAggregateApplication.java │ │ │ ├── bindings │ │ │ └── EmployeeListenerBinding.java │ │ │ └── services │ │ │ ├── EmployeeStreamListener.java │ │ │ └── RecordBuilder.java │ └── resources │ │ └── application.yaml │ └── test │ └── java │ └── guru │ └── learningjournal │ └── examples │ └── kafka │ └── kstreamaggregate │ └── KStreamAggregateApplicationTests.java ├── ktableaggregate ├── mvnw ├── mvnw.cmd ├── pom.xml ├── practice-resources │ ├── kafka-scripts.sh │ └── samples.txt └── src │ ├── main │ ├── avro │ │ ├── DepartmentAggregate.avsc │ │ └── Employee.avsc │ ├── java │ │ └── guru │ │ │ └── learningjournal │ │ │ └── examples │ │ │ └── kafka │ │ │ └── ktableaggregate │ │ │ ├── KTableAggregateApplication.java │ │ │ ├── bindings │ │ │ └── EmployeeListenerBinding.java │ │ │ └── services │ │ │ ├── EmployeeStreamListener.java │ │ │ └── RecordBuilder.java │ └── resources │ │ └── application.yaml │ └── test │ └── java │ └── guru │ └── learningjournal │ └── examples │ └── kafka │ └── ktableaggregate │ └── KTableAggregateApplicationTests.java ├── ktabledemo ├── build.gradle ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── practice-resources │ ├── kafka-scripts.sh │ └── sample.txt ├── settings.gradle └── src │ ├── main │ ├── java │ │ └── guru │ │ │ └── learningjournal │ │ │ └── examples │ │ │ └── kafka │ │ │ └── ktabledemo │ │ │ ├── KTableDemoApplication.java │ │ │ ├── bindings │ │ │ └── StockListenerBinding.java │ │ │ └── services │ │ │ └── StockTickListenerService.java │ └── resources │ │ └── application.yaml │ └── test │ └── java │ └── guru │ └── learningjournal │ └── examples │ └── kafka │ └── ktabledemo │ └── KTableDemoApplicationTests.java ├── lastlogin ├── mvnw ├── mvnw.cmd ├── pom.xml ├── practice-resources │ ├── kafka-scripts.sh │ └── samples.txt └── src │ ├── main │ ├── java │ │ └── guru │ │ │ └── learningjournal │ │ │ └── examples │ │ │ └── kafka │ │ │ └── lastlogin │ │ │ ├── LastLoginApplication.java │ │ │ ├── bindings │ │ │ └── UserListenerBinding.java │ │ │ ├── model │ │ │ ├── UserDetails.java │ │ │ └── UserLogin.java │ │ │ └── services │ │ │ └── LoginListenerService.java │ └── resources │ │ └── application.yaml │ └── test │ └── java │ └── guru │ └── learningjournal │ └── examples │ └── kafka │ └── lastlogin │ └── LastLoginApplicationTests.java ├── otpvalidation ├── mvnw ├── mvnw.cmd ├── pom.xml ├── practice-resources │ ├── kafka-scripts.sh │ └── samples.txt └── src │ ├── main │ ├── java │ │ └── guru │ │ │ └── learningjournal │ │ │ └── examples │ │ │ └── kafka │ │ │ └── otpvalidation │ │ │ ├── OtpValidationApplication.java │ │ │ ├── bindings │ │ │ └── OTPListenerBinding.java │ │ │ ├── configs │ │ │ ├── PaymentConfirmationTimeExtractor.java │ │ │ └── PaymentRequestTimeExtractor.java │ │ │ ├── model │ │ │ ├── PaymentConfirmation.java │ │ │ ├── PaymentRequest.java │ │ │ └── TransactionStatus.java │ │ │ └── services │ │ │ ├── OTPValidationService.java │ │ │ └── RecordBuilder.java │ └── resources │ │ └── application.yaml │ └── test │ └── java │ └── guru │ └── learningjournal │ └── examples │ └── kafka │ └── otpvalidation │ └── OtpValidationApplicationTests.java ├── rewards ├── mvnw ├── mvnw.cmd ├── pom.xml ├── practice-resources │ └── kafka-scripts.sh └── src │ ├── main │ ├── avro │ │ ├── DeliveryAddress.avsc │ │ ├── LineItem.avsc │ │ ├── Notification.avsc │ │ └── PosInvoice.avsc │ ├── java │ │ └── guru │ │ │ └── learningjournal │ │ │ └── examples │ │ │ └── kafka │ │ │ └── rewards │ │ │ ├── RewardsApplication.java │ │ │ ├── bindings │ │ │ └── PosListenerBinding.java │ │ │ └── services │ │ │ ├── LoyaltyService.java │ │ │ └── RecordBuilder.java │ └── resources │ │ └── application.yaml │ └── test │ └── java │ └── guru │ └── learningjournal │ └── examples │ └── kafka │ └── rewards │ └── RewardsApplicationTests.java ├── sessionwindow ├── mvnw ├── mvnw.cmd ├── pom.xml ├── practice-resources │ ├── kafka-scripts.sh │ └── samples.txt └── src │ ├── main │ ├── java │ │ └── guru │ │ │ └── learningjournal │ │ │ └── examples │ │ │ └── kafka │ │ │ └── sessionwindow │ │ │ ├── SessionWindowApplication.java │ │ │ ├── bindings │ │ │ └── ClickListenerBinding.java │ │ │ ├── configs │ │ │ └── ClickTimeExtractor.java │ │ │ ├── models │ │ │ └── UserClick.java │ │ │ └── services │ │ │ └── ClickListerService.java │ └── resources │ │ └── application.yaml │ └── test │ └── java │ └── guru │ └── learningjournal │ └── examples │ └── kafka │ └── sessionwindow │ └── SessionWindowApplicationTests.java ├── simpletest ├── mvnw ├── mvnw.cmd ├── pom.xml ├── practice-resources │ ├── kafka-scripts.sh │ └── samples.txt └── src │ ├── main │ ├── java │ │ └── guru │ │ │ └── learningjournal │ │ │ └── examples │ │ │ └── kafka │ │ │ └── simpletest │ │ │ ├── SimpleTestApplication.java │ │ │ ├── bindings │ │ │ └── ListenerBinding.java │ │ │ └── services │ │ │ └── ListenerService.java │ └── resources │ │ └── application.yaml │ └── test │ ├── java │ └── guru │ │ └── learningjournal │ │ └── examples │ │ └── kafka │ │ └── simpletest │ │ └── SimpleTestApplicationTests.java │ └── resources │ └── application.yaml ├── streamingaggregates ├── build.gradle ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── practice-resources │ ├── kafka-scripts.sh │ └── sample.txt ├── settings.gradle └── src │ ├── main │ ├── java │ │ └── guru │ │ │ └── learningjournal │ │ │ └── examples │ │ │ └── kafka │ │ │ └── streamingaggregates │ │ │ ├── StreamingAggregatesApplication.java │ │ │ ├── bindings │ │ │ └── WordListenerBinding.java │ │ │ └── services │ │ │ └── WordListenerService.java │ └── resources │ │ └── application.yaml │ └── test │ └── java │ └── guru │ └── learningjournal │ └── examples │ └── kafka │ └── streamingaggregates │ └── StreamingAggregatesApplicationTests.java ├── streamingtest ├── mvnw ├── mvnw.cmd ├── pom.xml ├── practice-resources │ ├── kafka-scripts.sh │ └── samples.txt └── src │ ├── main │ ├── java │ │ └── guru │ │ │ └── learningjournal │ │ │ └── examples │ │ │ └── kafka │ │ │ └── streamingtest │ │ │ ├── StreamingTestApplication.java │ │ │ └── configs │ │ │ └── ListenerService.java │ └── resources │ │ └── application.yaml │ └── test │ ├── java │ └── guru │ │ └── learningjournal │ │ └── examples │ │ └── kafka │ │ └── streamingtest │ │ └── StreamingTestApplicationTests.java │ └── resources │ └── application.yaml ├── top3spots ├── mvnw ├── mvnw.cmd ├── pom.xml ├── practice-resources │ ├── kafka-scripts.sh │ └── samples.txt └── src │ ├── main │ ├── java │ │ └── guru │ │ │ └── learningjournal │ │ │ └── examples │ │ │ └── kafka │ │ │ └── top3spots │ │ │ ├── Services │ │ │ └── ClickListenerService.java │ │ │ ├── Top3SpotsApplication.java │ │ │ ├── bindings │ │ │ └── ClicksListenerBinding.java │ │ │ └── models │ │ │ ├── AdClick.java │ │ │ ├── AdInventories.java │ │ │ ├── ClicksByNewsType.java │ │ │ └── Top3NewsTypes.java │ └── resources │ │ └── application.yaml │ └── test │ └── java │ └── guru │ └── learningjournal │ └── examples │ └── kafka │ └── top3spots │ └── Top3SpotsApplicationTests.java └── windowcount ├── mvnw ├── mvnw.cmd ├── pom.xml ├── practice-resources ├── kafka-scripts.sh └── samples.txt └── src └── main └── java └── guru └── learningjournal └── examples └── kafka └── windowcount ├── WindowCountApplication.java ├── bindings └── InvoiceListenerBinding.java ├── configs └── InvoiceTimeExtractor.java ├── model └── SimpleInvoice.java └── services └── InvoiceListenerService.java /Code Files/.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /Code Files/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | */state-store/ 3 | 4 | -------------------------------------------------------------------------------- /Code Files/Commands-used-in-section-1.txt: -------------------------------------------------------------------------------- 1 | Link for Steps to Install WSL 2 | ----------------------------------- 3 | https://docs.microsoft.com/en-us/windows/wsl/install-win10 4 | 5 | Check your JDK installation on Ubuntu 6 | ---------------------------------------- 7 | javac -version 8 | 9 | Installing JDK on Ubuntu 10 | ----------------------------- 11 | sudo apt-get update 12 | sudo apt install default-jdk 13 | 14 | Configure your Windows terminal 15 | --------------------------------- 16 | "startingDirectory": "\\\\wsl$\\Ubuntu-20.04\\home\\prashant" 17 | 18 | Copy your downloaded file to your Ubuntu home directory 19 | ----------------------------------------------------------- 20 | cp /mnt/e/demo/confluent-6.0.0.tar.gz . 21 | 22 | Copy your downloaded file to your Mac home directory 23 | ----------------------------------------------------------- 24 | cp ~/Downloads/confluent-6.0.0.tar ~ 25 | 26 | Untar the downloaded file 27 | ---------------------------- 28 | tar -zxvf confluent-6.0.0.tar.gz 29 | 30 | Setting the environment variables on windows. 31 | ------------------------------------------------- 32 | export CONFLUENT_HOME=/home/prashant/confluent-6.0.0 33 | export PATH=$PATH:$CONFLUENT_HOME/bin 34 | 35 | Setting the environment variable on Mac. 36 | ------------------------------------------ 37 | export CONFLUENT_HOME=/Users/prashant/confluent-6.0.0 38 | export PATH=$PATH:$CONFLUENT_HOME/bin 39 | 40 | Running Confluent services 41 | ---------------------------- 42 | confluent local services start 43 | confluent local services stop 44 | confluent local destroy 45 | 46 | Kafka Commands 47 | ----------------- 48 | kafka-console-producer --topic test-topic --broker-list localhost:9092 49 | kafka-console-consumer --topic test-topic --bootstrap-server localhost:9092 --from-beginning 50 | 51 | Sample Data 52 | -------------- 53 | {"name": "Kristie Cole", "age": 34,"gender": "female"} 54 | {"name": "Marsh Mccall", "age": 28, "gender": "male"} 55 | 56 | 57 | -------------------------------------------------------------------------------- /Code Files/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Prashant Kumar Pandey 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Code Files/advertclicks/.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | target/ 3 | !.mvn/wrapper/maven-wrapper.jar 4 | !**/src/main/**/target/ 5 | !**/src/test/**/target/ 6 | 7 | ### STS ### 8 | .apt_generated 9 | .classpath 10 | .factorypath 11 | .project 12 | .settings 13 | .springBeans 14 | .sts4-cache 15 | 16 | ### IntelliJ IDEA ### 17 | .idea 18 | *.iws 19 | *.iml 20 | *.ipr 21 | 22 | ### NetBeans ### 23 | /nbproject/private/ 24 | /nbbuild/ 25 | /dist/ 26 | /nbdist/ 27 | /.nb-gradle/ 28 | build/ 29 | !**/src/main/**/build/ 30 | !**/src/test/**/build/ 31 | 32 | ### VS Code ### 33 | .vscode/ 34 | -------------------------------------------------------------------------------- /Code Files/advertclicks/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Kafka-Streams-with-Spring-Cloud-Stream/aa6319b0454246240d0610994d06900b3ef73c9d/Code Files/advertclicks/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /Code Files/advertclicks/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip 2 | wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar 3 | -------------------------------------------------------------------------------- /Code Files/advertclicks/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework.boot 7 | spring-boot-starter-parent 8 | 2.4.2 9 | 10 | 11 | guru.learningjournal.examples.kafka 12 | advertclicks 13 | 0.0.1-SNAPSHOT 14 | Advert Clicks 15 | Advert Clicks Demo by Learning Journal 16 | 17 | 11 18 | Hoxton.SR9 19 | 20 | 21 | 22 | org.apache.kafka 23 | kafka-streams 24 | 25 | 26 | org.springframework.cloud 27 | spring-cloud-stream 28 | 29 | 30 | org.springframework.cloud 31 | spring-cloud-stream-binder-kafka-streams 32 | 33 | 34 | 35 | org.projectlombok 36 | lombok 37 | true 38 | 39 | 40 | org.springframework.boot 41 | spring-boot-starter-test 42 | test 43 | 44 | 45 | org.springframework.cloud 46 | spring-cloud-stream 47 | test 48 | test-binder 49 | test-jar 50 | 51 | 52 | 53 | 54 | 55 | org.springframework.cloud 56 | spring-cloud-dependencies 57 | ${spring-cloud.version} 58 | pom 59 | import 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | org.springframework.boot 68 | spring-boot-maven-plugin 69 | 70 | 71 | 72 | org.projectlombok 73 | lombok 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | spring-milestones 83 | Spring Milestones 84 | https://repo.spring.io/milestone 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /Code Files/advertclicks/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | confluent local services start 6 | 7 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic active-inventories 8 | 9 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic ad-clicks 10 | 11 | kafka-console-producer --broker-list localhost:9092 --topic active-inventories \ 12 | --property parse.key=true --property key.separator=":" 13 | 14 | kafka-console-producer --broker-list localhost:9092 --topic ad-clicks \ 15 | --property parse.key=true --property key.separator=":" 16 | 17 | confluent local destroy -------------------------------------------------------------------------------- /Code Files/advertclicks/practice-resources/samples.txt: -------------------------------------------------------------------------------- 1 | 1001:{"InventoryID": "1001", "NewsType": "Sports"} 2 | 1002:{"InventoryID": "1002", "NewsType": "Politics"} 3 | 1003:{"InventoryID": "1003", "NewsType": "LocalNews"} 4 | 1004:{"InventoryID": "1004", "NewsType": "WorldNews"} 5 | 1005:{"InventoryID": "1005", "NewsType": "Health"} 6 | 1006:{"InventoryID": "1006", "NewsType": "Lifestyle"} 7 | 1007:{"InventoryID": "1007", "NewsType": "Literature"} 8 | 1008:{"InventoryID": "1008", "NewsType": "Education"} 9 | 1009:{"InventoryID": "1009", "NewsType": "Social"} 10 | 1010:{"InventoryID": "1010", "NewsType": "Business"} 11 | 12 | 1001:{"InventoryID": "1001"} 13 | 1002:{"InventoryID": "1002"} 14 | 1003:{"InventoryID": "1003"} 15 | 1004:{"InventoryID": "1004"} 16 | 1004:{"InventoryID": "1004"} 17 | 1005:{"InventoryID": "1005"} 18 | 1006:{"InventoryID": "1006"} 19 | 1007:{"InventoryID": "1007"} 20 | 1008:{"InventoryID": "1008"} 21 | 1009:{"InventoryID": "1009"} 22 | 1010:{"InventoryID": "1010"} 23 | -------------------------------------------------------------------------------- /Code Files/advertclicks/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | inventories-channel: 6 | destination: active-inventories 7 | clicks-channel: 8 | destination: ad-clicks 9 | kafka: 10 | streams: 11 | binder: 12 | brokers: localhost:9092 13 | configuration: 14 | commit.interval.ms: 10000 15 | state.dir: state-store 16 | default: 17 | key: 18 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 19 | value: 20 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 21 | 22 | -------------------------------------------------------------------------------- /Code Files/avroposfanout/.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | target/ 3 | !.mvn/wrapper/maven-wrapper.jar 4 | !**/src/main/**/target/ 5 | !**/src/test/**/target/ 6 | 7 | ### STS ### 8 | .apt_generated 9 | .classpath 10 | .factorypath 11 | .project 12 | .settings 13 | .springBeans 14 | .sts4-cache 15 | 16 | ### IntelliJ IDEA ### 17 | .idea 18 | *.iws 19 | *.iml 20 | *.ipr 21 | 22 | ### NetBeans ### 23 | /nbproject/private/ 24 | /nbbuild/ 25 | /dist/ 26 | /nbdist/ 27 | /.nb-gradle/ 28 | build/ 29 | !**/src/main/**/build/ 30 | !**/src/test/**/build/ 31 | 32 | ### VS Code ### 33 | .vscode/ 34 | -------------------------------------------------------------------------------- /Code Files/avroposfanout/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Kafka-Streams-with-Spring-Cloud-Stream/aa6319b0454246240d0610994d06900b3ef73c9d/Code Files/avroposfanout/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /Code Files/avroposfanout/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip 2 | wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar 3 | -------------------------------------------------------------------------------- /Code Files/avroposfanout/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | netsh interface portproxy add v4tov4 listenport=8081 listenaddress=0.0.0.0 connectport=8081 connectaddress= 6 | 7 | confluent local services start 8 | 9 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic loyalty-topic 10 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic hadoop-sink-topic 11 | 12 | 13 | http://localhost:9021/ 14 | 15 | confluent local services stop 16 | confluent local destroy -------------------------------------------------------------------------------- /Code Files/avroposfanout/src/main/avro/DeliveryAddress.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "DeliveryAddress", 5 | "fields": [ 6 | {"name": "AddressLine","type": ["null","string"]}, 7 | {"name": "City","type": ["null","string"]}, 8 | {"name": "State","type": ["null","string"]}, 9 | {"name": "PinCode","type": ["null","string"]}, 10 | {"name": "ContactNumber","type": ["null","string"]} 11 | ] 12 | } -------------------------------------------------------------------------------- /Code Files/avroposfanout/src/main/avro/LineItem.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "LineItem", 5 | "fields": [ 6 | {"name": "ItemCode","type": ["null","string"]}, 7 | {"name": "ItemDescription","type": ["null","string"]}, 8 | {"name": "ItemPrice","type": ["null","double"]}, 9 | {"name": "ItemQty","type": ["null","int"]}, 10 | {"name": "TotalValue","type": ["null","double"]} 11 | ] 12 | } -------------------------------------------------------------------------------- /Code Files/avroposfanout/src/main/avro/PosInvoice.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "PosInvoice", 5 | "fields": [ 6 | {"name": "InvoiceNumber","type": ["null","string"]}, 7 | {"name": "CreatedTime","type": ["null","long"]}, 8 | {"name": "CustomerCardNo","type": ["null","string"]}, 9 | {"name": "TotalAmount","type": ["null","double"]}, 10 | {"name": "NumberOfItems","type": ["null","int"]}, 11 | {"name": "PaymentMethod","type": ["null","string"]}, 12 | {"name": "TaxableAmount","type": ["null","double"]}, 13 | {"name": "CGST","type": ["null","double"]}, 14 | {"name": "SGST","type": ["null","double"]}, 15 | {"name": "CESS","type": ["null","double"]}, 16 | {"name": "StoreID","type": ["null","string"]}, 17 | {"name": "PosID","type": ["null","string"]}, 18 | {"name": "CashierID","type": ["null","string"]}, 19 | {"name": "CustomerType","type": ["null","string"]}, 20 | {"name": "DeliveryType","type": ["null","string"]}, 21 | {"name": "DeliveryAddress","type": ["null","DeliveryAddress"]}, 22 | {"name": "InvoiceLineItems","type": {"type": "array", "items": "LineItem"}} 23 | ] 24 | } -------------------------------------------------------------------------------- /Code Files/avroposfanout/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | notification-input-channel: 6 | destination: avro-pos-topic 7 | notification-output-channel: 8 | destination: loyalty-topic 9 | hadoop-input-channel: 10 | destination: avro-pos-topic 11 | hadoop-output-channel: 12 | destination: hadoop-sink-topic 13 | kafka: 14 | streams: 15 | binder: 16 | brokers: localhost:9092 17 | configuration: 18 | schema.registry.url: http://localhost:8081 19 | bindings: 20 | notification-input-channel: 21 | consumer: 22 | valueSerde: io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde 23 | notification-output-channel: 24 | producer: 25 | valueSerde: io.confluent.kafka.streams.serdes.json.KafkaJsonSchemaSerde 26 | hadoop-input-channel: 27 | consumer: 28 | valueSerde: io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde 29 | hadoop-output-channel: 30 | producer: 31 | valueSerde: io.confluent.kafka.streams.serdes.json.KafkaJsonSchemaSerde 32 | -------------------------------------------------------------------------------- /Code Files/avroposgen/.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | target/ 3 | !.mvn/wrapper/maven-wrapper.jar 4 | !**/src/main/**/target/ 5 | !**/src/test/**/target/ 6 | 7 | ### STS ### 8 | .apt_generated 9 | .classpath 10 | .factorypath 11 | .project 12 | .settings 13 | .springBeans 14 | .sts4-cache 15 | 16 | ### IntelliJ IDEA ### 17 | .idea 18 | *.iws 19 | *.iml 20 | *.ipr 21 | 22 | ### NetBeans ### 23 | /nbproject/private/ 24 | /nbbuild/ 25 | /dist/ 26 | /nbdist/ 27 | /.nb-gradle/ 28 | build/ 29 | !**/src/main/**/build/ 30 | !**/src/test/**/build/ 31 | 32 | ### VS Code ### 33 | .vscode/ 34 | -------------------------------------------------------------------------------- /Code Files/avroposgen/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Kafka-Streams-with-Spring-Cloud-Stream/aa6319b0454246240d0610994d06900b3ef73c9d/Code Files/avroposgen/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /Code Files/avroposgen/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip 2 | wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar 3 | -------------------------------------------------------------------------------- /Code Files/avroposgen/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | netsh interface portproxy add v4tov4 listenport=8081 listenaddress=0.0.0.0 connectport=8081 connectaddress= 6 | 7 | confluent local services start 8 | 9 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic avro-pos-topic 10 | 11 | kafka-avro-console-consumer --bootstrap-server localhost:9092 --topic avro-pos-topic --from-beginning --property key.deserializer=org.apache.kafka.common.serialization.StringDeserializer --property print.key=true --property key.separator=":" 12 | 13 | confluent local services stop 14 | confluent local destroy -------------------------------------------------------------------------------- /Code Files/avroposgen/src/main/avro/DeliveryAddress.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "DeliveryAddress", 5 | "fields": [ 6 | {"name": "AddressLine","type": ["null","string"]}, 7 | {"name": "City","type": ["null","string"]}, 8 | {"name": "State","type": ["null","string"]}, 9 | {"name": "PinCode","type": ["null","string"]}, 10 | {"name": "ContactNumber","type": ["null","string"]} 11 | ] 12 | } -------------------------------------------------------------------------------- /Code Files/avroposgen/src/main/avro/LineItem.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "LineItem", 5 | "fields": [ 6 | {"name": "ItemCode","type": ["null","string"]}, 7 | {"name": "ItemDescription","type": ["null","string"]}, 8 | {"name": "ItemPrice","type": ["null","double"]}, 9 | {"name": "ItemQty","type": ["null","int"]}, 10 | {"name": "TotalValue","type": ["null","double"]} 11 | ] 12 | } -------------------------------------------------------------------------------- /Code Files/avroposgen/src/main/avro/PosInvoice.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "PosInvoice", 5 | "fields": [ 6 | {"name": "InvoiceNumber","type": ["null","string"]}, 7 | {"name": "CreatedTime","type": ["null","long"]}, 8 | {"name": "CustomerCardNo","type": ["null","string"]}, 9 | {"name": "TotalAmount","type": ["null","double"]}, 10 | {"name": "NumberOfItems","type": ["null","int"]}, 11 | {"name": "PaymentMethod","type": ["null","string"]}, 12 | {"name": "TaxableAmount","type": ["null","double"]}, 13 | {"name": "CGST","type": ["null","double"]}, 14 | {"name": "SGST","type": ["null","double"]}, 15 | {"name": "CESS","type": ["null","double"]}, 16 | {"name": "StoreID","type": ["null","string"]}, 17 | {"name": "PosID","type": ["null","string"]}, 18 | {"name": "CashierID","type": ["null","string"]}, 19 | {"name": "CustomerType","type": ["null","string"]}, 20 | {"name": "DeliveryType","type": ["null","string"]}, 21 | {"name": "DeliveryAddress","type": ["null","DeliveryAddress"]}, 22 | {"name": "InvoiceLineItems","type": {"type": "array", "items": "LineItem"}} 23 | ] 24 | } -------------------------------------------------------------------------------- /Code Files/avroposgen/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | kafka: 3 | producer: 4 | client-id: avro-pos-simulator 5 | bootstrap-servers: localhost:9092 6 | key-serializer: org.apache.kafka.common.serialization.StringSerializer 7 | value-serializer: io.confluent.kafka.serializers.KafkaAvroSerializer 8 | properties: 9 | schema.registry.url: http://localhost:8081 10 | 11 | application: 12 | configs: 13 | invoice.count: 60 14 | topic.name: avro-pos-topic 15 | 16 | 17 | -------------------------------------------------------------------------------- /Code Files/exactlyoncefanout/.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | target/ 3 | !.mvn/wrapper/maven-wrapper.jar 4 | !**/src/main/**/target/ 5 | !**/src/test/**/target/ 6 | 7 | ### STS ### 8 | .apt_generated 9 | .classpath 10 | .factorypath 11 | .project 12 | .settings 13 | .springBeans 14 | .sts4-cache 15 | 16 | ### IntelliJ IDEA ### 17 | .idea 18 | *.iws 19 | *.iml 20 | *.ipr 21 | 22 | ### NetBeans ### 23 | /nbproject/private/ 24 | /nbbuild/ 25 | /dist/ 26 | /nbdist/ 27 | /.nb-gradle/ 28 | build/ 29 | !**/src/main/**/build/ 30 | !**/src/test/**/build/ 31 | 32 | ### VS Code ### 33 | .vscode/ 34 | -------------------------------------------------------------------------------- /Code Files/exactlyoncefanout/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Kafka-Streams-with-Spring-Cloud-Stream/aa6319b0454246240d0610994d06900b3ef73c9d/Code Files/exactlyoncefanout/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /Code Files/exactlyoncefanout/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip 2 | wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar 3 | -------------------------------------------------------------------------------- /Code Files/exactlyoncefanout/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | netsh interface portproxy add v4tov4 listenport=8081 listenaddress=0.0.0.0 connectport=8081 connectaddress= 6 | 7 | confluent local services start 8 | 9 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic hadoop-sink-topic 10 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic loyalty-topic 11 | 12 | kafka-avro-console-consumer --bootstrap-server localhost:9092 --topic hadoop-sink-topic --from-beginning --property key.deserializer=org.apache.kafka.common.serialization.StringDeserializer --property print.key=true --property key.separator=":" 13 | kafka-avro-console-consumer --bootstrap-server localhost:9092 --topic loyalty-topic --from-beginning --property key.deserializer=org.apache.kafka.common.serialization.StringDeserializer --property print.key=true --property key.separator=":" 14 | 15 | confluent local services stop 16 | confluent local destroy -------------------------------------------------------------------------------- /Code Files/exactlyoncefanout/src/main/avro/DeliveryAddress.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "DeliveryAddress", 5 | "fields": [ 6 | {"name": "AddressLine","type": ["null","string"]}, 7 | {"name": "City","type": ["null","string"]}, 8 | {"name": "State","type": ["null","string"]}, 9 | {"name": "PinCode","type": ["null","string"]}, 10 | {"name": "ContactNumber","type": ["null","string"]} 11 | ] 12 | } -------------------------------------------------------------------------------- /Code Files/exactlyoncefanout/src/main/avro/HadoopRecord.avsc: -------------------------------------------------------------------------------- 1 | {"namespace": "guru.learningjournal.examples.kafka.model", 2 | "type": "record", 3 | "name": "HadoopRecord", 4 | "fields": [ 5 | {"name": "InvoiceNumber", "type": ["null", "string"]}, 6 | {"name": "CreatedTime", "type":["null", "long"]}, 7 | {"name": "StoreID", "type": ["null", "string"]}, 8 | {"name": "PosID", "type": ["null", "string"]}, 9 | {"name": "CustomerType", "type": ["null", "string"]}, 10 | {"name": "PaymentMethod", "type": ["null", "string"]}, 11 | {"name": "DeliveryType", "type": ["null", "string"]}, 12 | {"name": "City", "type": ["null", "string"]}, 13 | {"name": "State", "type": ["null", "string"]}, 14 | {"name": "PinCode", "type": ["null", "string"]}, 15 | {"name": "ItemCode", "type": ["null", "string"]}, 16 | {"name": "ItemDescription", "type": ["null", "string"]}, 17 | {"name": "ItemPrice", "type": ["null", "double"]}, 18 | {"name": "ItemQty", "type": ["null", "int"]}, 19 | {"name": "TotalValue", "type": ["null", "double"]} 20 | ] 21 | } -------------------------------------------------------------------------------- /Code Files/exactlyoncefanout/src/main/avro/LineItem.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "LineItem", 5 | "fields": [ 6 | {"name": "ItemCode","type": ["null","string"]}, 7 | {"name": "ItemDescription","type": ["null","string"]}, 8 | {"name": "ItemPrice","type": ["null","double"]}, 9 | {"name": "ItemQty","type": ["null","int"]}, 10 | {"name": "TotalValue","type": ["null","double"]} 11 | ] 12 | } -------------------------------------------------------------------------------- /Code Files/exactlyoncefanout/src/main/avro/Notification.avsc: -------------------------------------------------------------------------------- 1 | {"namespace": "guru.learningjournal.examples.kafka.model", 2 | "type": "record", 3 | "name": "Notification", 4 | "fields": [ 5 | {"name": "InvoiceNumber", "type": ["null", "string"]}, 6 | {"name": "CustomerCardNo", "type":["null", "string"]}, 7 | {"name": "TotalAmount", "type": ["null", "double"]}, 8 | {"name": "EarnedLoyaltyPoints", "type": ["null", "double"]} 9 | ] 10 | } -------------------------------------------------------------------------------- /Code Files/exactlyoncefanout/src/main/avro/PosInvoice.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "PosInvoice", 5 | "fields": [ 6 | {"name": "InvoiceNumber","type": ["null","string"]}, 7 | {"name": "CreatedTime","type": ["null","long"]}, 8 | {"name": "CustomerCardNo","type": ["null","string"]}, 9 | {"name": "TotalAmount","type": ["null","double"]}, 10 | {"name": "NumberOfItems","type": ["null","int"]}, 11 | {"name": "PaymentMethod","type": ["null","string"]}, 12 | {"name": "TaxableAmount","type": ["null","double"]}, 13 | {"name": "CGST","type": ["null","double"]}, 14 | {"name": "SGST","type": ["null","double"]}, 15 | {"name": "CESS","type": ["null","double"]}, 16 | {"name": "StoreID","type": ["null","string"]}, 17 | {"name": "PosID","type": ["null","string"]}, 18 | {"name": "CashierID","type": ["null","string"]}, 19 | {"name": "CustomerType","type": ["null","string"]}, 20 | {"name": "DeliveryType","type": ["null","string"]}, 21 | {"name": "DeliveryAddress","type": ["null","DeliveryAddress"]}, 22 | {"name": "InvoiceLineItems","type": {"type": "array", "items": "LineItem"}} 23 | ] 24 | } -------------------------------------------------------------------------------- /Code Files/exactlyoncefanout/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | pos-input-channel: 6 | destination: avro-pos-topic 7 | kafka: 8 | streams: 9 | binder: 10 | brokers: localhost:9092 11 | configuration: 12 | schema.registry.url: http://localhost:8081 13 | processing.guarantee: exactly_once 14 | default: 15 | key: 16 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 17 | value: 18 | serde: io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Packt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | # Kafka-Streams-with-Spring-Cloud-Stream 5 | Kafka Streams with Spring Cloud Stream, by Packt publishing. 6 | -------------------------------------------------------------------------------- /advertclicks/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework.boot 7 | spring-boot-starter-parent 8 | 2.4.2 9 | 10 | 11 | guru.learningjournal.examples.kafka 12 | advertclicks 13 | 0.0.1-SNAPSHOT 14 | Advert Clicks 15 | Advert Clicks Demo by Learning Journal 16 | 17 | 11 18 | Hoxton.SR9 19 | 20 | 21 | 22 | org.apache.kafka 23 | kafka-streams 24 | 25 | 26 | org.springframework.cloud 27 | spring-cloud-stream 28 | 29 | 30 | org.springframework.cloud 31 | spring-cloud-stream-binder-kafka-streams 32 | 33 | 34 | 35 | org.projectlombok 36 | lombok 37 | true 38 | 39 | 40 | org.springframework.boot 41 | spring-boot-starter-test 42 | test 43 | 44 | 45 | org.springframework.cloud 46 | spring-cloud-stream 47 | test 48 | test-binder 49 | test-jar 50 | 51 | 52 | 53 | 54 | 55 | org.springframework.cloud 56 | spring-cloud-dependencies 57 | ${spring-cloud.version} 58 | pom 59 | import 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | org.springframework.boot 68 | spring-boot-maven-plugin 69 | 70 | 71 | 72 | org.projectlombok 73 | lombok 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | spring-milestones 83 | Spring Milestones 84 | https://repo.spring.io/milestone 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /advertclicks/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | confluent local services start 6 | 7 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic active-inventories 8 | 9 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic ad-clicks 10 | 11 | kafka-console-producer --broker-list localhost:9092 --topic active-inventories \ 12 | --property parse.key=true --property key.separator=":" 13 | 14 | kafka-console-producer --broker-list localhost:9092 --topic ad-clicks \ 15 | --property parse.key=true --property key.separator=":" 16 | 17 | confluent local destroy -------------------------------------------------------------------------------- /advertclicks/practice-resources/samples.txt: -------------------------------------------------------------------------------- 1 | 1001:{"InventoryID": "1001", "NewsType": "Sports"} 2 | 1002:{"InventoryID": "1002", "NewsType": "Politics"} 3 | 1003:{"InventoryID": "1003", "NewsType": "LocalNews"} 4 | 1004:{"InventoryID": "1004", "NewsType": "WorldNews"} 5 | 1005:{"InventoryID": "1005", "NewsType": "Health"} 6 | 1006:{"InventoryID": "1006", "NewsType": "Lifestyle"} 7 | 1007:{"InventoryID": "1007", "NewsType": "Literature"} 8 | 1008:{"InventoryID": "1008", "NewsType": "Education"} 9 | 1009:{"InventoryID": "1009", "NewsType": "Social"} 10 | 1010:{"InventoryID": "1010", "NewsType": "Business"} 11 | 12 | 1001:{"InventoryID": "1001"} 13 | 1002:{"InventoryID": "1002"} 14 | 1003:{"InventoryID": "1003"} 15 | 1004:{"InventoryID": "1004"} 16 | 1004:{"InventoryID": "1004"} 17 | 1005:{"InventoryID": "1005"} 18 | 1006:{"InventoryID": "1006"} 19 | 1007:{"InventoryID": "1007"} 20 | 1008:{"InventoryID": "1008"} 21 | 1009:{"InventoryID": "1009"} 22 | 1010:{"InventoryID": "1010"} 23 | -------------------------------------------------------------------------------- /advertclicks/src/main/java/guru/learningjournal/examples/kafka/advertclicks/AdvertClicksApplication.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.advertclicks; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class AdvertClicksApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(AdvertClicksApplication.class, args); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /advertclicks/src/main/java/guru/learningjournal/examples/kafka/advertclicks/Services/ClickListenerService.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.advertclicks.Services; 2 | 3 | import guru.learningjournal.examples.kafka.advertclicks.bindings.ClicksListenerBinding; 4 | import guru.learningjournal.examples.kafka.advertclicks.models.AdClick; 5 | import guru.learningjournal.examples.kafka.advertclicks.models.AdInventories; 6 | import lombok.extern.log4j.Log4j2; 7 | import org.apache.kafka.common.serialization.Serdes; 8 | import org.apache.kafka.streams.kstream.GlobalKTable; 9 | import org.apache.kafka.streams.kstream.Grouped; 10 | import org.apache.kafka.streams.kstream.KStream; 11 | import org.springframework.cloud.stream.annotation.EnableBinding; 12 | import org.springframework.cloud.stream.annotation.Input; 13 | import org.springframework.cloud.stream.annotation.StreamListener; 14 | import org.springframework.kafka.support.serializer.JsonSerde; 15 | import org.springframework.stereotype.Service; 16 | 17 | @Log4j2 18 | @Service 19 | @EnableBinding(ClicksListenerBinding.class) 20 | public class ClickListenerService { 21 | 22 | @StreamListener 23 | public void process(@Input("inventories-channel") GlobalKTable inventory, 24 | @Input("clicks-channel") KStream click) { 25 | 26 | click.foreach((k, v) -> log.info("Click Key: {}, Value: {}",k, v)); 27 | 28 | click.join(inventory, 29 | (clickKey, clickValue) -> clickKey, 30 | (clickValue, inventoryValue) -> inventoryValue) 31 | .groupBy((joinedKey, joinedValue) -> joinedValue.getNewsType(), 32 | Grouped.with(Serdes.String(), 33 | new JsonSerde<>(AdInventories.class))) 34 | .count() 35 | .toStream() 36 | .foreach((k, v) -> log.info("Click Key: {}, Value: {}",k, v)); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /advertclicks/src/main/java/guru/learningjournal/examples/kafka/advertclicks/bindings/ClicksListenerBinding.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.advertclicks.bindings; 2 | 3 | import guru.learningjournal.examples.kafka.advertclicks.models.AdClick; 4 | import guru.learningjournal.examples.kafka.advertclicks.models.AdInventories; 5 | import org.apache.kafka.streams.kstream.GlobalKTable; 6 | import org.apache.kafka.streams.kstream.KStream; 7 | import org.springframework.cloud.stream.annotation.Input; 8 | 9 | public interface ClicksListenerBinding { 10 | 11 | @Input("inventories-channel") 12 | GlobalKTable inventoryInputStream(); 13 | 14 | @Input("clicks-channel") 15 | KStream clickInputStream(); 16 | 17 | } 18 | -------------------------------------------------------------------------------- /advertclicks/src/main/java/guru/learningjournal/examples/kafka/advertclicks/models/AdClick.java: -------------------------------------------------------------------------------- 1 | 2 | package guru.learningjournal.examples.kafka.advertclicks.models; 3 | 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import lombok.Data; 6 | 7 | @Data 8 | public class AdClick { 9 | 10 | @JsonProperty("InventoryID") 11 | private String inventoryID; 12 | 13 | } 14 | -------------------------------------------------------------------------------- /advertclicks/src/main/java/guru/learningjournal/examples/kafka/advertclicks/models/AdInventories.java: -------------------------------------------------------------------------------- 1 | 2 | package guru.learningjournal.examples.kafka.advertclicks.models; 3 | 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import lombok.Data; 6 | 7 | @Data 8 | public class AdInventories { 9 | 10 | @JsonProperty("InventoryID") 11 | private String inventoryID; 12 | @JsonProperty("NewsType") 13 | private String newsType; 14 | 15 | } 16 | -------------------------------------------------------------------------------- /advertclicks/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | inventories-channel: 6 | destination: active-inventories 7 | clicks-channel: 8 | destination: ad-clicks 9 | kafka: 10 | streams: 11 | binder: 12 | brokers: localhost:9092 13 | configuration: 14 | commit.interval.ms: 10000 15 | state.dir: state-store 16 | default: 17 | key: 18 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 19 | value: 20 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 21 | 22 | -------------------------------------------------------------------------------- /advertclicks/src/test/java/guru/learningjournal/examples/kafka/advertclicks/AdvertClicksApplicationTests.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.advertclicks; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class AdvertClicksApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /avroposfanout/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | netsh interface portproxy add v4tov4 listenport=8081 listenaddress=0.0.0.0 connectport=8081 connectaddress= 6 | 7 | confluent local services start 8 | 9 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic loyalty-topic 10 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic hadoop-sink-topic 11 | 12 | 13 | http://localhost:9021/ 14 | 15 | confluent local services stop 16 | confluent local destroy -------------------------------------------------------------------------------- /avroposfanout/src/main/avro/DeliveryAddress.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "DeliveryAddress", 5 | "fields": [ 6 | {"name": "AddressLine","type": ["null","string"]}, 7 | {"name": "City","type": ["null","string"]}, 8 | {"name": "State","type": ["null","string"]}, 9 | {"name": "PinCode","type": ["null","string"]}, 10 | {"name": "ContactNumber","type": ["null","string"]} 11 | ] 12 | } -------------------------------------------------------------------------------- /avroposfanout/src/main/avro/LineItem.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "LineItem", 5 | "fields": [ 6 | {"name": "ItemCode","type": ["null","string"]}, 7 | {"name": "ItemDescription","type": ["null","string"]}, 8 | {"name": "ItemPrice","type": ["null","double"]}, 9 | {"name": "ItemQty","type": ["null","int"]}, 10 | {"name": "TotalValue","type": ["null","double"]} 11 | ] 12 | } -------------------------------------------------------------------------------- /avroposfanout/src/main/avro/PosInvoice.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "PosInvoice", 5 | "fields": [ 6 | {"name": "InvoiceNumber","type": ["null","string"]}, 7 | {"name": "CreatedTime","type": ["null","long"]}, 8 | {"name": "CustomerCardNo","type": ["null","string"]}, 9 | {"name": "TotalAmount","type": ["null","double"]}, 10 | {"name": "NumberOfItems","type": ["null","int"]}, 11 | {"name": "PaymentMethod","type": ["null","string"]}, 12 | {"name": "TaxableAmount","type": ["null","double"]}, 13 | {"name": "CGST","type": ["null","double"]}, 14 | {"name": "SGST","type": ["null","double"]}, 15 | {"name": "CESS","type": ["null","double"]}, 16 | {"name": "StoreID","type": ["null","string"]}, 17 | {"name": "PosID","type": ["null","string"]}, 18 | {"name": "CashierID","type": ["null","string"]}, 19 | {"name": "CustomerType","type": ["null","string"]}, 20 | {"name": "DeliveryType","type": ["null","string"]}, 21 | {"name": "DeliveryAddress","type": ["null","DeliveryAddress"]}, 22 | {"name": "InvoiceLineItems","type": {"type": "array", "items": "LineItem"}} 23 | ] 24 | } -------------------------------------------------------------------------------- /avroposfanout/src/main/java/guru/learningjournal/examples/kafka/avroposfanout/AvroPosFanoutApplication.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.avroposfanout; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class AvroPosFanoutApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(AvroPosFanoutApplication.class, args); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /avroposfanout/src/main/java/guru/learningjournal/examples/kafka/avroposfanout/bindings/PosListenerBinding.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.avroposfanout.bindings; 2 | 3 | 4 | import guru.learningjournal.examples.kafka.avroposfanout.model.HadoopRecord; 5 | import guru.learningjournal.examples.kafka.avroposfanout.model.Notification; 6 | import guru.learningjournal.examples.kafka.model.PosInvoice; 7 | import org.apache.kafka.streams.kstream.KStream; 8 | import org.springframework.cloud.stream.annotation.Input; 9 | import org.springframework.cloud.stream.annotation.Output; 10 | 11 | public interface PosListenerBinding { 12 | 13 | @Input("notification-input-channel") 14 | KStream notificationInputStream(); 15 | 16 | @Output("notification-output-channel") 17 | KStream notificationOutputStream(); 18 | 19 | @Input("hadoop-input-channel") 20 | KStream hadoopInputStream(); 21 | 22 | @Output("hadoop-output-channel") 23 | KStream hadoopOutputStream(); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /avroposfanout/src/main/java/guru/learningjournal/examples/kafka/avroposfanout/model/HadoopRecord.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.avroposfanout.model; 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import lombok.Data; 6 | 7 | @Data 8 | @JsonInclude(JsonInclude.Include.NON_NULL) 9 | public class HadoopRecord { 10 | 11 | @JsonProperty("InvoiceNumber") 12 | private String InvoiceNumber; 13 | @JsonProperty("CreatedTime") 14 | private Long CreatedTime; 15 | @JsonProperty("StoreID") 16 | private String StoreID; 17 | @JsonProperty("PosID") 18 | private String PosID; 19 | @JsonProperty("CustomerType") 20 | private String CustomerType; 21 | @JsonProperty("PaymentMethod") 22 | private String PaymentMethod; 23 | @JsonProperty("DeliveryType") 24 | private String DeliveryType; 25 | @JsonProperty("City") 26 | private String City; 27 | @JsonProperty("State") 28 | private String State; 29 | @JsonProperty("PinCode") 30 | private String PinCode; 31 | @JsonProperty("ItemCode") 32 | private String ItemCode; 33 | @JsonProperty("ItemDescription") 34 | private String ItemDescription; 35 | @JsonProperty("ItemPrice") 36 | private Double ItemPrice; 37 | @JsonProperty("ItemQty") 38 | private Integer ItemQty; 39 | @JsonProperty("TotalValue") 40 | private Double TotalValue; 41 | } 42 | -------------------------------------------------------------------------------- /avroposfanout/src/main/java/guru/learningjournal/examples/kafka/avroposfanout/model/Notification.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.avroposfanout.model; 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import lombok.Data; 6 | 7 | @Data 8 | @JsonInclude(JsonInclude.Include.NON_NULL) 9 | public class Notification { 10 | 11 | @JsonProperty("InvoiceNumber") 12 | private String InvoiceNumber; 13 | @JsonProperty("CustomerCardNo") 14 | private String CustomerCardNo; 15 | @JsonProperty("TotalAmount") 16 | private Double TotalAmount; 17 | @JsonProperty("EarnedLoyaltyPoints") 18 | private Double EarnedLoyaltyPoints; 19 | } 20 | -------------------------------------------------------------------------------- /avroposfanout/src/main/java/guru/learningjournal/examples/kafka/avroposfanout/services/HadoopRecordProcessorService.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.avroposfanout.services; 2 | 3 | import guru.learningjournal.examples.kafka.avroposfanout.bindings.PosListenerBinding; 4 | import guru.learningjournal.examples.kafka.avroposfanout.model.HadoopRecord; 5 | import guru.learningjournal.examples.kafka.model.PosInvoice; 6 | import lombok.extern.log4j.Log4j2; 7 | import org.apache.kafka.streams.kstream.KStream; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.cloud.stream.annotation.EnableBinding; 10 | import org.springframework.cloud.stream.annotation.StreamListener; 11 | import org.springframework.messaging.handler.annotation.SendTo; 12 | import org.springframework.stereotype.Service; 13 | 14 | @Service 15 | @Log4j2 16 | @EnableBinding(PosListenerBinding.class) 17 | public class HadoopRecordProcessorService { 18 | 19 | @Autowired 20 | RecordBuilder recordBuilder; 21 | 22 | @StreamListener("hadoop-input-channel") 23 | @SendTo("hadoop-output-channel") 24 | public KStream process(KStream input) { 25 | 26 | KStream hadoopRecordKStream = input 27 | .mapValues( v -> recordBuilder.getMaskedInvoice(v)) 28 | .flatMapValues( v -> recordBuilder.getHadoopRecords(v)); 29 | 30 | hadoopRecordKStream.foreach((k, v) -> log.info(String.format("Hadoop Record:- Key: %s, Value: %s", k, v))); 31 | 32 | return hadoopRecordKStream; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /avroposfanout/src/main/java/guru/learningjournal/examples/kafka/avroposfanout/services/NotificationProcessorService.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.avroposfanout.services; 2 | 3 | import guru.learningjournal.examples.kafka.avroposfanout.bindings.PosListenerBinding; 4 | import guru.learningjournal.examples.kafka.avroposfanout.model.Notification; 5 | import guru.learningjournal.examples.kafka.model.PosInvoice; 6 | import lombok.extern.log4j.Log4j2; 7 | import org.apache.kafka.streams.kstream.KStream; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.cloud.stream.annotation.EnableBinding; 10 | import org.springframework.cloud.stream.annotation.StreamListener; 11 | import org.springframework.messaging.handler.annotation.SendTo; 12 | import org.springframework.stereotype.Service; 13 | 14 | @Service 15 | @Log4j2 16 | @EnableBinding(PosListenerBinding.class) 17 | public class NotificationProcessorService { 18 | 19 | @Autowired 20 | RecordBuilder recordBuilder; 21 | 22 | @StreamListener("notification-input-channel") 23 | @SendTo("notification-output-channel") 24 | public KStream process(KStream input) { 25 | 26 | KStream notificationKStream = input 27 | .filter((k, v) -> v.getCustomerType().equalsIgnoreCase("PRIME")) 28 | .mapValues(v -> recordBuilder.getNotification(v)); 29 | 30 | notificationKStream.foreach((k, v) -> log.info(String.format("Notification:- Key: %s, Value: %s", k, v))); 31 | 32 | return notificationKStream; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /avroposfanout/src/main/java/guru/learningjournal/examples/kafka/avroposfanout/services/RecordBuilder.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.avroposfanout.services; 2 | 3 | import guru.learningjournal.examples.kafka.avroposfanout.model.HadoopRecord; 4 | import guru.learningjournal.examples.kafka.avroposfanout.model.Notification; 5 | import guru.learningjournal.examples.kafka.model.LineItem; 6 | import guru.learningjournal.examples.kafka.model.PosInvoice; 7 | import org.springframework.stereotype.Service; 8 | 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | 12 | @Service 13 | public class RecordBuilder { 14 | 15 | public Notification getNotification(PosInvoice invoice){ 16 | Notification notification = new Notification(); 17 | notification.setInvoiceNumber(invoice.getInvoiceNumber()); 18 | notification.setCustomerCardNo(invoice.getCustomerCardNo()); 19 | notification.setTotalAmount(invoice.getTotalAmount()); 20 | notification.setEarnedLoyaltyPoints(invoice.getTotalAmount() * 0.02); 21 | return notification; 22 | } 23 | 24 | public PosInvoice getMaskedInvoice(PosInvoice invoice){ 25 | invoice.setCustomerCardNo(null); 26 | if (invoice.getDeliveryType().equalsIgnoreCase("HOME-DELIVERY")) { 27 | invoice.getDeliveryAddress().setAddressLine(null); 28 | invoice.getDeliveryAddress().setContactNumber(null); 29 | } 30 | return invoice; 31 | } 32 | 33 | public List getHadoopRecords(PosInvoice invoice){ 34 | List records = new ArrayList<>(); 35 | 36 | for (LineItem i : invoice.getInvoiceLineItems()) { 37 | HadoopRecord record = new HadoopRecord(); 38 | record.setInvoiceNumber(invoice.getInvoiceNumber()); 39 | record.setCreatedTime(invoice.getCreatedTime()); 40 | record.setStoreID(invoice.getStoreID()); 41 | record.setPosID(invoice.getPosID()); 42 | record.setCustomerType(invoice.getCustomerType()); 43 | record.setPaymentMethod(invoice.getPaymentMethod()); 44 | record.setDeliveryType(invoice.getDeliveryType()); 45 | record.setItemCode(i.getItemCode()); 46 | record.setItemDescription(i.getItemDescription()); 47 | record.setItemPrice(i.getItemPrice()); 48 | record.setItemQty(i.getItemQty()); 49 | record.setTotalValue(i.getTotalValue()); 50 | if (invoice.getDeliveryType().toString().equalsIgnoreCase("HOME-DELIVERY")) { 51 | record.setCity(invoice.getDeliveryAddress().getCity()); 52 | record.setState(invoice.getDeliveryAddress().getState()); 53 | record.setPinCode(invoice.getDeliveryAddress().getPinCode()); 54 | } 55 | records.add(record); 56 | } 57 | return records; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /avroposfanout/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | notification-input-channel: 6 | destination: avro-pos-topic 7 | notification-output-channel: 8 | destination: loyalty-topic 9 | hadoop-input-channel: 10 | destination: avro-pos-topic 11 | hadoop-output-channel: 12 | destination: hadoop-sink-topic 13 | kafka: 14 | streams: 15 | binder: 16 | brokers: localhost:9092 17 | configuration: 18 | schema.registry.url: http://localhost:8081 19 | bindings: 20 | notification-input-channel: 21 | consumer: 22 | valueSerde: io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde 23 | notification-output-channel: 24 | producer: 25 | valueSerde: io.confluent.kafka.streams.serdes.json.KafkaJsonSchemaSerde 26 | hadoop-input-channel: 27 | consumer: 28 | valueSerde: io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde 29 | hadoop-output-channel: 30 | producer: 31 | valueSerde: io.confluent.kafka.streams.serdes.json.KafkaJsonSchemaSerde 32 | -------------------------------------------------------------------------------- /avroposfanout/src/test/java/guru/learningjournal/examples/kafka/avroposfanout/AvroPosFanoutApplicationTests.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.avroposfanout; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class AvroPosFanoutApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /avroposgen/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework.boot 7 | spring-boot-starter-parent 8 | 2.4.0 9 | 10 | 11 | learningjournal.guru.examples 12 | avroposgen 13 | 0.0.1-SNAPSHOT 14 | Avro Pos Generator 15 | Avro Pos Generator by Learning Journal 16 | 17 | 18 | 11 19 | 20 | 21 | 22 | 23 | org.springframework.boot 24 | spring-boot-starter 25 | 26 | 27 | org.springframework.kafka 28 | spring-kafka 29 | 30 | 31 | 32 | org.apache.avro 33 | avro 34 | 1.9.2 35 | 36 | 37 | io.confluent 38 | kafka-avro-serializer 39 | 6.0.0 40 | 41 | 42 | 43 | org.projectlombok 44 | lombok 45 | true 46 | 47 | 48 | org.springframework.boot 49 | spring-boot-starter-test 50 | test 51 | 52 | 53 | org.springframework.kafka 54 | spring-kafka-test 55 | test 56 | 57 | 58 | 59 | 60 | 61 | 62 | org.springframework.boot 63 | spring-boot-maven-plugin 64 | 65 | 66 | org.apache.avro 67 | avro-maven-plugin 68 | 1.8.2 69 | 70 | 71 | generate-sources 72 | 73 | schema 74 | 75 | 76 | src/main/avro 77 | ${project.build.directory}/generated-sources 78 | 79 | ${project.basedir}/src/main/avro/LineItem.avsc 80 | ${project.basedir}/src/main/avro/DeliveryAddress.avsc 81 | 82 | String 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | confluent 93 | https://packages.confluent.io/maven/ 94 | 95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /avroposgen/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | netsh interface portproxy add v4tov4 listenport=8081 listenaddress=0.0.0.0 connectport=8081 connectaddress= 6 | 7 | confluent local services start 8 | 9 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic avro-pos-topic 10 | 11 | kafka-avro-console-consumer --bootstrap-server localhost:9092 --topic avro-pos-topic --from-beginning --property key.deserializer=org.apache.kafka.common.serialization.StringDeserializer --property print.key=true --property key.separator=":" 12 | 13 | confluent local services stop 14 | confluent local destroy -------------------------------------------------------------------------------- /avroposgen/src/main/avro/DeliveryAddress.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "DeliveryAddress", 5 | "fields": [ 6 | {"name": "AddressLine","type": ["null","string"]}, 7 | {"name": "City","type": ["null","string"]}, 8 | {"name": "State","type": ["null","string"]}, 9 | {"name": "PinCode","type": ["null","string"]}, 10 | {"name": "ContactNumber","type": ["null","string"]} 11 | ] 12 | } -------------------------------------------------------------------------------- /avroposgen/src/main/avro/LineItem.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "LineItem", 5 | "fields": [ 6 | {"name": "ItemCode","type": ["null","string"]}, 7 | {"name": "ItemDescription","type": ["null","string"]}, 8 | {"name": "ItemPrice","type": ["null","double"]}, 9 | {"name": "ItemQty","type": ["null","int"]}, 10 | {"name": "TotalValue","type": ["null","double"]} 11 | ] 12 | } -------------------------------------------------------------------------------- /avroposgen/src/main/avro/PosInvoice.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "PosInvoice", 5 | "fields": [ 6 | {"name": "InvoiceNumber","type": ["null","string"]}, 7 | {"name": "CreatedTime","type": ["null","long"]}, 8 | {"name": "CustomerCardNo","type": ["null","string"]}, 9 | {"name": "TotalAmount","type": ["null","double"]}, 10 | {"name": "NumberOfItems","type": ["null","int"]}, 11 | {"name": "PaymentMethod","type": ["null","string"]}, 12 | {"name": "TaxableAmount","type": ["null","double"]}, 13 | {"name": "CGST","type": ["null","double"]}, 14 | {"name": "SGST","type": ["null","double"]}, 15 | {"name": "CESS","type": ["null","double"]}, 16 | {"name": "StoreID","type": ["null","string"]}, 17 | {"name": "PosID","type": ["null","string"]}, 18 | {"name": "CashierID","type": ["null","string"]}, 19 | {"name": "CustomerType","type": ["null","string"]}, 20 | {"name": "DeliveryType","type": ["null","string"]}, 21 | {"name": "DeliveryAddress","type": ["null","DeliveryAddress"]}, 22 | {"name": "InvoiceLineItems","type": {"type": "array", "items": "LineItem"}} 23 | ] 24 | } -------------------------------------------------------------------------------- /avroposgen/src/main/java/guru/learningjournal/examples/kafka/avroposgen/AvroPosGeneratorApplication.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.avroposgen; 2 | 3 | import guru.learningjournal.examples.kafka.avroposgen.services.datagenerator.InvoiceGenerator; 4 | import guru.learningjournal.examples.kafka.avroposgen.services.KafkaProducerService; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.beans.factory.annotation.Value; 7 | import org.springframework.boot.ApplicationArguments; 8 | import org.springframework.boot.ApplicationRunner; 9 | import org.springframework.boot.SpringApplication; 10 | import org.springframework.boot.autoconfigure.SpringBootApplication; 11 | 12 | @SpringBootApplication 13 | public class AvroPosGeneratorApplication implements ApplicationRunner { 14 | 15 | public static void main(String[] args) { 16 | SpringApplication.run(AvroPosGeneratorApplication.class, args); 17 | } 18 | 19 | @Autowired 20 | private KafkaProducerService producerService; 21 | 22 | @Autowired 23 | private InvoiceGenerator invoiceGenerator; 24 | 25 | @Value("${application.configs.invoice.count}") 26 | private int INVOICE_COUNT; 27 | 28 | @Override 29 | public void run(ApplicationArguments args) throws Exception { 30 | 31 | for (int i = 0; i < INVOICE_COUNT; i++) { 32 | producerService.sendMessage(invoiceGenerator.getNextInvoice()); 33 | Thread.sleep(1000); 34 | } 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /avroposgen/src/main/java/guru/learningjournal/examples/kafka/avroposgen/services/KafkaProducerService.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.avroposgen.services; 2 | 3 | import guru.learningjournal.examples.kafka.model.PosInvoice; 4 | import lombok.extern.log4j.Log4j2; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.beans.factory.annotation.Value; 7 | import org.springframework.kafka.core.KafkaTemplate; 8 | import org.springframework.stereotype.Service; 9 | 10 | @Service 11 | @Log4j2 12 | public class KafkaProducerService { 13 | @Value("${application.configs.topic.name}") 14 | private String TOPIC_NAME; 15 | 16 | @Autowired 17 | private KafkaTemplate kafkaTemplate; 18 | 19 | public void sendMessage(PosInvoice invoice) { 20 | log.info(String.format("Producing Invoice No: %s", invoice.getInvoiceNumber())); 21 | kafkaTemplate.send(TOPIC_NAME, invoice.getStoreID(), invoice); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /avroposgen/src/main/java/guru/learningjournal/examples/kafka/avroposgen/services/datagenerator/AddressGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.examples.kafka.avroposgen.services.datagenerator; 17 | 18 | import com.fasterxml.jackson.databind.ObjectMapper; 19 | import guru.learningjournal.examples.kafka.model.DeliveryAddress; 20 | import org.springframework.stereotype.Service; 21 | 22 | import java.io.File; 23 | import java.util.Random; 24 | 25 | @Service 26 | class AddressGenerator { 27 | 28 | private final Random random; 29 | private final DeliveryAddress[] addresses; 30 | 31 | public AddressGenerator() { 32 | final String DATAFILE = "src/main/resources/data/address.json"; 33 | final ObjectMapper mapper; 34 | random = new Random(); 35 | mapper = new ObjectMapper(); 36 | try { 37 | addresses = mapper.readValue(new File(DATAFILE), DeliveryAddress[].class); 38 | } catch (Exception e) { 39 | throw new RuntimeException(e); 40 | } 41 | } 42 | 43 | private int getIndex() { 44 | return random.nextInt(100); 45 | } 46 | 47 | public DeliveryAddress getNextAddress() { 48 | return addresses[getIndex()]; 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /avroposgen/src/main/java/guru/learningjournal/examples/kafka/avroposgen/services/datagenerator/ProductGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.examples.kafka.avroposgen.services.datagenerator; 17 | 18 | import com.fasterxml.jackson.databind.ObjectMapper; 19 | import guru.learningjournal.examples.kafka.model.LineItem; 20 | import org.springframework.stereotype.Service; 21 | 22 | import java.io.File; 23 | import java.util.Random; 24 | 25 | @Service 26 | class ProductGenerator { 27 | private final Random random; 28 | private final Random qty; 29 | private final LineItem[] products; 30 | 31 | public ProductGenerator() { 32 | String DATAFILE = "src/main/resources/data/products.json"; 33 | ObjectMapper mapper = new ObjectMapper(); 34 | random = new Random(); 35 | qty = new Random(); 36 | try { 37 | products = mapper.readValue(new File(DATAFILE), LineItem[].class); 38 | } catch (Exception e) { 39 | throw new RuntimeException(e); 40 | } 41 | } 42 | 43 | private int getIndex() { 44 | return random.nextInt(100); 45 | } 46 | 47 | private int getQuantity() { 48 | return qty.nextInt(2) + 1; 49 | } 50 | 51 | public LineItem getNextProduct() { 52 | LineItem lineItem = products[getIndex()]; 53 | lineItem.setItemQty(getQuantity()); 54 | lineItem.setTotalValue(lineItem.getItemPrice() * lineItem.getItemQty()); 55 | return lineItem; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /avroposgen/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | kafka: 3 | producer: 4 | client-id: avro-pos-simulator 5 | bootstrap-servers: localhost:9092 6 | key-serializer: org.apache.kafka.common.serialization.StringSerializer 7 | value-serializer: io.confluent.kafka.serializers.KafkaAvroSerializer 8 | properties: 9 | schema.registry.url: http://localhost:8081 10 | 11 | application: 12 | configs: 13 | invoice.count: 60 14 | topic.name: avro-pos-topic 15 | 16 | 17 | -------------------------------------------------------------------------------- /avroposgen/src/test/java/guru/learningjournal/examples/kafka/avroposgen/AvroPosGeneratorApplicationTests.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.avroposgen; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class AvroPosGeneratorApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /exactlyoncefanout/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | netsh interface portproxy add v4tov4 listenport=8081 listenaddress=0.0.0.0 connectport=8081 connectaddress= 6 | 7 | confluent local services start 8 | 9 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic hadoop-sink-topic 10 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic loyalty-topic 11 | 12 | kafka-avro-console-consumer --bootstrap-server localhost:9092 --topic hadoop-sink-topic --from-beginning --property key.deserializer=org.apache.kafka.common.serialization.StringDeserializer --property print.key=true --property key.separator=":" 13 | kafka-avro-console-consumer --bootstrap-server localhost:9092 --topic loyalty-topic --from-beginning --property key.deserializer=org.apache.kafka.common.serialization.StringDeserializer --property print.key=true --property key.separator=":" 14 | 15 | confluent local services stop 16 | confluent local destroy -------------------------------------------------------------------------------- /exactlyoncefanout/src/main/avro/DeliveryAddress.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "DeliveryAddress", 5 | "fields": [ 6 | {"name": "AddressLine","type": ["null","string"]}, 7 | {"name": "City","type": ["null","string"]}, 8 | {"name": "State","type": ["null","string"]}, 9 | {"name": "PinCode","type": ["null","string"]}, 10 | {"name": "ContactNumber","type": ["null","string"]} 11 | ] 12 | } -------------------------------------------------------------------------------- /exactlyoncefanout/src/main/avro/HadoopRecord.avsc: -------------------------------------------------------------------------------- 1 | {"namespace": "guru.learningjournal.examples.kafka.model", 2 | "type": "record", 3 | "name": "HadoopRecord", 4 | "fields": [ 5 | {"name": "InvoiceNumber", "type": ["null", "string"]}, 6 | {"name": "CreatedTime", "type":["null", "long"]}, 7 | {"name": "StoreID", "type": ["null", "string"]}, 8 | {"name": "PosID", "type": ["null", "string"]}, 9 | {"name": "CustomerType", "type": ["null", "string"]}, 10 | {"name": "PaymentMethod", "type": ["null", "string"]}, 11 | {"name": "DeliveryType", "type": ["null", "string"]}, 12 | {"name": "City", "type": ["null", "string"]}, 13 | {"name": "State", "type": ["null", "string"]}, 14 | {"name": "PinCode", "type": ["null", "string"]}, 15 | {"name": "ItemCode", "type": ["null", "string"]}, 16 | {"name": "ItemDescription", "type": ["null", "string"]}, 17 | {"name": "ItemPrice", "type": ["null", "double"]}, 18 | {"name": "ItemQty", "type": ["null", "int"]}, 19 | {"name": "TotalValue", "type": ["null", "double"]} 20 | ] 21 | } -------------------------------------------------------------------------------- /exactlyoncefanout/src/main/avro/LineItem.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "LineItem", 5 | "fields": [ 6 | {"name": "ItemCode","type": ["null","string"]}, 7 | {"name": "ItemDescription","type": ["null","string"]}, 8 | {"name": "ItemPrice","type": ["null","double"]}, 9 | {"name": "ItemQty","type": ["null","int"]}, 10 | {"name": "TotalValue","type": ["null","double"]} 11 | ] 12 | } -------------------------------------------------------------------------------- /exactlyoncefanout/src/main/avro/Notification.avsc: -------------------------------------------------------------------------------- 1 | {"namespace": "guru.learningjournal.examples.kafka.model", 2 | "type": "record", 3 | "name": "Notification", 4 | "fields": [ 5 | {"name": "InvoiceNumber", "type": ["null", "string"]}, 6 | {"name": "CustomerCardNo", "type":["null", "string"]}, 7 | {"name": "TotalAmount", "type": ["null", "double"]}, 8 | {"name": "EarnedLoyaltyPoints", "type": ["null", "double"]} 9 | ] 10 | } -------------------------------------------------------------------------------- /exactlyoncefanout/src/main/avro/PosInvoice.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "PosInvoice", 5 | "fields": [ 6 | {"name": "InvoiceNumber","type": ["null","string"]}, 7 | {"name": "CreatedTime","type": ["null","long"]}, 8 | {"name": "CustomerCardNo","type": ["null","string"]}, 9 | {"name": "TotalAmount","type": ["null","double"]}, 10 | {"name": "NumberOfItems","type": ["null","int"]}, 11 | {"name": "PaymentMethod","type": ["null","string"]}, 12 | {"name": "TaxableAmount","type": ["null","double"]}, 13 | {"name": "CGST","type": ["null","double"]}, 14 | {"name": "SGST","type": ["null","double"]}, 15 | {"name": "CESS","type": ["null","double"]}, 16 | {"name": "StoreID","type": ["null","string"]}, 17 | {"name": "PosID","type": ["null","string"]}, 18 | {"name": "CashierID","type": ["null","string"]}, 19 | {"name": "CustomerType","type": ["null","string"]}, 20 | {"name": "DeliveryType","type": ["null","string"]}, 21 | {"name": "DeliveryAddress","type": ["null","DeliveryAddress"]}, 22 | {"name": "InvoiceLineItems","type": {"type": "array", "items": "LineItem"}} 23 | ] 24 | } -------------------------------------------------------------------------------- /exactlyoncefanout/src/main/java/guru/learningjournal/examples/kafka/exactlyoncefanout/ExactlyOnceFanoutApplication.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.exactlyoncefanout; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class ExactlyOnceFanoutApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(ExactlyOnceFanoutApplication.class, args); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /exactlyoncefanout/src/main/java/guru/learningjournal/examples/kafka/exactlyoncefanout/bindings/PosListenerBinding.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.exactlyoncefanout.bindings; 2 | 3 | 4 | import guru.learningjournal.examples.kafka.model.PosInvoice; 5 | import org.apache.kafka.streams.kstream.KStream; 6 | import org.springframework.cloud.stream.annotation.Input; 7 | 8 | public interface PosListenerBinding { 9 | 10 | @Input("pos-input-channel") 11 | KStream posInputStream(); 12 | 13 | } 14 | -------------------------------------------------------------------------------- /exactlyoncefanout/src/main/java/guru/learningjournal/examples/kafka/exactlyoncefanout/services/PosListenerService.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.exactlyoncefanout.services; 2 | 3 | import guru.learningjournal.examples.kafka.exactlyoncefanout.bindings.PosListenerBinding; 4 | import guru.learningjournal.examples.kafka.model.HadoopRecord; 5 | import guru.learningjournal.examples.kafka.model.Notification; 6 | import guru.learningjournal.examples.kafka.model.PosInvoice; 7 | import lombok.extern.log4j.Log4j2; 8 | import org.apache.kafka.streams.kstream.KStream; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.cloud.stream.annotation.EnableBinding; 11 | import org.springframework.cloud.stream.annotation.StreamListener; 12 | import org.springframework.stereotype.Service; 13 | 14 | @Service 15 | @Log4j2 16 | @EnableBinding(PosListenerBinding.class) 17 | public class PosListenerService { 18 | 19 | @Autowired 20 | RecordBuilder recordBuilder; 21 | 22 | @StreamListener("pos-input-channel") 23 | public void process(KStream input) { 24 | 25 | KStream hadoopRecordKStream = input 26 | .mapValues( v -> recordBuilder.getMaskedInvoice(v)) 27 | .flatMapValues( v -> recordBuilder.getHadoopRecords(v)); 28 | 29 | KStream notificationKStream = input 30 | .filter((k, v) -> v.getCustomerType().equalsIgnoreCase("PRIME")) 31 | .mapValues(v -> recordBuilder.getNotification(v)); 32 | 33 | hadoopRecordKStream.foreach((k, v) -> log.info(String.format("Hadoop Record:- Key: %s, Value: %s", k, v))); 34 | notificationKStream.foreach((k, v) -> log.info(String.format("Notification:- Key: %s, Value: %s", k, v))); 35 | 36 | hadoopRecordKStream.to("hadoop-sink-topic"); 37 | notificationKStream.to("loyalty-topic"); 38 | 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /exactlyoncefanout/src/main/java/guru/learningjournal/examples/kafka/exactlyoncefanout/services/RecordBuilder.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.exactlyoncefanout.services; 2 | 3 | import guru.learningjournal.examples.kafka.model.HadoopRecord; 4 | import guru.learningjournal.examples.kafka.model.LineItem; 5 | import guru.learningjournal.examples.kafka.model.Notification; 6 | import guru.learningjournal.examples.kafka.model.PosInvoice; 7 | import org.springframework.stereotype.Service; 8 | 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | 12 | @Service 13 | public class RecordBuilder { 14 | 15 | public Notification getNotification(PosInvoice invoice){ 16 | Notification notification = new Notification(); 17 | notification.setInvoiceNumber(invoice.getInvoiceNumber()); 18 | notification.setCustomerCardNo(invoice.getCustomerCardNo()); 19 | notification.setTotalAmount(invoice.getTotalAmount()); 20 | notification.setEarnedLoyaltyPoints(invoice.getTotalAmount() * 0.02); 21 | return notification; 22 | } 23 | 24 | public PosInvoice getMaskedInvoice(PosInvoice invoice){ 25 | invoice.setCustomerCardNo(null); 26 | if (invoice.getDeliveryType().equalsIgnoreCase("HOME-DELIVERY")) { 27 | invoice.getDeliveryAddress().setAddressLine(null); 28 | invoice.getDeliveryAddress().setContactNumber(null); 29 | } 30 | return invoice; 31 | } 32 | 33 | public List getHadoopRecords(PosInvoice invoice){ 34 | List records = new ArrayList<>(); 35 | 36 | for (LineItem i : invoice.getInvoiceLineItems()) { 37 | HadoopRecord record = new HadoopRecord(); 38 | record.setInvoiceNumber(invoice.getInvoiceNumber()); 39 | record.setCreatedTime(invoice.getCreatedTime()); 40 | record.setStoreID(invoice.getStoreID()); 41 | record.setPosID(invoice.getPosID()); 42 | record.setCustomerType(invoice.getCustomerType()); 43 | record.setPaymentMethod(invoice.getPaymentMethod()); 44 | record.setDeliveryType(invoice.getDeliveryType()); 45 | record.setItemCode(i.getItemCode()); 46 | record.setItemDescription(i.getItemDescription()); 47 | record.setItemPrice(i.getItemPrice()); 48 | record.setItemQty(i.getItemQty()); 49 | record.setTotalValue(i.getTotalValue()); 50 | if (invoice.getDeliveryType().toString().equalsIgnoreCase("HOME-DELIVERY")) { 51 | record.setCity(invoice.getDeliveryAddress().getCity()); 52 | record.setState(invoice.getDeliveryAddress().getState()); 53 | record.setPinCode(invoice.getDeliveryAddress().getPinCode()); 54 | } 55 | records.add(record); 56 | } 57 | return records; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /exactlyoncefanout/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | pos-input-channel: 6 | destination: avro-pos-topic 7 | kafka: 8 | streams: 9 | binder: 10 | brokers: localhost:9092 11 | configuration: 12 | schema.registry.url: http://localhost:8081 13 | processing.guarantee: exactly_once 14 | default: 15 | key: 16 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 17 | value: 18 | serde: io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde 19 | -------------------------------------------------------------------------------- /exactlyoncefanout/src/test/java/guru/learningjournal/examples/kafka/exactlyoncefanout/ExactlyOnceFanoutApplicationTests.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.exactlyoncefanout; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class ExactlyOnceFanoutApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /hello-streams/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'org.springframework.boot' version '2.3.5.RELEASE' 3 | id 'io.spring.dependency-management' version '1.0.10.RELEASE' 4 | id 'java' 5 | } 6 | 7 | group = 'guru.learningjournal.examples.kafka' 8 | version = '0.0.1-SNAPSHOT' 9 | sourceCompatibility = '11' 10 | 11 | configurations { 12 | compileOnly { 13 | extendsFrom annotationProcessor 14 | } 15 | } 16 | 17 | repositories { 18 | mavenCentral() 19 | } 20 | 21 | ext { 22 | set('springCloudVersion', "Hoxton.SR8") 23 | } 24 | 25 | dependencies { 26 | implementation 'org.apache.kafka:kafka-streams' 27 | implementation 'org.springframework.cloud:spring-cloud-stream' 28 | implementation 'org.springframework.cloud:spring-cloud-stream-binder-kafka-streams' 29 | compileOnly 'org.projectlombok:lombok' 30 | annotationProcessor 'org.projectlombok:lombok' 31 | testImplementation('org.springframework.boot:spring-boot-starter-test') { 32 | exclude group: 'org.junit.vintage', module: 'junit-vintage-engine' 33 | } 34 | testImplementation 'org.springframework.cloud:spring-cloud-stream-test-support' 35 | } 36 | 37 | dependencyManagement { 38 | imports { 39 | mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}" 40 | } 41 | } 42 | 43 | test { 44 | useJUnitPlatform() 45 | } 46 | -------------------------------------------------------------------------------- /hello-streams/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Kafka-Streams-with-Spring-Cloud-Stream/aa6319b0454246240d0610994d06900b3ef73c9d/hello-streams/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /hello-streams/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /hello-streams/practice-resources/data-samples.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | {"name": "Kristie Cole", "age": 34,"gender": "female"} 4 | 5 | {"name": "Marsh Mccall", "age": 28, "gender": "male"} -------------------------------------------------------------------------------- /hello-streams/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic users 6 | 7 | kafka-console-producer --broker-list localhost:9092 --topic users 8 | 9 | kafka-console-consumer --bootstrap-server localhost:9092 --topic users --from-beginning 10 | 11 | -------------------------------------------------------------------------------- /hello-streams/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'hello-streams' 2 | -------------------------------------------------------------------------------- /hello-streams/src/main/java/guru/learningjournal/examples/kafka/hellostreams/HelloStreamsApplication.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.hellostreams; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class HelloStreamsApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(HelloStreamsApplication.class, args); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /hello-streams/src/main/java/guru/learningjournal/examples/kafka/hellostreams/bindings/KafkaListenerBinding.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.hellostreams.bindings; 2 | 3 | import org.apache.kafka.streams.kstream.KStream; 4 | import org.springframework.cloud.stream.annotation.Input; 5 | 6 | public interface KafkaListenerBinding { 7 | @Input("input-channel-1") 8 | KStream inputStream(); 9 | } 10 | -------------------------------------------------------------------------------- /hello-streams/src/main/java/guru/learningjournal/examples/kafka/hellostreams/services/KafkaListenerService.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.hellostreams.services; 2 | 3 | import guru.learningjournal.examples.kafka.hellostreams.bindings.KafkaListenerBinding; 4 | import lombok.extern.log4j.Log4j2; 5 | import org.apache.kafka.streams.kstream.KStream; 6 | import org.springframework.cloud.stream.annotation.EnableBinding; 7 | import org.springframework.cloud.stream.annotation.StreamListener; 8 | import org.springframework.stereotype.Service; 9 | 10 | @Log4j2 11 | @Service 12 | @EnableBinding(KafkaListenerBinding.class) 13 | public class KafkaListenerService { 14 | 15 | @StreamListener("input-channel-1") 16 | public void process(KStream input) { 17 | input.foreach((k,v) -> log.info(String.format("Key: %s, Value: %s",k,v))); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /hello-streams/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | input-channel-1: 6 | destination: users 7 | kafka: 8 | streams: 9 | binder: 10 | applicationId: hellostreams 11 | brokers: localhost:9092 12 | configuration: 13 | default: 14 | key: 15 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 16 | value: 17 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde -------------------------------------------------------------------------------- /hello-streams/src/test/java/guru/learningjournal/examples/kafka/hellostreams/HelloStreamsApplicationTests.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.hellostreams; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class HelloStreamsApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /jsonposgen/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'org.springframework.boot' version '2.4.0' 3 | id 'io.spring.dependency-management' version '1.0.10.RELEASE' 4 | id 'java' 5 | } 6 | 7 | group = 'guru.learningjournal.kafka.examples' 8 | version = '0.0.1-SNAPSHOT' 9 | sourceCompatibility = '11' 10 | 11 | configurations { 12 | compileOnly { 13 | extendsFrom annotationProcessor 14 | } 15 | } 16 | 17 | repositories { 18 | mavenCentral() 19 | } 20 | 21 | dependencies { 22 | implementation 'org.springframework.boot:spring-boot-starter' 23 | implementation 'org.springframework.kafka:spring-kafka' 24 | implementation 'com.fasterxml.jackson.core:jackson-databind' 25 | //implementation 'org.apache.commons:commons-lang3' 26 | compileOnly 'org.projectlombok:lombok' 27 | annotationProcessor 'org.projectlombok:lombok' 28 | testImplementation 'org.springframework.boot:spring-boot-starter-test' 29 | testImplementation 'org.springframework.kafka:spring-kafka-test' 30 | } 31 | 32 | test { 33 | useJUnitPlatform() 34 | } 35 | -------------------------------------------------------------------------------- /jsonposgen/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Kafka-Streams-with-Spring-Cloud-Stream/aa6319b0454246240d0610994d06900b3ef73c9d/jsonposgen/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /jsonposgen/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /jsonposgen/gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /jsonposgen/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | confluent local services start 6 | 7 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic pos-topic 8 | 9 | kafka-console-consumer --bootstrap-server localhost:9092 --topic pos-topic --from-beginning --property print.key=true --property key.separator=":" 10 | 11 | confluent local services stop 12 | confluent local destroy -------------------------------------------------------------------------------- /jsonposgen/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'jsonposgen' 2 | -------------------------------------------------------------------------------- /jsonposgen/src/main/java/guru/learningjournal/examples/kafka/jsonposgen/JsonPosGeneratorApplication.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.jsonposgen; 2 | 3 | import guru.learningjournal.examples.kafka.jsonposgen.services.datagenerator.InvoiceGenerator; 4 | import guru.learningjournal.examples.kafka.jsonposgen.services.KafkaProducerService; 5 | import lombok.extern.log4j.Log4j2; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.beans.factory.annotation.Value; 8 | import org.springframework.boot.ApplicationArguments; 9 | import org.springframework.boot.ApplicationRunner; 10 | import org.springframework.boot.SpringApplication; 11 | import org.springframework.boot.autoconfigure.SpringBootApplication; 12 | 13 | @SpringBootApplication 14 | @Log4j2 15 | public class JsonPosGeneratorApplication implements ApplicationRunner { 16 | 17 | public static void main(String[] args) { 18 | SpringApplication.run(JsonPosGeneratorApplication.class, args); 19 | } 20 | 21 | 22 | @Autowired 23 | private KafkaProducerService producerService; 24 | 25 | @Autowired 26 | private InvoiceGenerator invoiceGenerator; 27 | 28 | @Value("${application.configs.invoice.count}") 29 | private int INVOICE_COUNT; 30 | 31 | @Override 32 | public void run(ApplicationArguments args) throws Exception { 33 | 34 | for (int i = 0; i < INVOICE_COUNT; i++) { 35 | producerService.sendMessage(invoiceGenerator.getNextInvoice()); 36 | Thread.sleep(1000); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /jsonposgen/src/main/java/guru/learningjournal/examples/kafka/jsonposgen/model/DeliveryAddress.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.jsonposgen.model; 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import lombok.Data; 6 | 7 | @Data 8 | @JsonInclude(JsonInclude.Include.NON_NULL) 9 | public class DeliveryAddress { 10 | 11 | @JsonProperty("AddressLine") 12 | private String addressLine; 13 | @JsonProperty("City") 14 | private String city; 15 | @JsonProperty("State") 16 | private String state; 17 | @JsonProperty("PinCode") 18 | private String pinCode; 19 | @JsonProperty("ContactNumber") 20 | private String contactNumber; 21 | 22 | } 23 | -------------------------------------------------------------------------------- /jsonposgen/src/main/java/guru/learningjournal/examples/kafka/jsonposgen/model/LineItem.java: -------------------------------------------------------------------------------- 1 | 2 | package guru.learningjournal.examples.kafka.jsonposgen.model; 3 | 4 | import com.fasterxml.jackson.annotation.JsonInclude; 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | import lombok.Data; 7 | 8 | @Data 9 | @JsonInclude(JsonInclude.Include.NON_NULL) 10 | public class LineItem { 11 | 12 | @JsonProperty("ItemCode") 13 | private String itemCode; 14 | @JsonProperty("ItemDescription") 15 | private String itemDescription; 16 | @JsonProperty("ItemPrice") 17 | private Double itemPrice; 18 | @JsonProperty("ItemQty") 19 | private Integer itemQty; 20 | @JsonProperty("TotalValue") 21 | private Double totalValue; 22 | 23 | } 24 | -------------------------------------------------------------------------------- /jsonposgen/src/main/java/guru/learningjournal/examples/kafka/jsonposgen/model/PosInvoice.java: -------------------------------------------------------------------------------- 1 | 2 | package guru.learningjournal.examples.kafka.jsonposgen.model; 3 | 4 | import com.fasterxml.jackson.annotation.JsonInclude; 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | import lombok.Data; 7 | 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | 11 | 12 | @Data 13 | @JsonInclude(JsonInclude.Include.NON_NULL) 14 | public class PosInvoice { 15 | 16 | @JsonProperty("InvoiceNumber") 17 | private String invoiceNumber; 18 | @JsonProperty("CreatedTime") 19 | private Long createdTime; 20 | @JsonProperty("StoreID") 21 | private String storeID; 22 | @JsonProperty("PosID") 23 | private String posID; 24 | @JsonProperty("CashierID") 25 | private String cashierID; 26 | @JsonProperty("CustomerType") 27 | private String customerType; 28 | @JsonProperty("CustomerCardNo") 29 | private String customerCardNo; 30 | @JsonProperty("TotalAmount") 31 | private Double totalAmount; 32 | @JsonProperty("NumberOfItems") 33 | private Integer numberOfItems; 34 | @JsonProperty("PaymentMethod") 35 | private String paymentMethod; 36 | @JsonProperty("TaxableAmount") 37 | private Double taxableAmount; 38 | @JsonProperty("CGST") 39 | private Double cGST; 40 | @JsonProperty("SGST") 41 | private Double sGST; 42 | @JsonProperty("CESS") 43 | private Double cESS; 44 | @JsonProperty("DeliveryType") 45 | private String deliveryType; 46 | @JsonProperty("DeliveryAddress") 47 | private DeliveryAddress deliveryAddress; 48 | @JsonProperty("InvoiceLineItems") 49 | private List invoiceLineItems = new ArrayList(); 50 | 51 | } 52 | -------------------------------------------------------------------------------- /jsonposgen/src/main/java/guru/learningjournal/examples/kafka/jsonposgen/services/KafkaProducerService.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.jsonposgen.services; 2 | 3 | import guru.learningjournal.examples.kafka.jsonposgen.model.PosInvoice; 4 | import lombok.extern.log4j.Log4j2; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.beans.factory.annotation.Value; 7 | import org.springframework.kafka.core.KafkaTemplate; 8 | import org.springframework.stereotype.Service; 9 | 10 | @Service 11 | @Log4j2 12 | public class KafkaProducerService { 13 | @Value("${application.configs.topic.name}") 14 | private String TOPIC_NAME; 15 | 16 | @Autowired 17 | private KafkaTemplate kafkaTemplate; 18 | 19 | public void sendMessage(PosInvoice invoice) { 20 | log.info(String.format("Producing Invoice No: %s Customer Type: %s", 21 | invoice.getInvoiceNumber(), 22 | invoice.getCustomerType())); 23 | kafkaTemplate.send(TOPIC_NAME, invoice.getStoreID(), invoice); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /jsonposgen/src/main/java/guru/learningjournal/examples/kafka/jsonposgen/services/datagenerator/AddressGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.examples.kafka.jsonposgen.services.datagenerator; 17 | 18 | import com.fasterxml.jackson.databind.ObjectMapper; 19 | import guru.learningjournal.examples.kafka.jsonposgen.model.DeliveryAddress; 20 | import org.springframework.stereotype.Service; 21 | 22 | import java.io.File; 23 | import java.util.Random; 24 | 25 | @Service 26 | class AddressGenerator { 27 | 28 | private final Random random; 29 | private final DeliveryAddress[] addresses; 30 | 31 | public AddressGenerator() { 32 | final String DATAFILE = "src/main/resources/data/address.json"; 33 | final ObjectMapper mapper; 34 | random = new Random(); 35 | mapper = new ObjectMapper(); 36 | try { 37 | addresses = mapper.readValue(new File(DATAFILE), DeliveryAddress[].class); 38 | } catch (Exception e) { 39 | throw new RuntimeException(e); 40 | } 41 | } 42 | 43 | private int getIndex() { 44 | return random.nextInt(100); 45 | } 46 | 47 | public DeliveryAddress getNextAddress() { 48 | return addresses[getIndex()]; 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /jsonposgen/src/main/java/guru/learningjournal/examples/kafka/jsonposgen/services/datagenerator/ProductGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.examples.kafka.jsonposgen.services.datagenerator; 17 | 18 | import com.fasterxml.jackson.databind.ObjectMapper; 19 | import guru.learningjournal.examples.kafka.jsonposgen.model.LineItem; 20 | import org.springframework.stereotype.Service; 21 | 22 | import java.io.File; 23 | import java.util.Random; 24 | 25 | @Service 26 | class ProductGenerator { 27 | private final Random random; 28 | private final Random qty; 29 | private final LineItem[] products; 30 | 31 | public ProductGenerator() { 32 | String DATAFILE = "src/main/resources/data/products.json"; 33 | ObjectMapper mapper = new ObjectMapper(); 34 | random = new Random(); 35 | qty = new Random(); 36 | try { 37 | products = mapper.readValue(new File(DATAFILE), LineItem[].class); 38 | } catch (Exception e) { 39 | throw new RuntimeException(e); 40 | } 41 | } 42 | 43 | private int getIndex() { 44 | return random.nextInt(100); 45 | } 46 | 47 | private int getQuantity() { 48 | return qty.nextInt(2) + 1; 49 | } 50 | 51 | public LineItem getNextProduct() { 52 | LineItem lineItem = products[getIndex()]; 53 | lineItem.setItemQty(getQuantity()); 54 | lineItem.setTotalValue(lineItem.getItemPrice() * lineItem.getItemQty()); 55 | return lineItem; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /jsonposgen/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | kafka: 3 | producer: 4 | client-id: json-pos-simulator 5 | bootstrap-servers: localhost:9092 6 | key-serializer: org.apache.kafka.common.serialization.StringSerializer 7 | value-serializer: org.springframework.kafka.support.serializer.JsonSerializer 8 | properties: 9 | spring.json.add.type.headers: false 10 | 11 | application: 12 | configs: 13 | invoice.count: 60 14 | topic.name: pos-topic 15 | 16 | 17 | -------------------------------------------------------------------------------- /jsonposgen/src/test/java/guru/learningjournal/examples/kafka/jsonposgen/JsonPosGeneratorApplicationTests.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.jsonposgen; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class JsonPosGeneratorApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /kafkaproducer/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'org.springframework.boot' version '2.4.0' 3 | id 'io.spring.dependency-management' version '1.0.10.RELEASE' 4 | id 'java' 5 | } 6 | 7 | group = 'guru.learningjournal.kafka.examples' 8 | version = '0.0.1-SNAPSHOT' 9 | sourceCompatibility = '11' 10 | 11 | configurations { 12 | compileOnly { 13 | extendsFrom annotationProcessor 14 | } 15 | } 16 | 17 | repositories { 18 | mavenCentral() 19 | } 20 | 21 | dependencies { 22 | implementation 'org.springframework.boot:spring-boot-starter-web' 23 | implementation 'org.springframework.kafka:spring-kafka' 24 | compileOnly 'org.projectlombok:lombok' 25 | annotationProcessor 'org.projectlombok:lombok' 26 | testImplementation 'org.springframework.boot:spring-boot-starter-test' 27 | testImplementation 'org.springframework.kafka:spring-kafka-test' 28 | } 29 | 30 | test { 31 | useJUnitPlatform() 32 | } 33 | -------------------------------------------------------------------------------- /kafkaproducer/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Kafka-Streams-with-Spring-Cloud-Stream/aa6319b0454246240d0610994d06900b3ef73c9d/kafkaproducer/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /kafkaproducer/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /kafkaproducer/gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /kafkaproducer/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | confluent local services start 6 | 7 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic users 8 | 9 | kafka-console-consumer --bootstrap-server localhost:9092 --topic users --from-beginning --property print.key=true --property key.separator=":" 10 | 11 | confluent local services stop 12 | confluent local destroy -------------------------------------------------------------------------------- /kafkaproducer/scratches/scratch.rest: -------------------------------------------------------------------------------- 1 | POST 2 | http://localhost:8080/post 3 | @Content-type: application/json 4 | {"topic":"users", "key":"101", "value":"Prashant Kumar Pandey"} 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /kstreamaggregate/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | netsh interface portproxy add v4tov4 listenport=8081 listenaddress=0.0.0.0 connectport=8081 connectaddress= 6 | 7 | confluent local services start 8 | 9 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic employees-topic 10 | 11 | kafka-avro-console-producer --broker-list localhost:9092 --topic employees-topic \ 12 | --property value.schema='{"namespace": "guru.learningjournal.examples.kafka.model","type": "record","name": "Employee","fields": [{"name": "id","type": "string"},{"name": "name","type": "string"},{"name": "department","type": "string"},{"name": "salary","type":"int"}]}' 13 | 14 | 15 | 16 | confluent local destroy -------------------------------------------------------------------------------- /kstreamaggregate/practice-resources/samples.txt: -------------------------------------------------------------------------------- 1 | {"id": "101", "name": "Prashant", "department": "engineering", "salary": 5000} 2 | {"id": "102", "name": "John", "department": "accounts", "salary": 8000} 3 | {"id": "103", "name": "Abdul", "department": "engineering", "salary": 3000} 4 | {"id": "104", "name": "Melinda", "department": "support", "salary": 7000} 5 | {"id": "105", "name": "Jimmy", "department": "support", "salary": 6000} 6 | 7 | 8 | {"id": "101", "name": "Prashant", "department": "support", "salary": 5000} 9 | {"id": "104", "name": "Melinda", "department": "engineering", "salary": 7000} -------------------------------------------------------------------------------- /kstreamaggregate/src/main/avro/DepartmentAggregate.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "DepartmentAggregate", 5 | "fields": [ 6 | {"name": "total_salary","type": ["null","int"]}, 7 | {"name": "employee_count","type": ["null","int"]}, 8 | {"name": "avg_salary","type": ["null","double"]} 9 | ] 10 | } -------------------------------------------------------------------------------- /kstreamaggregate/src/main/avro/Employee.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "Employee", 5 | "fields": [ 6 | {"name": "id","type": ["null","string"]}, 7 | {"name": "name","type": ["null","string"]}, 8 | {"name": "department","type": ["null","string"]}, 9 | {"name": "salary","type": ["null","int"]} 10 | ] 11 | } -------------------------------------------------------------------------------- /kstreamaggregate/src/main/java/guru/learningjournal/examples/kafka/kstreamaggregate/KStreamAggregateApplication.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.kstreamaggregate; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class KStreamAggregateApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(KStreamAggregateApplication.class, args); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /kstreamaggregate/src/main/java/guru/learningjournal/examples/kafka/kstreamaggregate/bindings/EmployeeListenerBinding.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.kstreamaggregate.bindings; 2 | 3 | import guru.learningjournal.examples.kafka.model.Employee; 4 | import org.apache.kafka.streams.kstream.KStream; 5 | import org.springframework.cloud.stream.annotation.Input; 6 | 7 | public interface EmployeeListenerBinding { 8 | 9 | @Input("employee-input-channel") 10 | KStream employeeInputStream(); 11 | 12 | } 13 | -------------------------------------------------------------------------------- /kstreamaggregate/src/main/java/guru/learningjournal/examples/kafka/kstreamaggregate/services/EmployeeStreamListener.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.kstreamaggregate.services; 2 | 3 | import guru.learningjournal.examples.kafka.kstreamaggregate.bindings.EmployeeListenerBinding; 4 | import guru.learningjournal.examples.kafka.model.Employee; 5 | import lombok.extern.log4j.Log4j2; 6 | import org.apache.kafka.streams.kstream.KStream; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.cloud.stream.annotation.EnableBinding; 9 | import org.springframework.cloud.stream.annotation.StreamListener; 10 | import org.springframework.stereotype.Service; 11 | 12 | @Log4j2 13 | @Service 14 | @EnableBinding(EmployeeListenerBinding.class) 15 | public class EmployeeStreamListener { 16 | 17 | @Autowired 18 | RecordBuilder recordBuilder; 19 | 20 | @StreamListener("employee-input-channel") 21 | public void process(KStream input) { 22 | 23 | input.peek((k, v) -> log.info("Key: {}, Value:{}", k, v)) 24 | .groupBy((k, v) -> v.getDepartment()) 25 | .aggregate( 26 | () -> recordBuilder.init(), 27 | (k, v, aggV) -> recordBuilder.aggregate(v, aggV) 28 | ).toStream() 29 | .foreach((k, v) -> log.info("Key = " + k + " Value = " + v.toString())); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /kstreamaggregate/src/main/java/guru/learningjournal/examples/kafka/kstreamaggregate/services/RecordBuilder.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.kstreamaggregate.services; 2 | 3 | import guru.learningjournal.examples.kafka.model.DepartmentAggregate; 4 | import guru.learningjournal.examples.kafka.model.Employee; 5 | import lombok.extern.log4j.Log4j2; 6 | import org.springframework.stereotype.Service; 7 | 8 | @Log4j2 9 | @Service 10 | public class RecordBuilder { 11 | public DepartmentAggregate init(){ 12 | DepartmentAggregate departmentAggregate = new DepartmentAggregate(); 13 | departmentAggregate.setEmployeeCount(0); 14 | departmentAggregate.setTotalSalary(0); 15 | departmentAggregate.setAvgSalary(0D); 16 | return departmentAggregate; 17 | } 18 | 19 | public DepartmentAggregate aggregate(Employee emp, DepartmentAggregate aggValue){ 20 | DepartmentAggregate departmentAggregate = new DepartmentAggregate(); 21 | departmentAggregate.setEmployeeCount(aggValue.getEmployeeCount() + 1); 22 | departmentAggregate.setTotalSalary(aggValue.getTotalSalary() + emp.getSalary()); 23 | departmentAggregate.setAvgSalary((aggValue.getTotalSalary() + emp.getSalary()) / (aggValue.getEmployeeCount() + 1D)); 24 | return departmentAggregate; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /kstreamaggregate/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | employee-input-channel: 6 | destination: employees-topic 7 | kafka: 8 | streams: 9 | binder: 10 | brokers: localhost:9092 11 | configuration: 12 | schema.registry.url: http://localhost:8081 13 | commit.interval.ms: 10000 14 | state.dir: state-store 15 | default: 16 | key: 17 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 18 | value: 19 | serde: io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde 20 | 21 | -------------------------------------------------------------------------------- /kstreamaggregate/src/test/java/guru/learningjournal/examples/kafka/kstreamaggregate/KStreamAggregateApplicationTests.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.kstreamaggregate; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class KStreamAggregateApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /ktableaggregate/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | netsh interface portproxy add v4tov4 listenport=8081 listenaddress=0.0.0.0 connectport=8081 connectaddress= 6 | 7 | confluent local services start 8 | 9 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic employees-topic 10 | 11 | kafka-avro-console-producer --broker-list localhost:9092 --topic employees-topic \ 12 | --property value.schema='{"namespace": "guru.learningjournal.examples.kafka.model","type": "record","name": "Employee","fields": [{"name": "id","type": "string"},{"name": "name","type": "string"},{"name": "department","type": "string"},{"name": "salary","type":"int"}]}' 13 | 14 | 15 | 16 | confluent local destroy -------------------------------------------------------------------------------- /ktableaggregate/practice-resources/samples.txt: -------------------------------------------------------------------------------- 1 | {"id": "101", "name": "Prashant", "department": "engineering", "salary": 5000} 2 | {"id": "102", "name": "John", "department": "accounts", "salary": 8000} 3 | {"id": "103", "name": "Abdul", "department": "engineering", "salary": 3000} 4 | {"id": "104", "name": "Melinda", "department": "support", "salary": 7000} 5 | {"id": "105", "name": "Jimmy", "department": "support", "salary": 6000} 6 | 7 | 8 | {"id": "101", "name": "Prashant", "department": "support", "salary": 5000} 9 | {"id": "104", "name": "Melinda", "department": "engineering", "salary": 7000} 10 | 11 | {"id": "101", "name": "Prashant", "department": "support", "salary": 6000} -------------------------------------------------------------------------------- /ktableaggregate/src/main/avro/DepartmentAggregate.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "DepartmentAggregate", 5 | "fields": [ 6 | {"name": "total_salary","type": ["null","int"]}, 7 | {"name": "employee_count","type": ["null","int"]}, 8 | {"name": "avg_salary","type": ["null","double"]} 9 | ] 10 | } -------------------------------------------------------------------------------- /ktableaggregate/src/main/avro/Employee.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "Employee", 5 | "fields": [ 6 | {"name": "id","type": ["null","string"]}, 7 | {"name": "name","type": ["null","string"]}, 8 | {"name": "department","type": ["null","string"]}, 9 | {"name": "salary","type": ["null","int"]} 10 | ] 11 | } -------------------------------------------------------------------------------- /ktableaggregate/src/main/java/guru/learningjournal/examples/kafka/ktableaggregate/KTableAggregateApplication.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.ktableaggregate; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class KTableAggregateApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(KTableAggregateApplication.class, args); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /ktableaggregate/src/main/java/guru/learningjournal/examples/kafka/ktableaggregate/bindings/EmployeeListenerBinding.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.ktableaggregate.bindings; 2 | 3 | import guru.learningjournal.examples.kafka.model.Employee; 4 | import org.apache.kafka.streams.kstream.KStream; 5 | import org.springframework.cloud.stream.annotation.Input; 6 | 7 | public interface EmployeeListenerBinding { 8 | 9 | @Input("employee-input-channel") 10 | KStream employeeInputStream(); 11 | 12 | } 13 | -------------------------------------------------------------------------------- /ktableaggregate/src/main/java/guru/learningjournal/examples/kafka/ktableaggregate/services/EmployeeStreamListener.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.ktableaggregate.services; 2 | 3 | import guru.learningjournal.examples.kafka.ktableaggregate.bindings.EmployeeListenerBinding; 4 | import guru.learningjournal.examples.kafka.model.Employee; 5 | import lombok.extern.log4j.Log4j2; 6 | import org.apache.kafka.streams.KeyValue; 7 | import org.apache.kafka.streams.kstream.KStream; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.cloud.stream.annotation.EnableBinding; 10 | import org.springframework.cloud.stream.annotation.StreamListener; 11 | import org.springframework.stereotype.Service; 12 | 13 | @Log4j2 14 | @Service 15 | @EnableBinding(EmployeeListenerBinding.class) 16 | public class EmployeeStreamListener { 17 | 18 | @Autowired 19 | RecordBuilder recordBuilder; 20 | 21 | @StreamListener("employee-input-channel") 22 | public void process(KStream input) { 23 | 24 | input.map((k, v) -> KeyValue.pair(v.getId(), v)) 25 | .peek((k, v) -> log.info("Key = " + k + " Value = " + v)) 26 | .toTable() 27 | .groupBy((k, v) -> KeyValue.pair(v.getDepartment(), v)) 28 | .aggregate( 29 | () -> recordBuilder.init(), 30 | (k, v, aggV) -> recordBuilder.aggregate(v, aggV), 31 | (k, v, aggV) -> recordBuilder.subtract(v, aggV) 32 | ).toStream() 33 | .foreach((k, v) -> log.info("Key = " + k + " Value = " + v)); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /ktableaggregate/src/main/java/guru/learningjournal/examples/kafka/ktableaggregate/services/RecordBuilder.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.ktableaggregate.services; 2 | 3 | import guru.learningjournal.examples.kafka.model.DepartmentAggregate; 4 | import guru.learningjournal.examples.kafka.model.Employee; 5 | import lombok.extern.log4j.Log4j2; 6 | import org.springframework.stereotype.Service; 7 | 8 | @Log4j2 9 | @Service 10 | public class RecordBuilder { 11 | public DepartmentAggregate init(){ 12 | DepartmentAggregate departmentAggregate = new DepartmentAggregate(); 13 | departmentAggregate.setEmployeeCount(0); 14 | departmentAggregate.setTotalSalary(0); 15 | departmentAggregate.setAvgSalary(0D); 16 | return departmentAggregate; 17 | } 18 | 19 | public DepartmentAggregate aggregate(Employee emp, DepartmentAggregate aggValue){ 20 | DepartmentAggregate departmentAggregate = new DepartmentAggregate(); 21 | departmentAggregate.setEmployeeCount(aggValue.getEmployeeCount() + 1); 22 | departmentAggregate.setTotalSalary(aggValue.getTotalSalary() + emp.getSalary()); 23 | departmentAggregate.setAvgSalary((aggValue.getTotalSalary() + emp.getSalary()) / (aggValue.getEmployeeCount() + 1D)); 24 | return departmentAggregate; 25 | } 26 | 27 | public DepartmentAggregate subtract(Employee emp, DepartmentAggregate aggValue){ 28 | DepartmentAggregate departmentAggregate = new DepartmentAggregate(); 29 | departmentAggregate.setEmployeeCount(aggValue.getEmployeeCount() - 1); 30 | departmentAggregate.setTotalSalary(aggValue.getTotalSalary() - emp.getSalary()); 31 | departmentAggregate.setAvgSalary((aggValue.getTotalSalary() - emp.getSalary()) / (aggValue.getEmployeeCount() - 1D)); 32 | return departmentAggregate; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /ktableaggregate/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | employee-input-channel: 6 | destination: employees-topic 7 | kafka: 8 | streams: 9 | binder: 10 | brokers: localhost:9092 11 | configuration: 12 | schema.registry.url: http://localhost:8081 13 | commit.interval.ms: 10000 14 | state.dir: state-store 15 | default: 16 | key: 17 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 18 | value: 19 | serde: io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde 20 | 21 | -------------------------------------------------------------------------------- /ktableaggregate/src/test/java/guru/learningjournal/examples/kafka/ktableaggregate/KTableAggregateApplicationTests.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.ktableaggregate; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class KTableAggregateApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /ktabledemo/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'org.springframework.boot' version '2.4.1' 3 | id 'io.spring.dependency-management' version '1.0.10.RELEASE' 4 | id 'java' 5 | } 6 | 7 | group = 'guru.learningjournal.examples.kafka' 8 | version = '0.0.1-SNAPSHOT' 9 | sourceCompatibility = '11' 10 | 11 | configurations { 12 | compileOnly { 13 | extendsFrom annotationProcessor 14 | } 15 | } 16 | 17 | repositories { 18 | mavenCentral() 19 | maven { url 'https://repo.spring.io/milestone' } 20 | } 21 | 22 | ext { 23 | set('springCloudVersion', "Hoxton.SR9") 24 | } 25 | 26 | dependencies { 27 | implementation 'org.apache.kafka:kafka-streams' 28 | implementation 'org.springframework.cloud:spring-cloud-stream' 29 | implementation 'org.springframework.cloud:spring-cloud-stream-binder-kafka-streams' 30 | compileOnly 'org.projectlombok:lombok' 31 | annotationProcessor 'org.projectlombok:lombok' 32 | testImplementation 'org.springframework.boot:spring-boot-starter-test' 33 | } 34 | 35 | dependencyManagement { 36 | imports { 37 | mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}" 38 | } 39 | } 40 | 41 | test { 42 | useJUnitPlatform() 43 | } 44 | -------------------------------------------------------------------------------- /ktabledemo/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Kafka-Streams-with-Spring-Cloud-Stream/aa6319b0454246240d0610994d06900b3ef73c9d/ktabledemo/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /ktabledemo/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.7.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /ktabledemo/gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /ktabledemo/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | confluent local services start 6 | 7 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic stock-tick-topic 8 | 9 | kafka-console-producer --topic stock-tick-topic --broker-list localhost:9092 --property parse.key=true --property key.separator=":" 10 | 11 | confluent local services stop 12 | confluent local destroy -------------------------------------------------------------------------------- /ktabledemo/practice-resources/sample.txt: -------------------------------------------------------------------------------- 1 | HDFCBANK:2120 2 | HDFCBANK:2150 3 | HDFCBANK:2180 4 | 5 | TCS:2920 6 | -------------------------------------------------------------------------------- /ktabledemo/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'ktabledemo' 2 | -------------------------------------------------------------------------------- /ktabledemo/src/main/java/guru/learningjournal/examples/kafka/ktabledemo/KTableDemoApplication.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.ktabledemo; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class KTableDemoApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(KTableDemoApplication.class, args); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /ktabledemo/src/main/java/guru/learningjournal/examples/kafka/ktabledemo/bindings/StockListenerBinding.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.ktabledemo.bindings; 2 | 3 | import org.apache.kafka.streams.kstream.KTable; 4 | import org.springframework.cloud.stream.annotation.Input; 5 | 6 | 7 | public interface StockListenerBinding { 8 | 9 | @Input("stock-input-channel") 10 | KTable stockInputStream(); 11 | } 12 | -------------------------------------------------------------------------------- /ktabledemo/src/main/java/guru/learningjournal/examples/kafka/ktabledemo/services/StockTickListenerService.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.ktabledemo.services; 2 | 3 | import guru.learningjournal.examples.kafka.ktabledemo.bindings.StockListenerBinding; 4 | import lombok.extern.log4j.Log4j2; 5 | import org.apache.kafka.streams.kstream.KTable; 6 | import org.springframework.cloud.stream.annotation.EnableBinding; 7 | import org.springframework.cloud.stream.annotation.StreamListener; 8 | import org.springframework.stereotype.Service; 9 | 10 | @Service 11 | @Log4j2 12 | @EnableBinding(StockListenerBinding.class) 13 | public class StockTickListenerService { 14 | 15 | @StreamListener("stock-input-channel") 16 | public void process(KTable input) { 17 | 18 | input.filter((key, value) -> key.contains("HDFCBANK")) 19 | .toStream() 20 | .foreach((k, v) -> System.out.println("Key = " + k + " Value = " + v)); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /ktabledemo/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | stock-input-channel: 6 | destination: stock-tick-topic 7 | kafka: 8 | streams: 9 | binder: 10 | brokers: localhost:9092 11 | configuration: 12 | commit.interval.ms: 10000 13 | state.dir: state-store 14 | bindings: 15 | stock-input-channel: 16 | consumer: 17 | materializedAs: stock-input-store 18 | 19 | -------------------------------------------------------------------------------- /ktabledemo/src/test/java/guru/learningjournal/examples/kafka/ktabledemo/KTableDemoApplicationTests.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.ktabledemo; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class KTableDemoApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /lastlogin/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework.boot 7 | spring-boot-starter-parent 8 | 2.4.2 9 | 10 | 11 | guru.learningjournal.examples.kafka 12 | lastlogin 13 | 0.0.1-SNAPSHOT 14 | Last Login 15 | Last Login Demo by Learning Journal 16 | 17 | 11 18 | Hoxton.SR9 19 | 20 | 21 | 22 | org.apache.kafka 23 | kafka-streams 24 | 25 | 26 | org.springframework.cloud 27 | spring-cloud-stream 28 | 29 | 30 | org.springframework.cloud 31 | spring-cloud-stream-binder-kafka-streams 32 | 33 | 34 | 35 | org.projectlombok 36 | lombok 37 | true 38 | 39 | 40 | org.springframework.boot 41 | spring-boot-starter-test 42 | test 43 | 44 | 45 | org.springframework.cloud 46 | spring-cloud-stream 47 | test 48 | test-binder 49 | test-jar 50 | 51 | 52 | 53 | 54 | 55 | org.springframework.cloud 56 | spring-cloud-dependencies 57 | ${spring-cloud.version} 58 | pom 59 | import 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | org.springframework.boot 68 | spring-boot-maven-plugin 69 | 70 | 71 | 72 | org.projectlombok 73 | lombok 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | spring-milestones 83 | Spring Milestones 84 | https://repo.spring.io/milestone 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /lastlogin/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | confluent local services start 6 | 7 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic user-master 8 | 9 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic user-login 10 | 11 | kafka-console-producer --broker-list localhost:9092 --topic user-master \ 12 | --property parse.key=true --property key.separator=":" 13 | 14 | kafka-console-producer --broker-list localhost:9092 --topic user-login \ 15 | --property parse.key=true --property key.separator=":" 16 | 17 | confluent local destroy -------------------------------------------------------------------------------- /lastlogin/practice-resources/samples.txt: -------------------------------------------------------------------------------- 1 | 100001:{"UserName": "Prashant", "LoginID": "100001", "LastLogin": 1550150109302} 2 | 100009:{"UserName": "Alisha", "LoginID": "100009", "LastLogin": 1550150280409} 3 | 100087:{"UserName": "Abdul", "LoginID": "100087", "LastLogin": 1550150290305} 4 | 5 | 6 | 100001:{"LoginID": "100001", "CreatedTime": 1550150291000} 7 | 100087:{"LoginID": "100087", "CreatedTime": 1550150580000} 8 | 9 | -------------------------------------------------------------------------------- /lastlogin/src/main/java/guru/learningjournal/examples/kafka/lastlogin/LastLoginApplication.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.lastlogin; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class LastLoginApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(LastLoginApplication.class, args); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /lastlogin/src/main/java/guru/learningjournal/examples/kafka/lastlogin/bindings/UserListenerBinding.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.lastlogin.bindings; 2 | 3 | import guru.learningjournal.examples.kafka.lastlogin.model.UserDetails; 4 | import guru.learningjournal.examples.kafka.lastlogin.model.UserLogin; 5 | import org.apache.kafka.streams.kstream.KTable; 6 | import org.springframework.cloud.stream.annotation.Input; 7 | 8 | public interface UserListenerBinding { 9 | 10 | @Input("user-master-channel") 11 | KTable userInputStream(); 12 | 13 | @Input("user-login-channel") 14 | KTable loginInputStream(); 15 | 16 | } 17 | -------------------------------------------------------------------------------- /lastlogin/src/main/java/guru/learningjournal/examples/kafka/lastlogin/model/UserDetails.java: -------------------------------------------------------------------------------- 1 | 2 | package guru.learningjournal.examples.kafka.lastlogin.model; 3 | 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import lombok.Data; 6 | 7 | @Data 8 | public class UserDetails { 9 | 10 | @JsonProperty("UserName") 11 | private String userName; 12 | @JsonProperty("LoginID") 13 | private String loginID; 14 | @JsonProperty("LastLogin") 15 | private Long lastLogin; 16 | 17 | } 18 | -------------------------------------------------------------------------------- /lastlogin/src/main/java/guru/learningjournal/examples/kafka/lastlogin/model/UserLogin.java: -------------------------------------------------------------------------------- 1 | 2 | package guru.learningjournal.examples.kafka.lastlogin.model; 3 | 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import lombok.Data; 6 | 7 | @Data 8 | public class UserLogin { 9 | 10 | @JsonProperty("LoginID") 11 | private String loginID; 12 | @JsonProperty("CreatedTime") 13 | private Long createdTime; 14 | 15 | } 16 | -------------------------------------------------------------------------------- /lastlogin/src/main/java/guru/learningjournal/examples/kafka/lastlogin/services/LoginListenerService.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.lastlogin.services; 2 | 3 | import guru.learningjournal.examples.kafka.lastlogin.bindings.UserListenerBinding; 4 | import guru.learningjournal.examples.kafka.lastlogin.model.UserDetails; 5 | import guru.learningjournal.examples.kafka.lastlogin.model.UserLogin; 6 | import lombok.extern.log4j.Log4j2; 7 | import org.apache.kafka.streams.kstream.KTable; 8 | import org.springframework.cloud.stream.annotation.EnableBinding; 9 | import org.springframework.cloud.stream.annotation.Input; 10 | import org.springframework.cloud.stream.annotation.StreamListener; 11 | import org.springframework.stereotype.Service; 12 | 13 | import java.time.Instant; 14 | import java.time.ZoneOffset; 15 | 16 | @Log4j2 17 | @Service 18 | @EnableBinding(UserListenerBinding.class) 19 | public class LoginListenerService { 20 | 21 | @StreamListener 22 | public void process(@Input("user-master-channel") KTable users, 23 | @Input("user-login-channel") KTable logins) { 24 | 25 | users.toStream().foreach((k, v) -> log.info("User Key: {}, Last Login: {}, Value{}", 26 | k, Instant.ofEpochMilli(v.getLastLogin()).atOffset(ZoneOffset.UTC), v)); 27 | 28 | logins.toStream().foreach((k, v) -> log.info("Login Key: {}, Last Login: {}, Value{}", 29 | k, Instant.ofEpochMilli(v.getCreatedTime()).atOffset(ZoneOffset.UTC), v)); 30 | 31 | logins.join(users, (l, u) -> { 32 | u.setLastLogin(l.getCreatedTime()); 33 | return u; 34 | }).toStream().foreach((k, v) -> log.info("Updated Last Login Key: {}, Last Login: {}", k, 35 | Instant.ofEpochMilli(v.getLastLogin()).atOffset(ZoneOffset.UTC))); 36 | 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /lastlogin/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | user-master-channel: 6 | destination: user-master 7 | user-login-channel: 8 | destination: user-login 9 | kafka: 10 | streams: 11 | binder: 12 | brokers: localhost:9092 13 | configuration: 14 | commit.interval.ms: 10000 15 | state.dir: state-store 16 | default: 17 | key: 18 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 19 | value: 20 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 21 | 22 | -------------------------------------------------------------------------------- /lastlogin/src/test/java/guru/learningjournal/examples/kafka/lastlogin/LastLoginApplicationTests.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.lastlogin; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class LastLoginApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /otpvalidation/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework.boot 7 | spring-boot-starter-parent 8 | 2.4.2 9 | 10 | 11 | guru.learningjournal.examples.kafka 12 | otpvalidation 13 | 0.0.1-SNAPSHOT 14 | OTP Validation 15 | OTP Validation Demo by Learning Journal 16 | 17 | 11 18 | Hoxton.SR9 19 | 20 | 21 | 22 | org.apache.kafka 23 | kafka-streams 24 | 25 | 26 | org.springframework.cloud 27 | spring-cloud-stream 28 | 29 | 30 | org.springframework.cloud 31 | spring-cloud-stream-binder-kafka-streams 32 | 33 | 34 | 35 | org.projectlombok 36 | lombok 37 | true 38 | 39 | 40 | org.springframework.boot 41 | spring-boot-starter-test 42 | test 43 | 44 | 45 | org.springframework.cloud 46 | spring-cloud-stream 47 | test 48 | test-binder 49 | test-jar 50 | 51 | 52 | 53 | 54 | 55 | org.springframework.cloud 56 | spring-cloud-dependencies 57 | ${spring-cloud.version} 58 | pom 59 | import 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | org.springframework.boot 68 | spring-boot-maven-plugin 69 | 70 | 71 | 72 | org.projectlombok 73 | lombok 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | spring-milestones 83 | Spring Milestones 84 | https://repo.spring.io/milestone 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /otpvalidation/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | confluent local services start 6 | 7 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic payment_request 8 | 9 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic payment_confirmation 10 | 11 | kafka-console-producer --broker-list localhost:9092 --topic payment_request \ 12 | --property parse.key=true --property key.separator=":" 13 | 14 | kafka-console-producer --broker-list localhost:9092 --topic payment_confirmation \ 15 | --property parse.key=true --property key.separator=":" 16 | 17 | confluent local destroy -------------------------------------------------------------------------------- /otpvalidation/practice-resources/samples.txt: -------------------------------------------------------------------------------- 1 | 100001:{"TransactionID": "100001", "CreatedTime": 1550149860000, "SourceAccountID": "131100", "TargetAccountID": "151837", "Amount": 3000, "OTP": 852960} 2 | 100002:{"TransactionID": "100002", "CreatedTime": 1550149920000, "SourceAccountID": "131200", "TargetAccountID": "151837", "Amount": 2000, "OTP": 931749} 3 | 100003:{"TransactionID": "100003", "CreatedTime": 1550149980000, "SourceAccountID": "131300", "TargetAccountID": "151837", "Amount": 5000, "OTP": 591296} 4 | 100004:{"TransactionID": "100004", "CreatedTime": 1550150100000, "SourceAccountID": "131400", "TargetAccountID": "151837", "Amount": 1000, "OTP": 283084} 5 | 6 | 100001:{"TransactionID": "100001", "CreatedTime": 1550150100000, "OTP": 852960} 7 | 100002:{"TransactionID": "100002", "CreatedTime": 1550150280000, "OTP": 931749} 8 | 100004:{"TransactionID": "100004", "CreatedTime": 1550150040000, "OTP": 283086} 9 | 10 | -------------------------------------------------------------------------------- /otpvalidation/src/main/java/guru/learningjournal/examples/kafka/otpvalidation/OtpValidationApplication.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.otpvalidation; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class OtpValidationApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(OtpValidationApplication.class, args); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /otpvalidation/src/main/java/guru/learningjournal/examples/kafka/otpvalidation/bindings/OTPListenerBinding.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.otpvalidation.bindings; 2 | 3 | import guru.learningjournal.examples.kafka.otpvalidation.model.PaymentConfirmation; 4 | import guru.learningjournal.examples.kafka.otpvalidation.model.PaymentRequest; 5 | import org.apache.kafka.streams.kstream.KStream; 6 | import org.springframework.cloud.stream.annotation.Input; 7 | 8 | public interface OTPListenerBinding { 9 | 10 | @Input("payment-request-channel") 11 | KStream requestInputStream(); 12 | 13 | @Input("payment-confirmation-channel") 14 | KStream confirmationInputStream(); 15 | 16 | } 17 | -------------------------------------------------------------------------------- /otpvalidation/src/main/java/guru/learningjournal/examples/kafka/otpvalidation/configs/PaymentConfirmationTimeExtractor.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.otpvalidation.configs; 2 | 3 | import guru.learningjournal.examples.kafka.otpvalidation.model.PaymentConfirmation; 4 | import lombok.extern.log4j.Log4j2; 5 | import org.apache.kafka.clients.consumer.ConsumerRecord; 6 | import org.apache.kafka.streams.processor.TimestampExtractor; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | 10 | @Configuration 11 | @Log4j2 12 | public class PaymentConfirmationTimeExtractor implements TimestampExtractor{ 13 | 14 | @Override 15 | public long extract(ConsumerRecord consumerRecord, long prevTime) { 16 | PaymentConfirmation confirmation = (PaymentConfirmation) consumerRecord.value(); 17 | return ((confirmation.getCreatedTime() > 0) ? confirmation.getCreatedTime() : prevTime); 18 | } 19 | 20 | @Bean 21 | public TimestampExtractor confirmationTimeExtractor() { 22 | return new PaymentConfirmationTimeExtractor(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /otpvalidation/src/main/java/guru/learningjournal/examples/kafka/otpvalidation/configs/PaymentRequestTimeExtractor.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.otpvalidation.configs; 2 | 3 | import guru.learningjournal.examples.kafka.otpvalidation.model.PaymentRequest; 4 | import lombok.extern.log4j.Log4j2; 5 | import org.apache.kafka.clients.consumer.ConsumerRecord; 6 | import org.apache.kafka.streams.processor.TimestampExtractor; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | 10 | @Configuration 11 | @Log4j2 12 | public class PaymentRequestTimeExtractor implements TimestampExtractor{ 13 | 14 | @Override 15 | public long extract(ConsumerRecord consumerRecord, long prevTime) { 16 | PaymentRequest request = (PaymentRequest) consumerRecord.value(); 17 | return ((request.getCreatedTime() > 0) ? request.getCreatedTime() : prevTime); 18 | } 19 | 20 | @Bean 21 | public TimestampExtractor requestTimeExtractor() { 22 | return new PaymentRequestTimeExtractor(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /otpvalidation/src/main/java/guru/learningjournal/examples/kafka/otpvalidation/model/PaymentConfirmation.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.otpvalidation.model; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import lombok.Data; 5 | 6 | @Data 7 | public class PaymentConfirmation { 8 | @JsonProperty("TransactionID") 9 | private String transactionID; 10 | @JsonProperty("CreatedTime") 11 | private Long createdTime; 12 | @JsonProperty("OTP") 13 | private String OTP; 14 | } 15 | -------------------------------------------------------------------------------- /otpvalidation/src/main/java/guru/learningjournal/examples/kafka/otpvalidation/model/PaymentRequest.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.otpvalidation.model; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import lombok.Data; 5 | 6 | @Data 7 | public class PaymentRequest { 8 | 9 | @JsonProperty("TransactionID") 10 | private String transactionID; 11 | @JsonProperty("CreatedTime") 12 | private Long createdTime; 13 | @JsonProperty("SourceAccountID") 14 | private String sourceAccountID; 15 | @JsonProperty("TargetAccountID") 16 | private String targetAccountID; 17 | @JsonProperty("Amount") 18 | private Double amount; 19 | @JsonProperty("OTP") 20 | private String OTP; 21 | } 22 | -------------------------------------------------------------------------------- /otpvalidation/src/main/java/guru/learningjournal/examples/kafka/otpvalidation/model/TransactionStatus.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.otpvalidation.model; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import lombok.Data; 5 | 6 | @Data 7 | public class TransactionStatus { 8 | @JsonProperty("TransactionID") 9 | private String transactionID; 10 | @JsonProperty("Status") 11 | private String status; 12 | } 13 | 14 | -------------------------------------------------------------------------------- /otpvalidation/src/main/java/guru/learningjournal/examples/kafka/otpvalidation/services/OTPValidationService.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.otpvalidation.services; 2 | 3 | import guru.learningjournal.examples.kafka.otpvalidation.bindings.OTPListenerBinding; 4 | import guru.learningjournal.examples.kafka.otpvalidation.model.PaymentConfirmation; 5 | import guru.learningjournal.examples.kafka.otpvalidation.model.PaymentRequest; 6 | import lombok.extern.log4j.Log4j2; 7 | import org.apache.kafka.common.serialization.Serdes; 8 | import org.apache.kafka.streams.kstream.JoinWindows; 9 | import org.apache.kafka.streams.kstream.KStream; 10 | import org.apache.kafka.streams.kstream.StreamJoined; 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.cloud.stream.annotation.EnableBinding; 13 | import org.springframework.cloud.stream.annotation.Input; 14 | import org.springframework.cloud.stream.annotation.StreamListener; 15 | import org.springframework.kafka.support.serializer.JsonSerde; 16 | import org.springframework.stereotype.Service; 17 | 18 | import java.time.Duration; 19 | import java.time.Instant; 20 | import java.time.ZoneOffset; 21 | 22 | 23 | @Log4j2 24 | @Service 25 | @EnableBinding(OTPListenerBinding.class) 26 | public class OTPValidationService { 27 | 28 | @Autowired 29 | private RecordBuilder recordBuilder; 30 | 31 | @StreamListener 32 | public void process(@Input("payment-request-channel") KStream request, 33 | @Input("payment-confirmation-channel") KStream confirmation) { 34 | 35 | request.foreach((k, v) -> log.info("Request Key = " + k + " Created Time = " 36 | + Instant.ofEpochMilli(v.getCreatedTime()).atOffset(ZoneOffset.UTC))); 37 | 38 | confirmation.foreach((k, v) -> log.info("Confirmation Key = " + k + " Created Time = " 39 | + Instant.ofEpochMilli(v.getCreatedTime()).atOffset(ZoneOffset.UTC))); 40 | 41 | 42 | request.join(confirmation, 43 | (r, c) -> recordBuilder.getTransactionStatus(r, c), 44 | JoinWindows.of(Duration.ofMinutes(5)), 45 | StreamJoined.with(Serdes.String(), 46 | new JsonSerde<>(PaymentRequest.class), 47 | new JsonSerde<>(PaymentConfirmation.class))) 48 | .foreach((k, v) -> log.info("Transaction ID = " + k + " Status = " + v.getStatus())); 49 | 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /otpvalidation/src/main/java/guru/learningjournal/examples/kafka/otpvalidation/services/RecordBuilder.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.otpvalidation.services; 2 | 3 | 4 | import guru.learningjournal.examples.kafka.otpvalidation.model.PaymentConfirmation; 5 | import guru.learningjournal.examples.kafka.otpvalidation.model.PaymentRequest; 6 | import guru.learningjournal.examples.kafka.otpvalidation.model.TransactionStatus; 7 | import lombok.extern.log4j.Log4j2; 8 | import org.springframework.stereotype.Service; 9 | 10 | @Service 11 | @Log4j2 12 | public class RecordBuilder { 13 | public TransactionStatus getTransactionStatus(PaymentRequest request, PaymentConfirmation confirmation){ 14 | String status = "Failure"; 15 | if(request.getOTP().equals(confirmation.getOTP())) 16 | status = "Success"; 17 | 18 | TransactionStatus transactionStatus = new TransactionStatus(); 19 | transactionStatus.setTransactionID(request.getTransactionID()); 20 | transactionStatus.setStatus(status); 21 | return transactionStatus; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /otpvalidation/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | payment-request-channel: 6 | destination: payment_request 7 | payment-confirmation-channel: 8 | destination: payment_confirmation 9 | kafka: 10 | streams: 11 | binder: 12 | brokers: localhost:9092 13 | configuration: 14 | commit.interval.ms: 10000 15 | state.dir: state-store 16 | default: 17 | key: 18 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 19 | value: 20 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 21 | bindings: 22 | payment-request-channel: 23 | consumer: 24 | timestampExtractorBeanName: requestTimeExtractor 25 | payment-confirmation-channel: 26 | consumer: 27 | timestampExtractorBeanName: confirmationTimeExtractor 28 | 29 | -------------------------------------------------------------------------------- /otpvalidation/src/test/java/guru/learningjournal/examples/kafka/otpvalidation/OtpValidationApplicationTests.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.otpvalidation; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class OtpValidationApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /rewards/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | netsh interface portproxy add v4tov4 listenport=8081 listenaddress=0.0.0.0 connectport=8081 connectaddress= 6 | 7 | confluent local services start 8 | 9 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic loyalty-topic 10 | 11 | kafka-avro-console-consumer --bootstrap-server localhost:9092 --topic loyalty-topic --from-beginning --property key.deserializer=org.apache.kafka.common.serialization.StringDeserializer --property print.key=true --property key.separator=":" 12 | 13 | confluent local destroy -------------------------------------------------------------------------------- /rewards/src/main/avro/DeliveryAddress.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "DeliveryAddress", 5 | "fields": [ 6 | {"name": "AddressLine","type": ["null","string"]}, 7 | {"name": "City","type": ["null","string"]}, 8 | {"name": "State","type": ["null","string"]}, 9 | {"name": "PinCode","type": ["null","string"]}, 10 | {"name": "ContactNumber","type": ["null","string"]} 11 | ] 12 | } -------------------------------------------------------------------------------- /rewards/src/main/avro/LineItem.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "LineItem", 5 | "fields": [ 6 | {"name": "ItemCode","type": ["null","string"]}, 7 | {"name": "ItemDescription","type": ["null","string"]}, 8 | {"name": "ItemPrice","type": ["null","double"]}, 9 | {"name": "ItemQty","type": ["null","int"]}, 10 | {"name": "TotalValue","type": ["null","double"]} 11 | ] 12 | } -------------------------------------------------------------------------------- /rewards/src/main/avro/Notification.avsc: -------------------------------------------------------------------------------- 1 | {"namespace": "guru.learningjournal.examples.kafka.model", 2 | "type": "record", 3 | "name": "Notification", 4 | "fields": [ 5 | {"name": "InvoiceNumber", "type": ["null", "string"]}, 6 | {"name": "CustomerCardNo", "type":["null", "string"]}, 7 | {"name": "TotalAmount", "type": ["null", "double"]}, 8 | {"name": "EarnedLoyaltyPoints", "type": ["null", "double"]}, 9 | {"name": "TotalLoyaltyPoints", "type": ["null", "double"]} 10 | ] 11 | } -------------------------------------------------------------------------------- /rewards/src/main/avro/PosInvoice.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "guru.learningjournal.examples.kafka.model", 3 | "type": "record", 4 | "name": "PosInvoice", 5 | "fields": [ 6 | {"name": "InvoiceNumber","type": ["null","string"]}, 7 | {"name": "CreatedTime","type": ["null","long"]}, 8 | {"name": "CustomerCardNo","type": ["null","string"]}, 9 | {"name": "TotalAmount","type": ["null","double"]}, 10 | {"name": "NumberOfItems","type": ["null","int"]}, 11 | {"name": "PaymentMethod","type": ["null","string"]}, 12 | {"name": "TaxableAmount","type": ["null","double"]}, 13 | {"name": "CGST","type": ["null","double"]}, 14 | {"name": "SGST","type": ["null","double"]}, 15 | {"name": "CESS","type": ["null","double"]}, 16 | {"name": "StoreID","type": ["null","string"]}, 17 | {"name": "PosID","type": ["null","string"]}, 18 | {"name": "CashierID","type": ["null","string"]}, 19 | {"name": "CustomerType","type": ["null","string"]}, 20 | {"name": "DeliveryType","type": ["null","string"]}, 21 | {"name": "DeliveryAddress","type": ["null","DeliveryAddress"]}, 22 | {"name": "InvoiceLineItems","type": {"type": "array", "items": "LineItem"}} 23 | ] 24 | } -------------------------------------------------------------------------------- /rewards/src/main/java/guru/learningjournal/examples/kafka/rewards/RewardsApplication.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.rewards; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class RewardsApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(RewardsApplication.class, args); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /rewards/src/main/java/guru/learningjournal/examples/kafka/rewards/bindings/PosListenerBinding.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.rewards.bindings; 2 | 3 | import guru.learningjournal.examples.kafka.model.Notification; 4 | import guru.learningjournal.examples.kafka.model.PosInvoice; 5 | import org.apache.kafka.streams.kstream.KStream; 6 | import org.springframework.cloud.stream.annotation.Input; 7 | import org.springframework.cloud.stream.annotation.Output; 8 | 9 | public interface PosListenerBinding { 10 | 11 | @Input("invoice-input-channel") 12 | KStream invoiceInputStream(); 13 | 14 | @Output("notification-output-channel") 15 | KStream notificationOutputStream(); 16 | 17 | } 18 | -------------------------------------------------------------------------------- /rewards/src/main/java/guru/learningjournal/examples/kafka/rewards/services/LoyaltyService.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.rewards.services; 2 | 3 | import guru.learningjournal.examples.kafka.model.Notification; 4 | import guru.learningjournal.examples.kafka.model.PosInvoice; 5 | import guru.learningjournal.examples.kafka.rewards.bindings.PosListenerBinding; 6 | import lombok.extern.log4j.Log4j2; 7 | import org.apache.kafka.streams.KeyValue; 8 | import org.apache.kafka.streams.kstream.KStream; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.cloud.stream.annotation.EnableBinding; 11 | import org.springframework.cloud.stream.annotation.StreamListener; 12 | import org.springframework.messaging.handler.annotation.SendTo; 13 | import org.springframework.stereotype.Service; 14 | 15 | @Log4j2 16 | @Service 17 | @EnableBinding(PosListenerBinding.class) 18 | public class LoyaltyService { 19 | 20 | @Autowired 21 | RecordBuilder recordBuilder; 22 | 23 | @StreamListener("invoice-input-channel") 24 | @SendTo("notification-output-channel") 25 | public KStream process(KStream input) { 26 | 27 | KStream notificationKStream = input 28 | .filter((k, v) -> v.getCustomerType().equalsIgnoreCase("PRIME")) 29 | .map((k, v) -> new KeyValue<>(v.getCustomerCardNo(), recordBuilder.getNotification(v))) 30 | .groupByKey() 31 | .reduce((aggValue, newValue) -> { 32 | newValue.setTotalLoyaltyPoints(newValue.getEarnedLoyaltyPoints() + aggValue.getTotalLoyaltyPoints()); 33 | return newValue; 34 | }).toStream(); 35 | 36 | notificationKStream.foreach((k, v) -> log.info(String.format("Notification:- Key: %s, Value: %s", k, v))); 37 | return notificationKStream; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /rewards/src/main/java/guru/learningjournal/examples/kafka/rewards/services/RecordBuilder.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.rewards.services; 2 | 3 | import guru.learningjournal.examples.kafka.model.Notification; 4 | import guru.learningjournal.examples.kafka.model.PosInvoice; 5 | import org.apache.kafka.common.serialization.Serdes; 6 | import org.springframework.stereotype.Service; 7 | 8 | @Service 9 | public class RecordBuilder { 10 | 11 | public Notification getNotification(PosInvoice invoice){ 12 | Notification notification = new Notification(); 13 | notification.setInvoiceNumber(invoice.getInvoiceNumber()); 14 | notification.setCustomerCardNo(invoice.getCustomerCardNo()); 15 | notification.setTotalAmount(invoice.getTotalAmount()); 16 | notification.setEarnedLoyaltyPoints(invoice.getTotalAmount() * 0.02); 17 | notification.setTotalLoyaltyPoints(notification.getEarnedLoyaltyPoints()); 18 | return notification; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /rewards/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | invoice-input-channel: 6 | destination: avro-pos-topic 7 | notification-output-channel: 8 | destination: loyalty-topic 9 | kafka: 10 | streams: 11 | binder: 12 | brokers: localhost:9092 13 | configuration: 14 | schema.registry.url: http://localhost:8081 15 | commit.interval.ms: 10000 16 | state.dir: state-store 17 | default: 18 | key: 19 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 20 | value: 21 | serde: io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde 22 | -------------------------------------------------------------------------------- /rewards/src/test/java/guru/learningjournal/examples/kafka/rewards/RewardsApplicationTests.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.rewards; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class RewardsApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /sessionwindow/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework.boot 7 | spring-boot-starter-parent 8 | 2.4.2 9 | 10 | 11 | guru.learningjournal.examples.kafka 12 | sessionwindow 13 | 0.0.1-SNAPSHOT 14 | Session Window 15 | Session Window Demo by Learning Journal 16 | 17 | 11 18 | Hoxton.SR9 19 | 20 | 21 | 22 | org.apache.kafka 23 | kafka-streams 24 | 25 | 26 | org.springframework.cloud 27 | spring-cloud-stream 28 | 29 | 30 | org.springframework.cloud 31 | spring-cloud-stream-binder-kafka-streams 32 | 33 | 34 | 35 | org.projectlombok 36 | lombok 37 | true 38 | 39 | 40 | org.springframework.boot 41 | spring-boot-starter-test 42 | test 43 | 44 | 45 | org.springframework.cloud 46 | spring-cloud-stream 47 | test 48 | test-binder 49 | test-jar 50 | 51 | 52 | 53 | 54 | 55 | org.springframework.cloud 56 | spring-cloud-dependencies 57 | ${spring-cloud.version} 58 | pom 59 | import 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | org.springframework.boot 68 | spring-boot-maven-plugin 69 | 70 | 71 | 72 | org.projectlombok 73 | lombok 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | spring-milestones 83 | Spring Milestones 84 | https://repo.spring.io/milestone 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /sessionwindow/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | confluent local services start 6 | 7 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic user-clicks-topic 8 | 9 | kafka-console-producer --broker-list localhost:9092 --topic user-clicks-topic \ 10 | --property parse.key=true --property key.separator=":" 11 | 12 | confluent local destroy -------------------------------------------------------------------------------- /sessionwindow/practice-resources/samples.txt: -------------------------------------------------------------------------------- 1 | USR101:{"UserID": "USR101","CreatedTime": "1549360860000","CurrentLink": "NULL", "NextLink": "Home"} 2 | USR102:{"UserID": "USR102","CreatedTime": "1549360920000","CurrentLink": "NULL", "NextLink": "Home"} 3 | USR101:{"UserID": "USR101","CreatedTime": "1549361040000","CurrentLink": "Home", "NextLink": "Books"} 4 | USR101:{"UserID": "USR101","CreatedTime": "1549361400000","CurrentLink": "Books", "NextLink": "Kafka"} 5 | USR102:{"UserID": "USR102","CreatedTime": "1549360920000","CurrentLink": "NULL", "NextLink": "Home"} 6 | USR102:{"UserID": "USR102","CreatedTime": "1549361400000","CurrentLink": "Home", "NextLink": "Courses"} 7 | USR101:{"UserID": "USR101","CreatedTime": "1549361220000","CurrentLink": "Kafka", "NextLink": "Preview"} 8 | USR101:{"UserID": "USR101","CreatedTime": "1549361940000","CurrentLink": "Preview", "NextLink": "Buy"} -------------------------------------------------------------------------------- /sessionwindow/src/main/java/guru/learningjournal/examples/kafka/sessionwindow/SessionWindowApplication.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.sessionwindow; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class SessionWindowApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(SessionWindowApplication.class, args); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /sessionwindow/src/main/java/guru/learningjournal/examples/kafka/sessionwindow/bindings/ClickListenerBinding.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.sessionwindow.bindings; 2 | 3 | import guru.learningjournal.examples.kafka.sessionwindow.models.UserClick; 4 | import org.apache.kafka.streams.kstream.KStream; 5 | import org.springframework.cloud.stream.annotation.Input; 6 | 7 | public interface ClickListenerBinding { 8 | 9 | @Input("click-input-channel") 10 | KStream clickInputStream(); 11 | 12 | } 13 | -------------------------------------------------------------------------------- /sessionwindow/src/main/java/guru/learningjournal/examples/kafka/sessionwindow/configs/ClickTimeExtractor.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.sessionwindow.configs; 2 | 3 | import guru.learningjournal.examples.kafka.sessionwindow.models.UserClick; 4 | import lombok.extern.log4j.Log4j2; 5 | import org.apache.kafka.clients.consumer.ConsumerRecord; 6 | import org.apache.kafka.streams.processor.TimestampExtractor; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | 10 | @Configuration 11 | @Log4j2 12 | public class ClickTimeExtractor implements TimestampExtractor{ 13 | 14 | @Override 15 | public long extract(ConsumerRecord consumerRecord, long prevTime) { 16 | UserClick click = (UserClick) consumerRecord.value(); 17 | log.info("Click Time: {}", click.getCreatedTime()); 18 | return ((click.getCreatedTime() > 0) ? click.getCreatedTime() : prevTime); 19 | } 20 | 21 | @Bean 22 | public TimestampExtractor userClickTimeExtractor() { 23 | return new ClickTimeExtractor(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /sessionwindow/src/main/java/guru/learningjournal/examples/kafka/sessionwindow/models/UserClick.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.sessionwindow.models; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import lombok.Data; 5 | 6 | @Data 7 | public class UserClick { 8 | @JsonProperty("UserID") 9 | private String userID; 10 | @JsonProperty("CreatedTime") 11 | private Long createdTime; 12 | @JsonProperty("CurrentLink") 13 | private String currentLink; 14 | @JsonProperty("NextLink") 15 | private String nextLink; 16 | } 17 | -------------------------------------------------------------------------------- /sessionwindow/src/main/java/guru/learningjournal/examples/kafka/sessionwindow/services/ClickListerService.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.sessionwindow.services; 2 | 3 | import guru.learningjournal.examples.kafka.sessionwindow.bindings.ClickListenerBinding; 4 | import guru.learningjournal.examples.kafka.sessionwindow.models.UserClick; 5 | import lombok.extern.log4j.Log4j2; 6 | import org.apache.kafka.streams.kstream.KStream; 7 | import org.apache.kafka.streams.kstream.SessionWindows; 8 | import org.springframework.cloud.stream.annotation.EnableBinding; 9 | import org.springframework.cloud.stream.annotation.StreamListener; 10 | import org.springframework.stereotype.Service; 11 | 12 | import java.time.Duration; 13 | import java.time.Instant; 14 | import java.time.ZoneOffset; 15 | 16 | @Log4j2 17 | @Service 18 | @EnableBinding(ClickListenerBinding.class) 19 | public class ClickListerService { 20 | 21 | @StreamListener("click-input-channel") 22 | public void process(KStream input) { 23 | 24 | input.peek((k, v) -> log.info("Key = " + k + " Created Time = " 25 | + Instant.ofEpochMilli(v.getCreatedTime()).atOffset(ZoneOffset.UTC))) 26 | .groupByKey() 27 | .windowedBy(SessionWindows.with(Duration.ofMinutes(5))) 28 | .count() 29 | .toStream() 30 | .foreach((k, v) -> log.info( 31 | "UserID: " + k.key() + 32 | " Window start: " + 33 | Instant.ofEpochMilli(k.window().start()) 34 | .atOffset(ZoneOffset.UTC) + 35 | " Window end: " + 36 | Instant.ofEpochMilli(k.window().end()) 37 | .atOffset(ZoneOffset.UTC) + 38 | " Count: " + v + 39 | " Window#: " + k.window().hashCode() 40 | )); 41 | 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /sessionwindow/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | click-input-channel: 6 | destination: user-clicks-topic 7 | kafka: 8 | streams: 9 | binder: 10 | brokers: localhost:9092 11 | configuration: 12 | commit.interval.ms: 10000 13 | state.dir: state-store 14 | default: 15 | key: 16 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 17 | value: 18 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 19 | bindings: 20 | click-input-channel: 21 | consumer: 22 | timestampExtractorBeanName: userClickTimeExtractor 23 | 24 | -------------------------------------------------------------------------------- /sessionwindow/src/test/java/guru/learningjournal/examples/kafka/sessionwindow/SessionWindowApplicationTests.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.sessionwindow; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class SessionWindowApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /simpletest/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | confluent local services start 6 | 7 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic input-topic 8 | 9 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic output-topic 10 | 11 | kafka-console-producer --broker-list localhost:9092 --topic input-topic 12 | 13 | kafka-console-consumer --topic output-topic --from-beginning --bootstrap-server localhost:9092 14 | 15 | confluent local destroy -------------------------------------------------------------------------------- /simpletest/practice-resources/samples.txt: -------------------------------------------------------------------------------- 1 | test string 1 2 | test string 2 3 | -------------------------------------------------------------------------------- /simpletest/src/main/java/guru/learningjournal/examples/kafka/simpletest/SimpleTestApplication.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.simpletest; 2 | 3 | import org.apache.kafka.common.utils.Bytes; 4 | import org.apache.kafka.streams.kstream.KStream; 5 | import org.springframework.boot.SpringApplication; 6 | import org.springframework.boot.autoconfigure.SpringBootApplication; 7 | import org.springframework.context.annotation.Bean; 8 | 9 | import java.util.function.Function; 10 | 11 | @SpringBootApplication 12 | public class SimpleTestApplication { 13 | 14 | public static void main(String[] args) { 15 | SpringApplication.run(SimpleTestApplication.class, args); 16 | } 17 | 18 | 19 | } 20 | -------------------------------------------------------------------------------- /simpletest/src/main/java/guru/learningjournal/examples/kafka/simpletest/bindings/ListenerBinding.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.simpletest.bindings; 2 | 3 | import org.apache.kafka.streams.kstream.KStream; 4 | import org.springframework.cloud.stream.annotation.Input; 5 | import org.springframework.cloud.stream.annotation.Output; 6 | 7 | public interface ListenerBinding { 8 | 9 | @Input("process-in-0") 10 | KStream inputStream(); 11 | 12 | @Output("process-out-0") 13 | KStream outStream(); 14 | 15 | } 16 | -------------------------------------------------------------------------------- /simpletest/src/main/java/guru/learningjournal/examples/kafka/simpletest/services/ListenerService.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.simpletest.services; 2 | 3 | import guru.learningjournal.examples.kafka.simpletest.bindings.ListenerBinding; 4 | import lombok.extern.log4j.Log4j2; 5 | import org.apache.kafka.streams.kstream.KStream; 6 | import org.springframework.cloud.stream.annotation.EnableBinding; 7 | import org.springframework.cloud.stream.annotation.StreamListener; 8 | import org.springframework.messaging.handler.annotation.SendTo; 9 | import org.springframework.stereotype.Service; 10 | 11 | @Log4j2 12 | @Service 13 | @EnableBinding(ListenerBinding.class) 14 | public class ListenerService { 15 | 16 | @StreamListener("process-in-0") 17 | @SendTo("process-out-0") 18 | public KStream process(KStream input) { 19 | 20 | input.foreach((k,v) -> log.info("Received Input: {}",v)); 21 | return input.mapValues(v -> v.toUpperCase()); 22 | 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /simpletest/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | process-in-0: 6 | destination: input-topic 7 | process-out-0: 8 | destination: output-topic 9 | kafka: 10 | streams: 11 | binder: 12 | brokers: localhost:9092 13 | configuration: 14 | commit.interval.ms: 100 15 | default: 16 | key.serde: org.apache.kafka.common.serialization.Serdes$StringSerde 17 | value.serde: org.apache.kafka.common.serialization.Serdes$StringSerde 18 | -------------------------------------------------------------------------------- /simpletest/src/test/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | process-in-0: 6 | destination: input-topic 7 | process-out-0: 8 | destination: output-topic 9 | kafka: 10 | streams: 11 | binder: 12 | brokers: ${spring.embedded.kafka.brokers} 13 | configuration: 14 | commit.interval.ms: 100 15 | default: 16 | key.serde: org.apache.kafka.common.serialization.Serdes$StringSerde 17 | value.serde: org.apache.kafka.common.serialization.Serdes$StringSerde 18 | -------------------------------------------------------------------------------- /streamingaggregates/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'org.springframework.boot' version '2.4.1' 3 | id 'io.spring.dependency-management' version '1.0.10.RELEASE' 4 | id 'java' 5 | } 6 | 7 | group = 'guru.learningjournal.examples.kafka' 8 | version = '0.0.1-SNAPSHOT' 9 | sourceCompatibility = '11' 10 | 11 | configurations { 12 | compileOnly { 13 | extendsFrom annotationProcessor 14 | } 15 | } 16 | 17 | repositories { 18 | mavenCentral() 19 | maven { url 'https://repo.spring.io/milestone' } 20 | } 21 | 22 | ext { 23 | set('springCloudVersion', "Hoxton.SR9") 24 | } 25 | 26 | dependencies { 27 | implementation 'org.apache.kafka:kafka-streams' 28 | implementation 'org.springframework.cloud:spring-cloud-stream' 29 | implementation 'org.springframework.cloud:spring-cloud-stream-binder-kafka-streams' 30 | compileOnly 'org.projectlombok:lombok' 31 | annotationProcessor 'org.projectlombok:lombok' 32 | testImplementation 'org.springframework.boot:spring-boot-starter-test' 33 | } 34 | 35 | dependencyManagement { 36 | imports { 37 | mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}" 38 | } 39 | } 40 | 41 | test { 42 | useJUnitPlatform() 43 | } 44 | -------------------------------------------------------------------------------- /streamingaggregates/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Kafka-Streams-with-Spring-Cloud-Stream/aa6319b0454246240d0610994d06900b3ef73c9d/streamingaggregates/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /streamingaggregates/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /streamingaggregates/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | confluent local services start 6 | 7 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic streaming-words-topic 8 | 9 | kafka-console-producer --topic streaming-words-topic --broker-list localhost:9092 10 | 11 | confluent local services stop 12 | confluent local destroy -------------------------------------------------------------------------------- /streamingaggregates/practice-resources/sample.txt: -------------------------------------------------------------------------------- 1 | Hello World 2 | Hello Kafka 3 | Hello Kafka Streams 4 | -------------------------------------------------------------------------------- /streamingaggregates/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'streamingaggregates' 2 | -------------------------------------------------------------------------------- /streamingaggregates/src/main/java/guru/learningjournal/examples/kafka/streamingaggregates/StreamingAggregatesApplication.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.streamingaggregates; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class StreamingAggregatesApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(StreamingAggregatesApplication.class, args); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /streamingaggregates/src/main/java/guru/learningjournal/examples/kafka/streamingaggregates/bindings/WordListenerBinding.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.streamingaggregates.bindings; 2 | 3 | 4 | import org.apache.kafka.streams.kstream.KStream; 5 | import org.springframework.cloud.stream.annotation.Input; 6 | 7 | public interface WordListenerBinding { 8 | 9 | @Input("words-input-channel") 10 | KStream wordsInputStream(); 11 | 12 | } 13 | -------------------------------------------------------------------------------- /streamingaggregates/src/main/java/guru/learningjournal/examples/kafka/streamingaggregates/services/WordListenerService.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.streamingaggregates.services; 2 | 3 | 4 | import guru.learningjournal.examples.kafka.streamingaggregates.bindings.WordListenerBinding; 5 | import lombok.extern.log4j.Log4j2; 6 | import org.apache.kafka.streams.kstream.KStream; 7 | import org.springframework.cloud.stream.annotation.EnableBinding; 8 | import org.springframework.cloud.stream.annotation.StreamListener; 9 | import org.springframework.stereotype.Service; 10 | 11 | import java.util.Arrays; 12 | 13 | @Service 14 | @Log4j2 15 | @EnableBinding(WordListenerBinding.class) 16 | public class WordListenerService { 17 | 18 | @StreamListener("words-input-channel") 19 | public void process(KStream input) { 20 | 21 | KStream wordStream = input 22 | .flatMapValues(value -> Arrays.asList(value.toLowerCase().split(" "))); 23 | 24 | wordStream.groupBy((key, value) -> value) 25 | .count() 26 | .toStream() 27 | .peek((k, v) -> log.info("Word: {} Count: {}", k, v)); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /streamingaggregates/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | words-input-channel: 6 | destination: streaming-words-topic 7 | kafka: 8 | streams: 9 | binder: 10 | brokers: localhost:9092 11 | configuration: 12 | commit.interval.ms: 10000 13 | state.dir: state-store 14 | default: 15 | key: 16 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 17 | value: 18 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 19 | 20 | 21 | -------------------------------------------------------------------------------- /streamingaggregates/src/test/java/guru/learningjournal/examples/kafka/streamingaggregates/StreamingAggregatesApplicationTests.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.streamingaggregates; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class StreamingAggregatesApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /streamingtest/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | confluent local services start 6 | 7 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic input-topic 8 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic output-topic 9 | 10 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic process-in-0 11 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic process-out-0 12 | 13 | kafka-console-producer --broker-list localhost:9092 --topic process-in-0 14 | kafka-console-consumer --topic process-out-0 --from-beginning --bootstrap-server localhost:9092 15 | 16 | kafka-console-producer --broker-list localhost:9092 --topic input-topic 17 | kafka-console-consumer --topic output-topic --from-beginning --bootstrap-server localhost:9092 18 | 19 | 20 | confluent local destroy -------------------------------------------------------------------------------- /streamingtest/practice-resources/samples.txt: -------------------------------------------------------------------------------- 1 | test string 1 2 | test string 2 3 | -------------------------------------------------------------------------------- /streamingtest/src/main/java/guru/learningjournal/examples/kafka/streamingtest/StreamingTestApplication.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.streamingtest; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class StreamingTestApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(StreamingTestApplication.class, args); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /streamingtest/src/main/java/guru/learningjournal/examples/kafka/streamingtest/configs/ListenerService.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.streamingtest.configs; 2 | 3 | import org.apache.kafka.streams.kstream.KStream; 4 | import org.springframework.boot.autoconfigure.EnableAutoConfiguration; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | 8 | import java.util.function.Function; 9 | 10 | @Configuration 11 | @EnableAutoConfiguration 12 | public class ListenerService { 13 | 14 | @Bean 15 | public Function, KStream> process() { 16 | 17 | return input -> input.mapValues(i -> i.toUpperCase()); 18 | 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /streamingtest/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | function: 5 | definition: process 6 | bindings: 7 | process-in-0: 8 | destination: input-topic 9 | process-out-0: 10 | destination: output-topic 11 | kafka: 12 | streams: 13 | binder: 14 | brokers: localhost:9092 15 | configuration: 16 | commit.interval.ms: 100 17 | default: 18 | key.serde: org.apache.kafka.common.serialization.Serdes$StringSerde 19 | value.serde: org.apache.kafka.common.serialization.Serdes$StringSerde 20 | -------------------------------------------------------------------------------- /streamingtest/src/test/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | process-in-0: 6 | destination: input-topic 7 | process-out-0: 8 | destination: output-topic 9 | kafka: 10 | streams: 11 | binder: 12 | brokers: ${spring.embedded.kafka.brokers} 13 | configuration: 14 | commit.interval.ms: 100 15 | default: 16 | key.serde: org.apache.kafka.common.serialization.Serdes$StringSerde 17 | value.serde: org.apache.kafka.common.serialization.Serdes$StringSerde 18 | -------------------------------------------------------------------------------- /top3spots/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework.boot 7 | spring-boot-starter-parent 8 | 2.4.2 9 | 10 | 11 | guru.learningjournal.examples.kafka 12 | top3spots 13 | 0.0.1-SNAPSHOT 14 | Top3 Spots 15 | Top3 Spots Demo by Learning Journal 16 | 17 | 11 18 | Hoxton.SR9 19 | 20 | 21 | 22 | org.apache.kafka 23 | kafka-streams 24 | 25 | 26 | org.springframework.cloud 27 | spring-cloud-stream 28 | 29 | 30 | org.springframework.cloud 31 | spring-cloud-stream-binder-kafka-streams 32 | 33 | 34 | 35 | org.projectlombok 36 | lombok 37 | true 38 | 39 | 40 | 41 | org.springframework.boot 42 | spring-boot-starter-test 43 | test 44 | 45 | 46 | org.springframework.cloud 47 | spring-cloud-stream 48 | test 49 | test-binder 50 | test-jar 51 | 52 | 53 | 54 | 55 | 56 | 57 | org.springframework.cloud 58 | spring-cloud-dependencies 59 | ${spring-cloud.version} 60 | pom 61 | import 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | org.springframework.boot 70 | spring-boot-maven-plugin 71 | 72 | 73 | 74 | org.projectlombok 75 | lombok 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | spring-milestones 85 | Spring Milestones 86 | https://repo.spring.io/milestone 87 | 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /top3spots/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | confluent local services start 6 | 7 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic active-inventories 8 | 9 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic ad-clicks 10 | 11 | kafka-console-producer --broker-list localhost:9092 --topic active-inventories \ 12 | --property parse.key=true --property key.separator=":" 13 | 14 | kafka-console-producer --broker-list localhost:9092 --topic ad-clicks \ 15 | --property parse.key=true --property key.separator=":" 16 | 17 | confluent local destroy -------------------------------------------------------------------------------- /top3spots/practice-resources/samples.txt: -------------------------------------------------------------------------------- 1 | 1001:{"InventoryID": "1001", "NewsType": "Sports"} 2 | 1002:{"InventoryID": "1002", "NewsType": "Politics"} 3 | 1003:{"InventoryID": "1003", "NewsType": "LocalNews"} 4 | 1004:{"InventoryID": "1004", "NewsType": "WorldNews"} 5 | 1005:{"InventoryID": "1005", "NewsType": "Health"} 6 | 1006:{"InventoryID": "1006", "NewsType": "Lifestyle"} 7 | 1007:{"InventoryID": "1007", "NewsType": "Literature"} 8 | 1008:{"InventoryID": "1008", "NewsType": "Education"} 9 | 1009:{"InventoryID": "1009", "NewsType": "Social"} 10 | 1010:{"InventoryID": "1010", "NewsType": "Business"} 11 | 12 | 13 | 1001:{"InventoryID": "1001"} 14 | 1002:{"InventoryID": "1002"} 15 | 1003:{"InventoryID": "1003"} 16 | 1004:{"InventoryID": "1004"} 17 | 1004:{"InventoryID": "1004"} 18 | 1002:{"InventoryID": "1002"} 19 | 20 | -------------------------------------------------------------------------------- /top3spots/src/main/java/guru/learningjournal/examples/kafka/top3spots/Top3SpotsApplication.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.top3spots; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class Top3SpotsApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(Top3SpotsApplication.class, args); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /top3spots/src/main/java/guru/learningjournal/examples/kafka/top3spots/bindings/ClicksListenerBinding.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.top3spots.bindings; 2 | 3 | import guru.learningjournal.examples.kafka.top3spots.models.AdClick; 4 | import guru.learningjournal.examples.kafka.top3spots.models.AdInventories; 5 | import org.apache.kafka.streams.kstream.GlobalKTable; 6 | import org.apache.kafka.streams.kstream.KStream; 7 | import org.springframework.cloud.stream.annotation.Input; 8 | 9 | public interface ClicksListenerBinding { 10 | 11 | @Input("inventories-channel") 12 | GlobalKTable inventoryInputStream(); 13 | 14 | @Input("clicks-channel") 15 | KStream clickInputStream(); 16 | 17 | } 18 | -------------------------------------------------------------------------------- /top3spots/src/main/java/guru/learningjournal/examples/kafka/top3spots/models/AdClick.java: -------------------------------------------------------------------------------- 1 | 2 | package guru.learningjournal.examples.kafka.top3spots.models; 3 | 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import lombok.Data; 6 | 7 | @Data 8 | public class AdClick { 9 | 10 | @JsonProperty("InventoryID") 11 | private String inventoryID; 12 | 13 | } 14 | -------------------------------------------------------------------------------- /top3spots/src/main/java/guru/learningjournal/examples/kafka/top3spots/models/AdInventories.java: -------------------------------------------------------------------------------- 1 | 2 | package guru.learningjournal.examples.kafka.top3spots.models; 3 | 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import lombok.Data; 6 | 7 | @Data 8 | public class AdInventories { 9 | 10 | @JsonProperty("InventoryID") 11 | private String inventoryID; 12 | @JsonProperty("NewsType") 13 | private String newsType; 14 | 15 | } 16 | -------------------------------------------------------------------------------- /top3spots/src/main/java/guru/learningjournal/examples/kafka/top3spots/models/ClicksByNewsType.java: -------------------------------------------------------------------------------- 1 | 2 | package guru.learningjournal.examples.kafka.top3spots.models; 3 | 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import lombok.Data; 6 | 7 | @Data 8 | public class ClicksByNewsType { 9 | 10 | @JsonProperty("NewsType") 11 | private String newsType; 12 | @JsonProperty("Clicks") 13 | private Long clicks; 14 | } 15 | -------------------------------------------------------------------------------- /top3spots/src/main/java/guru/learningjournal/examples/kafka/top3spots/models/Top3NewsTypes.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.top3spots.models; 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import com.fasterxml.jackson.annotation.JsonPropertyOrder; 6 | import com.fasterxml.jackson.core.JsonProcessingException; 7 | import com.fasterxml.jackson.databind.ObjectMapper; 8 | 9 | import java.io.IOException; 10 | import java.util.TreeSet; 11 | 12 | @JsonInclude(JsonInclude.Include.NON_NULL) 13 | @JsonPropertyOrder({ 14 | "top3Sorted" 15 | }) 16 | public class Top3NewsTypes { 17 | 18 | private ObjectMapper mapper = new ObjectMapper(); 19 | 20 | private final TreeSet top3Sorted = new TreeSet<>((o1, o2) -> { 21 | final int result = o2.getClicks().compareTo(o1.getClicks()); 22 | if (result != 0) 23 | return result; 24 | else 25 | return o1.getNewsType().compareTo(o2.getNewsType()); 26 | }); 27 | 28 | public void add(ClicksByNewsType newValue) { 29 | top3Sorted.add(newValue); 30 | if (top3Sorted.size() > 3) { 31 | top3Sorted.remove(top3Sorted.last()); 32 | } 33 | } 34 | 35 | public void remove(ClicksByNewsType oldValue){ 36 | top3Sorted.remove(oldValue); 37 | } 38 | 39 | @JsonProperty("top3Sorted") 40 | public String getTop3Sorted() throws JsonProcessingException { 41 | return mapper.writeValueAsString(top3Sorted); 42 | } 43 | 44 | @JsonProperty("top3Sorted") 45 | public void setTop3Sorted(String top3String) throws IOException { 46 | ClicksByNewsType[] top3 = mapper.readValue(top3String, ClicksByNewsType[].class); 47 | for (ClicksByNewsType i:top3){ 48 | add(i); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /top3spots/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | cloud: 3 | stream: 4 | bindings: 5 | inventories-channel: 6 | destination: active-inventories 7 | clicks-channel: 8 | destination: ad-clicks 9 | kafka: 10 | streams: 11 | binder: 12 | brokers: localhost:9092 13 | configuration: 14 | commit.interval.ms: 10000 15 | state.dir: state-store 16 | default: 17 | key: 18 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 19 | value: 20 | serde: org.apache.kafka.common.serialization.Serdes$StringSerde 21 | 22 | -------------------------------------------------------------------------------- /top3spots/src/test/java/guru/learningjournal/examples/kafka/top3spots/Top3SpotsApplicationTests.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.top3spots; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class Top3SpotsApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /windowcount/practice-resources/kafka-scripts.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | netsh interface portproxy add v4tov4 listenport=9092 listenaddress=0.0.0.0 connectport=9092 connectaddress= 4 | 5 | confluent local services start 6 | 7 | kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic simple-invoice-topic 8 | 9 | kafka-console-producer --broker-list localhost:9092 --topic simple-invoice-topic \ 10 | --property parse.key=true --property key.separator=":" 11 | 12 | confluent local destroy -------------------------------------------------------------------------------- /windowcount/practice-resources/samples.txt: -------------------------------------------------------------------------------- 1 | STR1534:{"InvoiceNumber": 101,"CreatedTime": "1549360860000","StoreID": "STR1534", "TotalAmount": 1920} 2 | STR1535:{"InvoiceNumber": 102,"CreatedTime": "1549360900000","StoreID": "STR1535", "TotalAmount": 1860} 3 | STR1534:{"InvoiceNumber": 103,"CreatedTime": "1549360999000","StoreID": "STR1534", "TotalAmount": 2400} 4 | 5 | STR1536:{"InvoiceNumber": 104,"CreatedTime": "1549361160000","StoreID": "STR1536", "TotalAmount": 8936} 6 | STR1534:{"InvoiceNumber": 105,"CreatedTime": "1549361270000","StoreID": "STR1534", "TotalAmount": 6375} 7 | STR1536:{"InvoiceNumber": 106,"CreatedTime": "1549361370000","StoreID": "STR1536", "TotalAmount": 9365} -------------------------------------------------------------------------------- /windowcount/src/main/java/guru/learningjournal/examples/kafka/windowcount/WindowCountApplication.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.windowcount; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class WindowCountApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(WindowCountApplication.class, args); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /windowcount/src/main/java/guru/learningjournal/examples/kafka/windowcount/bindings/InvoiceListenerBinding.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.windowcount.bindings; 2 | 3 | import guru.learningjournal.examples.kafka.windowcount.model.SimpleInvoice; 4 | import org.apache.kafka.streams.kstream.KStream; 5 | import org.springframework.cloud.stream.annotation.Input; 6 | 7 | public interface InvoiceListenerBinding { 8 | 9 | @Input("invoice-input-channel") 10 | KStream invoiceInputStream(); 11 | 12 | } 13 | -------------------------------------------------------------------------------- /windowcount/src/main/java/guru/learningjournal/examples/kafka/windowcount/configs/InvoiceTimeExtractor.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.windowcount.configs; 2 | 3 | import guru.learningjournal.examples.kafka.windowcount.model.SimpleInvoice; 4 | import org.apache.kafka.clients.consumer.ConsumerRecord; 5 | import org.apache.kafka.streams.processor.TimestampExtractor; 6 | import org.springframework.context.annotation.Bean; 7 | import org.springframework.context.annotation.Configuration; 8 | 9 | @Configuration 10 | public class InvoiceTimeExtractor implements TimestampExtractor{ 11 | 12 | @Override 13 | public long extract(ConsumerRecord consumerRecord, long prevTime) { 14 | SimpleInvoice invoice = (SimpleInvoice) consumerRecord.value(); 15 | return ((invoice.getCreatedTime() > 0) ? invoice.getCreatedTime() : prevTime); 16 | } 17 | 18 | @Bean 19 | public TimestampExtractor invoiceTimesExtractor() { 20 | return new InvoiceTimeExtractor(); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /windowcount/src/main/java/guru/learningjournal/examples/kafka/windowcount/model/SimpleInvoice.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.windowcount.model; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import lombok.Data; 5 | 6 | @Data 7 | public class SimpleInvoice { 8 | @JsonProperty("InvoiceNumber") 9 | private String invoiceNumber; 10 | @JsonProperty("CreatedTime") 11 | private Long createdTime; 12 | @JsonProperty("StoreID") 13 | private String storeID; 14 | @JsonProperty("TotalAmount") 15 | private Double totalAmount; 16 | } 17 | -------------------------------------------------------------------------------- /windowcount/src/main/java/guru/learningjournal/examples/kafka/windowcount/services/InvoiceListenerService.java: -------------------------------------------------------------------------------- 1 | package guru.learningjournal.examples.kafka.windowcount.services; 2 | 3 | import guru.learningjournal.examples.kafka.windowcount.bindings.InvoiceListenerBinding; 4 | import guru.learningjournal.examples.kafka.windowcount.model.SimpleInvoice; 5 | import lombok.extern.log4j.Log4j2; 6 | import org.apache.kafka.streams.kstream.KStream; 7 | import org.apache.kafka.streams.kstream.TimeWindows; 8 | import org.apache.kafka.streams.kstream.Window; 9 | import org.apache.kafka.streams.kstream.Windows; 10 | import org.springframework.cloud.stream.annotation.EnableBinding; 11 | import org.springframework.cloud.stream.annotation.StreamListener; 12 | import org.springframework.stereotype.Service; 13 | 14 | import java.time.Duration; 15 | import java.time.Instant; 16 | import java.time.ZoneOffset; 17 | 18 | @Log4j2 19 | @Service 20 | @EnableBinding(InvoiceListenerBinding.class) 21 | public class InvoiceListenerService { 22 | 23 | @StreamListener("invoice-input-channel") 24 | public void process(KStream input) { 25 | 26 | input.peek((k, v) -> log.info("Key = " + k + " Created Time = " 27 | + Instant.ofEpochMilli(v.getCreatedTime()).atOffset(ZoneOffset.UTC))) 28 | .groupByKey() 29 | .windowedBy(TimeWindows.of(Duration.ofMinutes(5))) 30 | .count() 31 | .toStream() 32 | .foreach((k, v) -> log.info( 33 | "StoreID: " + k.key() + 34 | " Window start: " + 35 | Instant.ofEpochMilli(k.window().start()) 36 | .atOffset(ZoneOffset.UTC) + 37 | " Window end: " + 38 | Instant.ofEpochMilli(k.window().end()) 39 | .atOffset(ZoneOffset.UTC) + 40 | " Count: " + v + 41 | " Window#: " + k.window().hashCode() 42 | )); 43 | 44 | } 45 | } 46 | --------------------------------------------------------------------------------