├── .gitignore ├── Figures ├── Figure-11.1.jpg ├── Figure-11.2.jpg ├── Figure-11.3.jpg ├── Figure-13.1.jpg ├── Figure-13.2.jpg ├── Figure-13.3.jpg ├── Figure-13.4.jpg ├── Figure-13.5.jpg ├── Figure-13.6.jpg ├── Figure-14.1.jpg ├── Figure-14.10.jpg ├── Figure-14.11.jpg ├── Figure-14.12.jpg ├── Figure-14.13.jpg ├── Figure-14.14.jpg ├── Figure-14.2.jpg ├── Figure-14.3.jpg ├── Figure-14.4.jpg ├── Figure-14.5.jpg ├── Figure-14.6.jpg ├── Figure-14.7.jpg ├── Figure-14.8.jpg ├── Figure-14.9.jpg ├── Figure-15.1.jpg ├── Figure-15.2.jpg ├── Figure-15.3.jpg ├── Figure-17.1.jpg ├── Figure-17.2.jpg ├── Figure-17.3.jpg ├── Figure-17.4.jpg ├── Figure-18.1.jpg ├── Figure-19.1.jpg ├── Figure-19.2.jpg ├── Figure-19.3.jpg ├── Figure-19.4.jpg ├── Figure-19.5.jpg ├── Figure-2.1.jpg ├── Figure-2.2.jpg ├── Figure-2.3.jpg ├── Figure-20.1.jpg ├── Figure-20.2.jpg ├── Figure-3.1.jpg ├── Figure-3.2.jpg ├── Figure-3.3.jpg ├── Figure-3.4.jpg ├── Figure-3.5.jpg ├── Figure-3.6.jpg ├── Figure-3.7.jpg ├── Figure-4.1.jpg ├── Figure-4.2.jpg ├── Figure-4.3.jpg ├── Figure-4.4.jpg ├── Figure-4.5.jpg ├── Figure-4.6.jpg ├── Figure-4.7.jpg ├── Figure-5.1.jpg ├── Figure-6.1.jpg ├── Figure-6.10.jpg ├── Figure-6.2.jpg ├── Figure-6.3.jpg ├── Figure-6.4.jpg ├── Figure-6.5.jpg ├── Figure-6.6.jpg ├── Figure-6.7.jpg ├── Figure-6.8.jpg ├── Figure-6.9.jpg ├── Figure-7.1.jpg ├── Figure-7.2.jpg ├── Figure-7.3.jpg ├── Figure-7.4.jpg ├── Figure-7.5.jpg ├── Figure-7.6.jpg ├── Figure-A.1.jpg ├── Figure-A.2.jpg ├── Figure-A.3.jpg ├── Figure-A.4.jpg ├── Figure-A.5.jpg ├── Figure-A.6.jpg └── Figure-A.7.jpg ├── LICENSE ├── README.md ├── ad-campaigns ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ ├── start-producer_ad-clicks.cmd │ ├── start-producer_ad-impressions.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── CampaignPerformanceApp.java │ │ ├── common │ │ ├── AppConfigs.java │ │ └── AppSerdes.java │ │ ├── serde │ │ ├── JsonDeserializer.java │ │ └── JsonSerializer.java │ │ └── types │ │ ├── AdClick.java │ │ ├── AdImpression.java │ │ └── CampaignPerformance.java │ └── resources │ ├── data │ └── sample.txt │ ├── log4j2.xml │ └── schema │ ├── AdClick.json │ ├── AdImpression.json │ └── CampaignPerformance.json ├── age-count ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ ├── start-producer.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── AgeCountDemo.java │ │ └── AppConfigs.java │ └── resources │ ├── data │ └── data.txt │ └── log4j2.xml ├── avro-consumer ├── README.md ├── pom.xml ├── scripts │ ├── start-kafka-server.cmd │ ├── start-schema-registry.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── AvroConsumer.java │ │ └── StockData.java │ └── resources │ ├── kafka.properties │ ├── log4j2.xml │ └── schema │ └── srock-data.avsc ├── avro-producer ├── README.md ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-consumer.cmd │ ├── start-kafka-server.cmd │ ├── start-schema-registry.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── AvroProducerDemo.java │ │ ├── Dispatcher.java │ │ └── StockData.java │ └── resources │ ├── data │ ├── NSE05NOV2018BHAV.csv │ └── NSE06NOV2018BHAV.csv │ ├── kafka.properties │ ├── log4j2.xml │ └── schema │ └── srock-data.avsc ├── callback-producer ├── README.md ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-consumer.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ └── CallbackProducer.java │ └── resources │ └── log4j2.xml ├── counting-session ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ ├── start-producer.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── CountingSessionApp.java │ │ ├── common │ │ ├── AppConfigs.java │ │ ├── AppSerdes.java │ │ └── AppTimestampExtractor.java │ │ ├── serde │ │ ├── JsonDeserializer.java │ │ └── JsonSerializer.java │ │ └── types │ │ └── UserClicks.java │ └── resources │ ├── data │ └── sample-data.txt │ ├── log4j2.xml │ └── schema │ └── UserClicks.json ├── counting-window ├── README.md ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ ├── start-producer.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── AppConfigs.java │ │ ├── AppSerdes.java │ │ ├── CountingWindowApp.java │ │ ├── InvoiceTimeExtractor.java │ │ ├── serde │ │ ├── JsonDeserializer.java │ │ └── JsonSerializer.java │ │ └── types │ │ └── SimpleInvoice.java │ └── resources │ ├── data │ └── sample-invoices.txt │ ├── log4j2.xml │ └── schema │ └── Invoice.json ├── custom-sink ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── producer-topic-table-map.cmd │ ├── producer-topic1.cmd │ ├── producer-topic2.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── AppConfigs.java │ │ ├── AppSerdes.java │ │ ├── CustomSinkApp.java │ │ ├── RecordTransformer.java │ │ ├── SinkProcessor.java │ │ ├── serde │ │ ├── JsonDeserializer.java │ │ └── JsonSerializer.java │ │ └── types │ │ ├── GenericRecord.java │ │ └── TableMap.java │ └── resources │ ├── data │ └── data.txt │ ├── log4j2.xml │ └── schema │ ├── generic_record.json │ └── table-map.json ├── custom-window ├── pom.xml ├── scripts │ ├── consumer.cmd │ ├── create-topic.cmd │ ├── producer.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── AppConfigs.java │ │ ├── AppSerdes.java │ │ ├── StockFeedApp.java │ │ ├── StockProcessor.java │ │ ├── StockPunctuator.java │ │ ├── serde │ │ ├── JsonDeserializer.java │ │ └── JsonSerializer.java │ │ └── types │ │ ├── StockTicker.java │ │ └── TickerStack.java │ └── resources │ ├── data │ └── data.txt │ ├── log4j2.xml │ └── schema │ └── stock-ticker.json ├── hello-producer ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-consumer.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── HelloProducer.java │ │ └── common │ │ └── AppConfigs.java │ └── resources │ └── log4j2.xml ├── hello-streams ├── README.md ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-consumer.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── HelloProducer.java │ │ └── HelloStreams.java │ └── resources │ └── log4j2.xml ├── json-consumer ├── README.md ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-consumer.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── JsonConsumer.java │ │ ├── JsonDeserializer.java │ │ ├── RunnableConsumer.java │ │ └── StockData.java │ └── resources │ ├── kafka.properties │ ├── log4j2.xml │ └── schema │ └── stock-data.json ├── json-producer ├── README.md ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-consumer.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── Dispatcher.java │ │ ├── JsonProducerDemo.java │ │ ├── JsonSerializer.java │ │ └── StockData.java │ └── resources │ ├── data │ ├── NSE05NOV2018BHAV.csv │ └── NSE06NOV2018BHAV.csv │ ├── kafka.properties │ ├── log4j2.xml │ └── schema │ └── stock-data.json ├── kstream-aggregate ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ ├── start-producer.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── AppConfigs.java │ │ ├── AppSerdes.java │ │ ├── KStreamAggDemo.java │ │ ├── serde │ │ ├── JsonDeserializer.java │ │ └── JsonSerializer.java │ │ └── types │ │ ├── DepartmentAggregate.java │ │ └── Employee.java │ └── resources │ ├── data │ └── data.txt │ ├── log4j2.xml │ └── schema │ ├── department_aggregates.json │ └── employees.json ├── ktable-aggregate-query ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ ├── start-producer.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── AppConfigs.java │ │ ├── AppRestService.java │ │ ├── AppSerdes.java │ │ ├── KTableAggDemo.java │ │ ├── serde │ │ ├── JsonDeserializer.java │ │ └── JsonSerializer.java │ │ └── types │ │ ├── DepartmentAggregate.java │ │ └── Employee.java │ └── resources │ ├── data │ └── data.txt │ ├── log4j2.xml │ └── schema │ ├── department_aggregates.json │ └── employees.json ├── ktable-aggregate ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ ├── start-producer.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── AppConfigs.java │ │ ├── AppSerdes.java │ │ ├── KTableAggDemo.java │ │ ├── serde │ │ ├── JsonDeserializer.java │ │ └── JsonSerializer.java │ │ └── types │ │ ├── DepartmentAggregate.java │ │ └── Employee.java │ └── resources │ ├── data │ └── data.txt │ ├── log4j2.xml │ └── schema │ ├── department_aggregates.json │ └── employees.json ├── last-login ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ ├── start-producer-user-login.cmd │ ├── start-producer-user-master.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── LastLoginDemo.java │ │ ├── common │ │ ├── AppConfigs.java │ │ └── AppSerdes.java │ │ ├── serde │ │ ├── JsonDeserializer.java │ │ └── JsonSerializer.java │ │ └── types │ │ ├── UserDetails.java │ │ └── UserLogin.java │ └── resources │ ├── data │ └── sample.txt │ ├── log4j2.xml │ └── schema │ ├── UserDetails.json │ └── UserLogins.json ├── partitioned-producer ├── README.md ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-consumer.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── OddEvenPartitioner.java │ │ └── PartitionedProducer.java │ └── resources │ └── log4j2.xml ├── pojo-from-avro ├── README.md ├── pom.xml └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ └── types │ │ └── Notification.java │ └── resources │ ├── log4j2.xml │ └── schema │ └── notification.avsc ├── pojo-from-json ├── pom.xml └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ └── types │ │ └── Notification.java │ └── resources │ ├── log4j2.xml │ └── schema │ └── notification.json ├── pos-fanout-avro ├── README.md ├── pom.xml ├── scripts │ ├── start-hadoop-sink-consumer.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ ├── start-schema-registry.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── FanOutConfigs.java │ │ ├── PosFanOutApp.java │ │ ├── PosSerdes.java │ │ ├── RecordBuilder.java │ │ └── types │ │ ├── DeliveryAddress.java │ │ ├── HadoopRecord.java │ │ ├── Notification.java │ │ ├── PosInvoice.java │ │ └── PosLineItem.java │ └── resources │ ├── log4j2.xml │ └── schema │ ├── HadoopRecord.avsc │ ├── Invoice.avsc │ └── Notification.avsc ├── pos-fanout ├── README.md ├── pom.xml ├── scripts │ ├── create-hadoop-topic.cmd │ ├── create-loyalty-topic.cmd │ ├── create-shipment-topic.cmd │ ├── start-hadoop-sink-consumer.cmd │ ├── start-loyalty-consumer.cmd │ └── start-shipment-consumer.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── FanOutConfigs.java │ │ ├── PosFanOutApp.java │ │ ├── PosSerdes.java │ │ ├── RecordBuilder.java │ │ ├── serde │ │ ├── JsonDeserializer.java │ │ └── JsonSerializer.java │ │ └── types │ │ ├── DeliveryAddress.java │ │ ├── HadoopRecord.java │ │ ├── Notification.java │ │ ├── PosInvoice.java │ │ └── PosLineItem.java │ └── resources │ ├── log4j2.xml │ └── schema │ ├── HadoopRecord.json │ ├── Invoice.json │ └── Notification.json ├── pos-simulator-avro ├── README.md ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-consumer.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ ├── start-schema-registry.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── PosSimulator.java │ │ ├── RunnableProducer.java │ │ ├── datagenerator │ │ ├── AddressGenerator.java │ │ ├── InvoiceGenerator.java │ │ └── ProductGenerator.java │ │ └── types │ │ ├── DeliveryAddress.java │ │ ├── PosInvoice.java │ │ └── PosLineItem.java │ └── resources │ ├── data │ ├── Invoice.json │ ├── address.json │ └── products.json │ ├── log4j2.xml │ └── schema │ └── Invoice.avsc ├── pos-simulator ├── README.md ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-consumer.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── PosSimulator.java │ │ ├── RunnableProducer.java │ │ ├── datagenerator │ │ ├── AddressGenerator.java │ │ ├── InvoiceGenerator.java │ │ └── ProductGenerator.java │ │ ├── serde │ │ └── JsonSerializer.java │ │ └── types │ │ ├── DeliveryAddress.java │ │ ├── PosInvoice.java │ │ └── PosLineItem.java │ └── resources │ ├── data │ ├── Invoice.json │ ├── address.json │ └── products.json │ ├── log4j2.xml │ └── schema │ └── Invoice.json ├── producer-threads ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-consumer.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── Dispatcher.java │ │ └── DispatcherDemo.java │ └── resources │ ├── data │ ├── NSE05NOV2018BHAV.csv │ └── NSE06NOV2018BHAV.csv │ ├── kafka.properties │ └── log4j2.xml ├── rewards-in-memory ├── pom.xml ├── scripts │ ├── create-intermediate-topic.cmd │ ├── create-loyalty-topic.cmd │ └── start-loyalty-consumer.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── AppConfigs.java │ │ ├── PosSerdes.java │ │ ├── RecordBuilder.java │ │ ├── RewardsApp.java │ │ ├── RewardsPartitioner.java │ │ ├── RewardsTransformer.java │ │ ├── serde │ │ ├── JsonDeserializer.java │ │ └── JsonSerializer.java │ │ └── types │ │ ├── DeliveryAddress.java │ │ ├── Notification.java │ │ ├── PosInvoice.java │ │ └── PosLineItem.java │ └── resources │ ├── log4j2.xml │ └── schema │ ├── Invoice.json │ └── Notification.json ├── rewards-table ├── pom.xml ├── scripts │ ├── create-loyalty-topic.cmd │ └── start-loyalty-consumer.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── AppConfigs.java │ │ ├── Notifications.java │ │ ├── PosSerdes.java │ │ ├── RewardsApp.java │ │ ├── serde │ │ ├── JsonDeserializer.java │ │ └── JsonSerializer.java │ │ └── types │ │ ├── DeliveryAddress.java │ │ ├── Notification.java │ │ ├── PosInvoice.java │ │ └── PosLineItem.java │ └── resources │ ├── log4j2.xml │ └── schema │ ├── Invoice.json │ └── Notification.json ├── stream-join ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ ├── start-producer_pay_confirm.cmd │ ├── start-producer_pay_request.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── StreamJoinDemo.java │ │ ├── common │ │ ├── AppConfigs.java │ │ ├── AppSerdes.java │ │ └── AppTimeExtractor.java │ │ ├── serde │ │ ├── JsonDeserializer.java │ │ └── JsonSerializer.java │ │ └── types │ │ ├── PaymentConfirmation.java │ │ ├── PaymentRequest.java │ │ └── TransactionStatus.java │ └── resources │ ├── data │ └── sample.txt │ ├── log4j2.xml │ └── schema │ ├── PaymentConfirmation.json │ ├── PaymentRequest.json │ └── TransactionStatus.json ├── streaming-table ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ ├── start-producer.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ └── StreamingTable.java │ └── resources │ └── log4j2.xml ├── streaming-word-count ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ ├── start-producer.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ └── StreamingWordCount.java │ └── resources │ └── log4j2.xml ├── sync-producer ├── README.md ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-consumer.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ └── SynchronousHelloProducer.java │ └── resources │ └── log4j2.xml ├── time-check ├── pom.xml └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── AppConfigs.java │ │ ├── InvoiceTimeExtractor.java │ │ ├── PosSerdes.java │ │ ├── TimeCheckDemo.java │ │ ├── serde │ │ ├── JsonDeserializer.java │ │ └── JsonSerializer.java │ │ └── types │ │ ├── DeliveryAddress.java │ │ ├── PosInvoice.java │ │ └── PosLineItem.java │ └── resources │ ├── log4j2.xml │ └── schema │ └── Invoice.json ├── top3-inventories ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ ├── start-producer-inventories.cmd │ ├── start-producer_ad-clicks.cmd │ └── start-zookeeper.cmd └── src │ └── main │ ├── java │ └── guru │ │ └── learningjournal │ │ └── kafka │ │ └── examples │ │ ├── Top3NewsTypesDemo.java │ │ ├── common │ │ ├── AppConfigs.java │ │ ├── AppSerdes.java │ │ └── Top3NewsTypes.java │ │ ├── serde │ │ ├── JsonDeserializer.java │ │ └── JsonSerializer.java │ │ └── types │ │ ├── AdClick.java │ │ ├── AdInventories.java │ │ └── ClicksByNewsType.java │ └── resources │ ├── data │ └── sample.txt │ ├── log4j2.xml │ └── schema │ ├── AdClick.json │ ├── AdInventories.json │ └── ClicksByNewsType.json ├── topology-test ├── pom.xml ├── scripts │ ├── create-topic.cmd │ ├── start-kafka-server-0.cmd │ ├── start-kafka-server-1.cmd │ ├── start-kafka-server-2.cmd │ ├── start-producer_ad-clicks.cmd │ ├── start-producer_ad-impressions.cmd │ └── start-zookeeper.cmd └── src │ ├── main │ ├── java │ │ └── guru │ │ │ └── learningjournal │ │ │ └── kafka │ │ │ └── examples │ │ │ ├── AppTopology.java │ │ │ ├── CampaignPerformanceApp.java │ │ │ ├── common │ │ │ ├── AppConfigs.java │ │ │ └── AppSerdes.java │ │ │ ├── serde │ │ │ ├── JsonDeserializer.java │ │ │ └── JsonSerializer.java │ │ │ └── types │ │ │ ├── AdClick.java │ │ │ ├── AdImpression.java │ │ │ └── CampaignPerformance.java │ └── resources │ │ ├── data │ │ └── sample.txt │ │ ├── log4j2.xml │ │ └── schema │ │ ├── AdClick.json │ │ ├── AdImpression.json │ │ └── CampaignPerformance.json │ └── test │ └── java │ └── guru │ └── learningjournal │ └── kafka │ └── examples │ ├── AppEmbeddedTest.java │ └── AppTopologyTest.java └── transactional-producer ├── README.md ├── pom.xml ├── scripts ├── create-topic.cmd ├── start-consumer1.cmd ├── start-consumer2.cmd ├── start-kafka-server-0.cmd ├── start-kafka-server-1.cmd ├── start-kafka-server-2.cmd └── start-zookeeper.cmd └── src └── main ├── java └── guru │ └── learningjournal │ └── kafka │ └── examples │ ├── Dispatcher.java │ ├── JsonSerializer.java │ ├── StockData.java │ └── TransactionalProducer.java └── resources ├── data ├── NSE05NOV2018BHAV.csv └── NSE06NOV2018BHAV.csv ├── kafka.properties ├── log4j2.xml └── schema └── stock-data.json /.gitignore: -------------------------------------------------------------------------------- 1 | # Leave entire .idea folder 2 | */.idea/* 3 | */*.iml 4 | 5 | # Leave kafka_logs and zookeeper_data and state store 6 | *kafka_logs* 7 | *zookeeper_data* 8 | *state-store* 9 | */logs/* 10 | 11 | # Leave target directory 12 | */target/* 13 | -------------------------------------------------------------------------------- /Figures/Figure-11.1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-11.1.jpg -------------------------------------------------------------------------------- /Figures/Figure-11.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-11.2.jpg -------------------------------------------------------------------------------- /Figures/Figure-11.3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-11.3.jpg -------------------------------------------------------------------------------- /Figures/Figure-13.1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-13.1.jpg -------------------------------------------------------------------------------- /Figures/Figure-13.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-13.2.jpg -------------------------------------------------------------------------------- /Figures/Figure-13.3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-13.3.jpg -------------------------------------------------------------------------------- /Figures/Figure-13.4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-13.4.jpg -------------------------------------------------------------------------------- /Figures/Figure-13.5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-13.5.jpg -------------------------------------------------------------------------------- /Figures/Figure-13.6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-13.6.jpg -------------------------------------------------------------------------------- /Figures/Figure-14.1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-14.1.jpg -------------------------------------------------------------------------------- /Figures/Figure-14.10.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-14.10.jpg -------------------------------------------------------------------------------- /Figures/Figure-14.11.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-14.11.jpg -------------------------------------------------------------------------------- /Figures/Figure-14.12.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-14.12.jpg -------------------------------------------------------------------------------- /Figures/Figure-14.13.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-14.13.jpg -------------------------------------------------------------------------------- /Figures/Figure-14.14.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-14.14.jpg -------------------------------------------------------------------------------- /Figures/Figure-14.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-14.2.jpg -------------------------------------------------------------------------------- /Figures/Figure-14.3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-14.3.jpg -------------------------------------------------------------------------------- /Figures/Figure-14.4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-14.4.jpg -------------------------------------------------------------------------------- /Figures/Figure-14.5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-14.5.jpg -------------------------------------------------------------------------------- /Figures/Figure-14.6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-14.6.jpg -------------------------------------------------------------------------------- /Figures/Figure-14.7.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-14.7.jpg -------------------------------------------------------------------------------- /Figures/Figure-14.8.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-14.8.jpg -------------------------------------------------------------------------------- /Figures/Figure-14.9.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-14.9.jpg -------------------------------------------------------------------------------- /Figures/Figure-15.1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-15.1.jpg -------------------------------------------------------------------------------- /Figures/Figure-15.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-15.2.jpg -------------------------------------------------------------------------------- /Figures/Figure-15.3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-15.3.jpg -------------------------------------------------------------------------------- /Figures/Figure-17.1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-17.1.jpg -------------------------------------------------------------------------------- /Figures/Figure-17.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-17.2.jpg -------------------------------------------------------------------------------- /Figures/Figure-17.3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-17.3.jpg -------------------------------------------------------------------------------- /Figures/Figure-17.4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-17.4.jpg -------------------------------------------------------------------------------- /Figures/Figure-18.1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-18.1.jpg -------------------------------------------------------------------------------- /Figures/Figure-19.1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-19.1.jpg -------------------------------------------------------------------------------- /Figures/Figure-19.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-19.2.jpg -------------------------------------------------------------------------------- /Figures/Figure-19.3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-19.3.jpg -------------------------------------------------------------------------------- /Figures/Figure-19.4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-19.4.jpg -------------------------------------------------------------------------------- /Figures/Figure-19.5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-19.5.jpg -------------------------------------------------------------------------------- /Figures/Figure-2.1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-2.1.jpg -------------------------------------------------------------------------------- /Figures/Figure-2.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-2.2.jpg -------------------------------------------------------------------------------- /Figures/Figure-2.3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-2.3.jpg -------------------------------------------------------------------------------- /Figures/Figure-20.1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-20.1.jpg -------------------------------------------------------------------------------- /Figures/Figure-20.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-20.2.jpg -------------------------------------------------------------------------------- /Figures/Figure-3.1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-3.1.jpg -------------------------------------------------------------------------------- /Figures/Figure-3.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-3.2.jpg -------------------------------------------------------------------------------- /Figures/Figure-3.3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-3.3.jpg -------------------------------------------------------------------------------- /Figures/Figure-3.4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-3.4.jpg -------------------------------------------------------------------------------- /Figures/Figure-3.5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-3.5.jpg -------------------------------------------------------------------------------- /Figures/Figure-3.6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-3.6.jpg -------------------------------------------------------------------------------- /Figures/Figure-3.7.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-3.7.jpg -------------------------------------------------------------------------------- /Figures/Figure-4.1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-4.1.jpg -------------------------------------------------------------------------------- /Figures/Figure-4.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-4.2.jpg -------------------------------------------------------------------------------- /Figures/Figure-4.3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-4.3.jpg -------------------------------------------------------------------------------- /Figures/Figure-4.4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-4.4.jpg -------------------------------------------------------------------------------- /Figures/Figure-4.5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-4.5.jpg -------------------------------------------------------------------------------- /Figures/Figure-4.6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-4.6.jpg -------------------------------------------------------------------------------- /Figures/Figure-4.7.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-4.7.jpg -------------------------------------------------------------------------------- /Figures/Figure-5.1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-5.1.jpg -------------------------------------------------------------------------------- /Figures/Figure-6.1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-6.1.jpg -------------------------------------------------------------------------------- /Figures/Figure-6.10.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-6.10.jpg -------------------------------------------------------------------------------- /Figures/Figure-6.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-6.2.jpg -------------------------------------------------------------------------------- /Figures/Figure-6.3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-6.3.jpg -------------------------------------------------------------------------------- /Figures/Figure-6.4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-6.4.jpg -------------------------------------------------------------------------------- /Figures/Figure-6.5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-6.5.jpg -------------------------------------------------------------------------------- /Figures/Figure-6.6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-6.6.jpg -------------------------------------------------------------------------------- /Figures/Figure-6.7.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-6.7.jpg -------------------------------------------------------------------------------- /Figures/Figure-6.8.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-6.8.jpg -------------------------------------------------------------------------------- /Figures/Figure-6.9.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-6.9.jpg -------------------------------------------------------------------------------- /Figures/Figure-7.1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-7.1.jpg -------------------------------------------------------------------------------- /Figures/Figure-7.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-7.2.jpg -------------------------------------------------------------------------------- /Figures/Figure-7.3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-7.3.jpg -------------------------------------------------------------------------------- /Figures/Figure-7.4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-7.4.jpg -------------------------------------------------------------------------------- /Figures/Figure-7.5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-7.5.jpg -------------------------------------------------------------------------------- /Figures/Figure-7.6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-7.6.jpg -------------------------------------------------------------------------------- /Figures/Figure-A.1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-A.1.jpg -------------------------------------------------------------------------------- /Figures/Figure-A.2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-A.2.jpg -------------------------------------------------------------------------------- /Figures/Figure-A.3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-A.3.jpg -------------------------------------------------------------------------------- /Figures/Figure-A.4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-A.4.jpg -------------------------------------------------------------------------------- /Figures/Figure-A.5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-A.5.jpg -------------------------------------------------------------------------------- /Figures/Figure-A.6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-A.6.jpg -------------------------------------------------------------------------------- /Figures/Figure-A.7.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LearningJournal/Kafka-Streams-Real-time-Stream-Processing/3787a27b049ebad591881cec65f62e84641228df/Figures/Figure-A.7.jpg -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Prashant Kumar Pandey 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /ad-campaigns/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic ad-impressions & kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic ad-clicks -------------------------------------------------------------------------------- /ad-campaigns/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /ad-campaigns/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /ad-campaigns/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /ad-campaigns/scripts/start-producer_ad-clicks.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic ad-clicks --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /ad-campaigns/scripts/start-producer_ad-impressions.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic ad-impressions --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /ad-campaigns/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /ad-campaigns/src/main/java/guru/learningjournal/kafka/examples/common/AppConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples.common; 17 | 18 | /** 19 | * Application Config 20 | * 21 | * @author prashant 22 | * @author www.learningjournal.guru 23 | */ 24 | 25 | public class AppConfigs { 26 | public final static String applicationID = "AdCampaignsDemo"; 27 | public final static String bootstrapServers = "localhost:9092,localhost:9093"; 28 | public final static String impressionTopic = "ad-impressions"; 29 | public final static String clicksTopic = "ad-clicks"; 30 | public final static String stateStoreName = "logs/state-store"; 31 | } 32 | -------------------------------------------------------------------------------- /ad-campaigns/src/main/java/guru/learningjournal/kafka/examples/types/AdClick.java: -------------------------------------------------------------------------------- 1 | 2 | package guru.learningjournal.kafka.examples.types; 3 | 4 | import com.fasterxml.jackson.annotation.JsonInclude; 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | import com.fasterxml.jackson.annotation.JsonPropertyOrder; 7 | import org.apache.commons.lang.builder.ToStringBuilder; 8 | 9 | @JsonInclude(JsonInclude.Include.NON_NULL) 10 | @JsonPropertyOrder({ 11 | "ImpressionID", 12 | "Campaigner" 13 | }) 14 | public class AdClick { 15 | 16 | @JsonProperty("ImpressionID") 17 | private String impressionID; 18 | @JsonProperty("Campaigner") 19 | private String campaigner; 20 | 21 | @JsonProperty("ImpressionID") 22 | public String getImpressionID() { 23 | return impressionID; 24 | } 25 | 26 | @JsonProperty("ImpressionID") 27 | public void setImpressionID(String impressionID) { 28 | this.impressionID = impressionID; 29 | } 30 | 31 | public AdClick withImpressionID(String impressionID) { 32 | this.impressionID = impressionID; 33 | return this; 34 | } 35 | 36 | @JsonProperty("Campaigner") 37 | public String getCampaigner() { 38 | return campaigner; 39 | } 40 | 41 | @JsonProperty("Campaigner") 42 | public void setCampaigner(String campaigner) { 43 | this.campaigner = campaigner; 44 | } 45 | 46 | public AdClick withCampaigner(String campaigner) { 47 | this.campaigner = campaigner; 48 | return this; 49 | } 50 | 51 | @Override 52 | public String toString() { 53 | return new ToStringBuilder(this).append("impressionID", impressionID).append("campaigner", campaigner).toString(); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /ad-campaigns/src/main/java/guru/learningjournal/kafka/examples/types/AdImpression.java: -------------------------------------------------------------------------------- 1 | 2 | package guru.learningjournal.kafka.examples.types; 3 | 4 | import com.fasterxml.jackson.annotation.JsonInclude; 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | import com.fasterxml.jackson.annotation.JsonPropertyOrder; 7 | import org.apache.commons.lang.builder.ToStringBuilder; 8 | 9 | @JsonInclude(JsonInclude.Include.NON_NULL) 10 | @JsonPropertyOrder({ 11 | "ImpressionID", 12 | "Campaigner" 13 | }) 14 | public class AdImpression { 15 | 16 | @JsonProperty("ImpressionID") 17 | private String impressionID; 18 | @JsonProperty("Campaigner") 19 | private String campaigner; 20 | 21 | @JsonProperty("ImpressionID") 22 | public String getImpressionID() { 23 | return impressionID; 24 | } 25 | 26 | @JsonProperty("ImpressionID") 27 | public void setImpressionID(String impressionID) { 28 | this.impressionID = impressionID; 29 | } 30 | 31 | public AdImpression withImpressionID(String impressionID) { 32 | this.impressionID = impressionID; 33 | return this; 34 | } 35 | 36 | @JsonProperty("Campaigner") 37 | public String getCampaigner() { 38 | return campaigner; 39 | } 40 | 41 | @JsonProperty("Campaigner") 42 | public void setCampaigner(String campaigner) { 43 | this.campaigner = campaigner; 44 | } 45 | 46 | public AdImpression withCampaigner(String campaigner) { 47 | this.campaigner = campaigner; 48 | return this; 49 | } 50 | 51 | @Override 52 | public String toString() { 53 | return new ToStringBuilder(this).append("impressionID", impressionID).append("campaigner", campaigner).toString(); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /ad-campaigns/src/main/resources/data/sample.txt: -------------------------------------------------------------------------------- 1 | 100001:{"ImpressionID": "100001", "Campaigner": "ABC Ltd"} 2 | 100002:{"ImpressionID": "100002", "Campaigner": "XYZ Ltd"} 3 | 100003:{"ImpressionID": "100003", "Campaigner": "XYZ Ltd"} 4 | 100004:{"ImpressionID": "100004", "Campaigner": "XYZ Ltd"} 5 | 100005:{"ImpressionID": "100005", "Campaigner": "XYZ Ltd"} 6 | 100006:{"ImpressionID": "100006", "Campaigner": "ABC Ltd"} 7 | 100007:{"ImpressionID": "100007", "Campaigner": "ABC Ltd"} 8 | 100008:{"ImpressionID": "100008", "Campaigner": "ABC Ltd"} 9 | 100009:{"ImpressionID": "100009", "Campaigner": "ABC Ltd"} 10 | 100010:{"ImpressionID": "100010", "Campaigner": "ABC Ltd"} 11 | 12 | 100009:{"ImpressionID": "100009", "Campaigner": "ABC Ltd"} 13 | 100010:{"ImpressionID": "100010", "Campaigner": "ABC Ltd"} 14 | 100002:{"ImpressionID": "100002", "Campaigner": "XYZ Ltd"} -------------------------------------------------------------------------------- /ad-campaigns/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /ad-campaigns/src/main/resources/schema/AdClick.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.AdClick", 4 | "properties": { 5 | "ImpressionID": { 6 | "type": "string" 7 | }, 8 | "Campaigner": { 9 | "type": "string" 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /ad-campaigns/src/main/resources/schema/AdImpression.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.AdImpression", 4 | "properties": { 5 | "ImpressionID": { 6 | "type": "string" 7 | }, 8 | "Campaigner": { 9 | "type": "string" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /ad-campaigns/src/main/resources/schema/CampaignPerformance.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.CampaignPerformance", 4 | "properties": { 5 | "Campaigner": { 6 | "type": "string" 7 | }, 8 | "AdImpressions": { 9 | "javaType": "java.lang.Long" 10 | }, 11 | "AdClicks": { 12 | "javaType": "java.lang.Long" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /age-count/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic person-age -------------------------------------------------------------------------------- /age-count/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /age-count/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /age-count/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /age-count/scripts/start-producer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic person-age --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /age-count/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /age-count/src/main/java/guru/learningjournal/kafka/examples/AppConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples; 17 | 18 | /** 19 | * Application Config 20 | * 21 | * @author prashant 22 | * @author www.learningjournal.guru 23 | */ 24 | class AppConfigs { 25 | 26 | final static String applicationID = "AgeCountDemo"; 27 | final static String bootstrapServers = "localhost:9092"; 28 | final static String topicName = "person-age"; 29 | final static String stateStoreLocation = "logs/state-store"; 30 | } 31 | -------------------------------------------------------------------------------- /age-count/src/main/resources/data/data.txt: -------------------------------------------------------------------------------- 1 | Abdul:26 2 | Prashant:41 3 | John:38 4 | Milli:26 5 | 6 | Abdul:27 -------------------------------------------------------------------------------- /age-count/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /avro-consumer/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | This directory contains Kafka Producer example. 4 | 5 | The demo application takes following Program Arguments: 6 | 7 | `nse-bhav-avro group1` -------------------------------------------------------------------------------- /avro-consumer/scripts/start-kafka-server.cmd: -------------------------------------------------------------------------------- 1 | rem Start Kafka Server from the Confluent platform 2 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\kafka-server-start.bat C:\Users\prashant\Downloads\confluent-5.0.1\etc\kafka\server.properties -------------------------------------------------------------------------------- /avro-consumer/scripts/start-schema-registry.cmd: -------------------------------------------------------------------------------- 1 | rem Start Schema registry from the Confluent platform 2 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\schema-registry-start.bat C:\Users\prashant\Downloads\confluent-5.0.1\etc\schema-registry\schema-registry.properties -------------------------------------------------------------------------------- /avro-consumer/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | rem Start Zookeeper from the Confluent platform 2 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\zookeeper-server-start.bat C:\Users\prashant\Downloads\confluent-5.0.1\etc\kafka\zookeeper.properties -------------------------------------------------------------------------------- /avro-consumer/src/main/resources/kafka.properties: -------------------------------------------------------------------------------- 1 | client.id=AvroConsumer 2 | bootstrap.servers=localhost:9092 -------------------------------------------------------------------------------- /avro-consumer/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /avro-consumer/src/main/resources/schema/srock-data.avsc: -------------------------------------------------------------------------------- 1 | {"namespace": "guru.learningjournal.kafka.examples", 2 | "type": "record", 3 | "name": "StockData", 4 | "fields": [ 5 | {"name": "symbol", "type": ["null", "string"]}, 6 | {"name": "series", "type":["null", "string"]}, 7 | {"name": "open", "type": ["null", "float"]}, 8 | {"name": "high", "type": ["null", "float"]}, 9 | {"name": "low", "type": ["null", "float"]}, 10 | {"name": "close", "type": ["null", "float"]}, 11 | {"name": "last", "type": ["null", "float"]}, 12 | {"name": "previousClose", "type": ["null", "float"]}, 13 | {"name": "totalTradedQty", "type": ["null", "long"]}, 14 | {"name": "totalTradedVal", "type": ["null", "double"]}, 15 | {"name": "tradeDate", "type": ["null", "string"]}, 16 | {"name": "totalTrades", "type": ["null", "int"]}, 17 | {"name": "isinCode", "type": ["null", "string"]} 18 | ] 19 | } 20 | 21 | -------------------------------------------------------------------------------- /avro-producer/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | This directory contains Kafka Producer example. 4 | 5 | The demo application takes following Program Arguments: 6 | 7 | `nse-bhav-avro /data/NSE05NOV2018BHAV.csv /data/NSE06NOV2018BHAV.csv` -------------------------------------------------------------------------------- /avro-producer/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | rem Create topic in Confluent Kafka 2 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic nse-bhav-avro -------------------------------------------------------------------------------- /avro-producer/scripts/start-consumer.cmd: -------------------------------------------------------------------------------- 1 | rem This consumer will show you some junk characters as the messages are in Avro binary format 2 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic nse-bhav-avro --from-beginning -------------------------------------------------------------------------------- /avro-producer/scripts/start-kafka-server.cmd: -------------------------------------------------------------------------------- 1 | rem Start Kafka server from the Confluent platform 2 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\kafka-server-start.bat C:\Users\prashant\Downloads\confluent-5.0.1\etc\kafka\server.properties -------------------------------------------------------------------------------- /avro-producer/scripts/start-schema-registry.cmd: -------------------------------------------------------------------------------- 1 | rem Start schema registry from the Confluent platform 2 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\schema-registry-start.bat C:\Users\prashant\Downloads\confluent-5.0.1\etc\schema-registry\schema-registry.properties -------------------------------------------------------------------------------- /avro-producer/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | rem Start Zookeeper from the Confluent platform 2 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\zookeeper-server-start.bat C:\Users\prashant\Downloads\confluent-5.0.1\etc\kafka\zookeeper.properties -------------------------------------------------------------------------------- /avro-producer/src/main/resources/kafka.properties: -------------------------------------------------------------------------------- 1 | client.id=AvroProducer 2 | bootstrap.servers=localhost:9092,localhost:9093 -------------------------------------------------------------------------------- /avro-producer/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /avro-producer/src/main/resources/schema/srock-data.avsc: -------------------------------------------------------------------------------- 1 | {"namespace": "guru.learningjournal.kafka.examples", 2 | "type": "record", 3 | "name": "StockData", 4 | "fields": [ 5 | {"name": "symbol", "type": ["null", "string"]}, 6 | {"name": "series", "type":["null", "string"]}, 7 | {"name": "open", "type": ["null", "float"]}, 8 | {"name": "high", "type": ["null", "float"]}, 9 | {"name": "low", "type": ["null", "float"]}, 10 | {"name": "close", "type": ["null", "float"]}, 11 | {"name": "last", "type": ["null", "float"]}, 12 | {"name": "previousClose", "type": ["null", "float"]}, 13 | {"name": "totalTradedQty", "type": ["null", "long"]}, 14 | {"name": "totalTradedVal", "type": ["null", "double"]}, 15 | {"name": "tradeDate", "type": ["null", "string"]}, 16 | {"name": "totalTrades", "type": ["null", "int"]}, 17 | {"name": "isinCode", "type": ["null", "string"]} 18 | ] 19 | } 20 | 21 | -------------------------------------------------------------------------------- /callback-producer/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | This directory contains Kafka Producer example. 4 | 5 | The demo application takes following Program Arguments: 6 | 7 | `hello-producer 10` -------------------------------------------------------------------------------- /callback-producer/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic hello-producer -------------------------------------------------------------------------------- /callback-producer/scripts/start-consumer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic hello-producer --from-beginning -------------------------------------------------------------------------------- /callback-producer/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\server-0.properties -------------------------------------------------------------------------------- /callback-producer/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\server-1.properties -------------------------------------------------------------------------------- /callback-producer/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\server-2.properties -------------------------------------------------------------------------------- /callback-producer/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\zookeeper.properties -------------------------------------------------------------------------------- /callback-producer/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /counting-session/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic user-clicks --config min.insync.replicas=2 -------------------------------------------------------------------------------- /counting-session/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /counting-session/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /counting-session/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /counting-session/scripts/start-producer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic user-clicks --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /counting-session/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /counting-session/src/main/java/guru/learningjournal/kafka/examples/common/AppConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples.common; 17 | 18 | /** 19 | * Application Configs 20 | * 21 | * @author prashant 22 | * @author www.learningjournal.guru 23 | */ 24 | public class AppConfigs { 25 | 26 | public final static String applicationID = "CountingSessionApp"; 27 | public final static String bootstrapServers = "192.168.0.105:9092"; 28 | public final static String posTopicName = "user-clicks"; 29 | public final static String stateStoreName = "ss-user-clicks"; 30 | } -------------------------------------------------------------------------------- /counting-session/src/main/java/guru/learningjournal/kafka/examples/common/AppTimestampExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples.common; 17 | 18 | import guru.learningjournal.kafka.examples.types.UserClicks; 19 | import org.apache.kafka.clients.consumer.ConsumerRecord; 20 | import org.apache.kafka.streams.processor.TimestampExtractor; 21 | 22 | /** 23 | * Custom TimestampExtractor for the Application 24 | * 25 | * @author prashant 26 | * @author www.learningjournal.guru 27 | */ 28 | 29 | public class AppTimestampExtractor implements TimestampExtractor { 30 | 31 | @Override 32 | public long extract(ConsumerRecord consumerRecord, long prevTime) { 33 | UserClicks record = (UserClicks) consumerRecord.value(); 34 | return ((record.getCreatedTime() > 0) ? record.getCreatedTime() : prevTime); 35 | } 36 | } -------------------------------------------------------------------------------- /counting-session/src/main/resources/data/sample-data.txt: -------------------------------------------------------------------------------- 1 | USR101:{"UserID": "USR101","CreatedTime": "1549360860000","CurrentLink": "NULL","NextLink": "Home"} 2 | USR102:{"UserID": "USR102","CreatedTime": "1549360920000","CurrentLink": "NULL","NextLink": "Home"} 3 | USR101:{"UserID": "USR101","CreatedTime": "1549361040000","CurrentLink": "Home","NextLink": "Books"} 4 | USR101:{"UserID": "USR101","CreatedTime": "1549361400000","CurrentLink": "Books","NextLink": "Kafka"} 5 | USR102:{"UserID": "USR102","CreatedTime": "1549361400000","CurrentLink": "Home","NextLink": "Courses"} 6 | USR101:{"UserID": "USR101","CreatedTime": "1549361220000","CurrentLink": "Kafka","NextLink": "Preview"} 7 | USR101:{"UserID": "USR101","CreatedTime": "1549361940000","CurrentLink": "Preview","NextLink": "Buy"} -------------------------------------------------------------------------------- /counting-session/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /counting-session/src/main/resources/schema/UserClicks.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.UserClicks", 4 | "properties": { 5 | "UserID": { 6 | "type": "string" 7 | }, 8 | "CreatedTime": { 9 | "javaType": "java.lang.Long", 10 | "type": "object" 11 | }, 12 | "CurrentLink": { 13 | "type": "string" 14 | }, 15 | "NextLink": { 16 | "type": "string" 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /counting-window/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | This directory contains Kafka Producer example. 4 | 5 | The demo application takes following Program Arguments: 6 | 7 | hello-producer 10 -------------------------------------------------------------------------------- /counting-window/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 3 --topic simple-invoice --config min.insync.replicas=2 -------------------------------------------------------------------------------- /counting-window/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /counting-window/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /counting-window/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /counting-window/scripts/start-producer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic simple-invoice --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /counting-window/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /counting-window/src/main/java/guru/learningjournal/kafka/examples/AppConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples; 17 | 18 | /** 19 | * Application Configs 20 | * 21 | * @author prashant 22 | * @author www.learningjournal.guru 23 | */ 24 | class AppConfigs { 25 | 26 | final static String applicationID = "CountingWindowApp"; 27 | final static String bootstrapServers = "localhost:9092"; 28 | final static String posTopicName = "simple-invoice"; 29 | final static String stateStoreName = "state-store"; 30 | } 31 | -------------------------------------------------------------------------------- /counting-window/src/main/java/guru/learningjournal/kafka/examples/InvoiceTimeExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples; 17 | 18 | import guru.learningjournal.kafka.examples.types.SimpleInvoice; 19 | import org.apache.kafka.clients.consumer.ConsumerRecord; 20 | import org.apache.kafka.streams.processor.TimestampExtractor; 21 | 22 | /** 23 | * Extract timestamp from Invoice field 24 | * 25 | * @author prashant 26 | * @author www.learningjournal.guru 27 | */ 28 | 29 | public class InvoiceTimeExtractor implements TimestampExtractor { 30 | 31 | @Override 32 | public long extract(ConsumerRecord consumerRecord, long prevTime) { 33 | SimpleInvoice invoice = (SimpleInvoice) consumerRecord.value(); 34 | return ((invoice.getCreatedTime() > 0) ? invoice.getCreatedTime() : prevTime); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /counting-window/src/main/resources/data/sample-invoices.txt: -------------------------------------------------------------------------------- 1 | STR1534:{"InvoiceNumber": 101,"CreatedTime": "1549360860000","StoreID": "STR1534","TotalAmount": 1920} 2 | STR1535:{"InvoiceNumber": 102,"CreatedTime": "1549360900000","StoreID": "STR1535","TotalAmount": 1860} 3 | STR1534:{"InvoiceNumber": 103,"CreatedTime": "1549360999000","StoreID": "STR1534","TotalAmount": 2400} 4 | STR1536:{"InvoiceNumber": 104,"CreatedTime": "1549361160000","StoreID": "STR1536","TotalAmount": 8936} 5 | STR1534:{"InvoiceNumber": 105,"CreatedTime": "1549361270000","StoreID": "STR1534","TotalAmount": 6375} 6 | STR1536:{"InvoiceNumber": 106,"CreatedTime": "1549361370000","StoreID": "STR1536","TotalAmount": 9365} -------------------------------------------------------------------------------- /counting-window/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /counting-window/src/main/resources/schema/Invoice.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.SimpleInvoice", 4 | "properties": { 5 | "InvoiceNumber": { 6 | "type": "string" 7 | }, 8 | "CreatedTime": { 9 | "javaType": "java.lang.Long", 10 | "type": "object" 11 | }, 12 | "StoreID": { 13 | "type": "string" 14 | }, 15 | "TotalAmount": { 16 | "type": "number" 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /custom-sink/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 1 --topic topic1 & kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 1 --topic topic2 & kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 1 --topic table-map -------------------------------------------------------------------------------- /custom-sink/scripts/producer-topic-table-map.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic table-map --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /custom-sink/scripts/producer-topic1.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic topic1 -------------------------------------------------------------------------------- /custom-sink/scripts/producer-topic2.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic topic2 -------------------------------------------------------------------------------- /custom-sink/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /custom-sink/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /custom-sink/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /custom-sink/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /custom-sink/src/main/java/guru/learningjournal/kafka/examples/AppConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples; 17 | 18 | import java.util.regex.Pattern; 19 | 20 | /** 21 | * Application Config 22 | * 23 | * @author prashant 24 | * @author www.learningjournal.guru 25 | */ 26 | 27 | class AppConfigs { 28 | final static String applicationID = "CustomSinkApp"; 29 | final static String bootstrapServers = "localhost:9092"; 30 | final static Pattern topicPattern = Pattern.compile("topic\\d"); 31 | final static String topicTableMap = "table-map"; 32 | final static String stateStoreLocation = "logs/state-store"; 33 | final static String targetTableField = "TargetTable"; 34 | } 35 | -------------------------------------------------------------------------------- /custom-sink/src/main/java/guru/learningjournal/kafka/examples/RecordTransformer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | package guru.learningjournal.kafka.examples; 16 | 17 | import guru.learningjournal.kafka.examples.types.GenericRecord; 18 | import org.apache.kafka.streams.KeyValue; 19 | import org.apache.kafka.streams.kstream.Transformer; 20 | import org.apache.kafka.streams.processor.ProcessorContext; 21 | 22 | /** 23 | * Transform record to modify key as topic name 24 | * 25 | * @author prashant 26 | * @author www.learningjournal.guru 27 | */ 28 | 29 | public class RecordTransformer 30 | implements Transformer> { 31 | 32 | private ProcessorContext context; 33 | 34 | @Override 35 | public void init(ProcessorContext processorContext) { 36 | this.context = processorContext; 37 | } 38 | 39 | @Override 40 | public KeyValue transform(String key, GenericRecord genericRecord) { 41 | return KeyValue.pair(context.topic(), genericRecord); 42 | } 43 | 44 | @Override 45 | public void close() { 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /custom-sink/src/main/resources/data/data.txt: -------------------------------------------------------------------------------- 1 | Data for topic1 2 | ---------------- 3 | {"rowID": "101", "name": "Prashant", "department": "engineering", "salary": "5000"} 4 | {"rowID": "102", "name": "John", "department": "accounts", "salary": "8000"} 5 | {"rowID": "103", "name": "Abdul", "department": "support", "salary": "7000"} 6 | 7 | Data for Topic2 8 | ---------------- 9 | {"rowID": "501", "name": "ABC Ltd", "city": "Bangalore"} 10 | {"rowID": "502", "name": "XYZ Ltd", "city": "London"} 11 | {"rowID": "503", "name": "JKL LLC", "city": "New York City"} 12 | 13 | Data for topic-table-map 14 | -------------------------- 15 | topic1:{"TargetTable": "employee"} 16 | topic2:{"TargetTable": "customers"} -------------------------------------------------------------------------------- /custom-sink/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /custom-sink/src/main/resources/schema/generic_record.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.GenericRecord", 4 | "properties": { 5 | "rowID": { 6 | "type": "string" 7 | } 8 | }, 9 | "additionalProperties": { 10 | "type": "string" 11 | } 12 | } -------------------------------------------------------------------------------- /custom-sink/src/main/resources/schema/table-map.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.TableMap", 4 | "properties": { 5 | "TargetTable": { 6 | "type": "string" 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /custom-window/scripts/consumer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic training-feed --property print.key=true --property key.separator=":" -------------------------------------------------------------------------------- /custom-window/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic stock-tick & kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic training-feed -------------------------------------------------------------------------------- /custom-window/scripts/producer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic stock-tick --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /custom-window/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /custom-window/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /custom-window/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /custom-window/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /custom-window/src/main/java/guru/learningjournal/kafka/examples/AppConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples; 17 | 18 | /** 19 | * Application Config 20 | * 21 | * @author prashant 22 | * @author www.learningjournal.guru 23 | */ 24 | 25 | public class AppConfigs { 26 | final static String applicationID = "StockFeedApp"; 27 | final static String bootstrapServers = "localhost:9092"; 28 | final static String sourceTopicName = "stock-tick"; 29 | final static String targetTopicName = "training-feed"; 30 | final static String sourceName = "stock-tick-source"; 31 | final static String sinkName = "training-feed-sink"; 32 | final static String stateStoreLocation = "logs/state-store"; 33 | public final static int ltpStackSize = 10; 34 | public final static long secondsDelay = 10; 35 | 36 | } 37 | -------------------------------------------------------------------------------- /custom-window/src/main/resources/data/data.txt: -------------------------------------------------------------------------------- 1 | TCS:{"tickerSymbol": "TCS", "tickerTime": 1551774312763, "lastTradedPrice": 1983} 2 | TCS:{"tickerSymbol": "TCS", "tickerTime": 1551774352763, "lastTradedPrice": 2005} 3 | TCS:{"tickerSymbol": "TCS", "tickerTime": 1551774382763, "lastTradedPrice": 2035} 4 | 5 | ABB:{"tickerSymbol": "ABB", "tickerTime": 1551774313883, "lastTradedPrice": 1234} 6 | ABB:{"tickerSymbol": "ABB", "tickerTime": 1551774353793, "lastTradedPrice": 1195} 7 | ABB:{"tickerSymbol": "ABB", "tickerTime": 1551774383963, "lastTradedPrice": 1155} 8 | 9 | -------------------------------------------------------------------------------- /custom-window/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /custom-window/src/main/resources/schema/stock-ticker.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.StockTicker", 4 | "properties": { 5 | "tickerSymbol": { 6 | "type": "string" 7 | }, 8 | "tickerTime": { 9 | "type":"string", 10 | "format":"utc-millisec" 11 | }, 12 | "lastTradedPrice": { 13 | "type": "number" 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /hello-producer/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic hello-producer -------------------------------------------------------------------------------- /hello-producer/scripts/start-consumer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic hello-producer --from-beginning 2 | -------------------------------------------------------------------------------- /hello-producer/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /hello-producer/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /hello-producer/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /hello-producer/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /hello-producer/src/main/java/guru/learningjournal/kafka/examples/common/AppConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples.common; 17 | 18 | /** 19 | * Application Configs 20 | * 21 | * @author prashant 22 | * @author www.learningjournal.guru 23 | */ 24 | public class AppConfigs { 25 | 26 | public final static String applicationID = "HelloProducer"; 27 | public final static String bootstrapServers = "localhost:9092,localhost:9093"; 28 | public final static String topicName = "hello-producer"; 29 | public final static int numEvents = 10; 30 | } -------------------------------------------------------------------------------- /hello-producer/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /hello-streams/README.md: -------------------------------------------------------------------------------- 1 | 2 | This directory contains Kafka Producer and a Kafka Streams consumer. 3 | 4 | The demo application implements two main methods 5 | and they take following Program Arguments: 6 | 7 | HelloProducer class takes two arguments 8 | 9 | `hello-producer 10` 10 | 11 | HelloStreams class takes one argument 12 | 13 | `hello-producer` 14 | -------------------------------------------------------------------------------- /hello-streams/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic hello-producer -------------------------------------------------------------------------------- /hello-streams/scripts/start-consumer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic hello-producer-demo --from-beginning -------------------------------------------------------------------------------- /hello-streams/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /hello-streams/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /hello-streams/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /hello-streams/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /hello-streams/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /json-consumer/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | This directory contains Kafka Producer example. 4 | 5 | The demo application takes following Program Arguments: 6 | 7 | `2 test-group nse-bhav-json` -------------------------------------------------------------------------------- /json-consumer/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic nse-bhav-json -------------------------------------------------------------------------------- /json-consumer/scripts/start-consumer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic hello-producer-demo --from-beginning -------------------------------------------------------------------------------- /json-consumer/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\server-0.properties -------------------------------------------------------------------------------- /json-consumer/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\server-1.properties -------------------------------------------------------------------------------- /json-consumer/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\server-2.properties -------------------------------------------------------------------------------- /json-consumer/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\zookeeper.properties -------------------------------------------------------------------------------- /json-consumer/src/main/resources/kafka.properties: -------------------------------------------------------------------------------- 1 | bootstrap.servers=localhost:9092,localhost:9093 -------------------------------------------------------------------------------- /json-consumer/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /json-consumer/src/main/resources/schema/stock-data.json: -------------------------------------------------------------------------------- 1 | { 2 | "type":"object", 3 | "javaType" : "guru.learningjournal.kafka.examples.StockData", 4 | "properties": { 5 | "symbol": { 6 | "type": "string" 7 | }, 8 | "series": { 9 | "type": "string" 10 | }, 11 | "open": { 12 | "type": "string" 13 | }, 14 | "high": { 15 | "type": "number" 16 | }, 17 | "low": { 18 | "type": "number" 19 | }, 20 | "close": { 21 | "type": "number" 22 | }, 23 | "last": { 24 | "type": "number" 25 | }, 26 | "previousClose": { 27 | "type": "number" 28 | }, 29 | "totalTradedQty": { 30 | "type": "number" 31 | }, 32 | "totalTradedVal": { 33 | "type": "number" 34 | }, 35 | "tradeDate": { 36 | "type": "number", 37 | "format":"date-time", 38 | "customDateTimePattern" : "dd-MMM-yyyy", 39 | "customTimezone": "IST" 40 | }, 41 | "totalTrades": { 42 | "type": "number" 43 | }, 44 | "isinCode": { 45 | "type": "string" 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /json-producer/README.md: -------------------------------------------------------------------------------- 1 | The demo application takes following Program Arguments: 2 | 3 | `nse-bhav-json /data/NSE05NOV2018BHAV.csv /data/NSE06NOV2018BHAV.csv` 4 | -------------------------------------------------------------------------------- /json-producer/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic nse-bhav-json -------------------------------------------------------------------------------- /json-producer/scripts/start-consumer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic nse-bhav-json --from-beginning -------------------------------------------------------------------------------- /json-producer/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /json-producer/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /json-producer/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /json-producer/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /json-producer/src/main/resources/kafka.properties: -------------------------------------------------------------------------------- 1 | client.id=test_client 2 | bootstrap.servers=localhost:9092,localhost:9093 -------------------------------------------------------------------------------- /json-producer/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /json-producer/src/main/resources/schema/stock-data.json: -------------------------------------------------------------------------------- 1 | { 2 | "type":"object", 3 | "javaType" : "guru.learningjournal.kafka.examples.StockData", 4 | "properties": { 5 | "symbol": { 6 | "type": "string" 7 | }, 8 | "series": { 9 | "type": "string" 10 | }, 11 | "open": { 12 | "type": "string" 13 | }, 14 | "high": { 15 | "type": "number" 16 | }, 17 | "low": { 18 | "type": "number" 19 | }, 20 | "close": { 21 | "type": "number" 22 | }, 23 | "last": { 24 | "type": "number" 25 | }, 26 | "previousClose": { 27 | "type": "number" 28 | }, 29 | "totalTradedQty": { 30 | "type": "number" 31 | }, 32 | "totalTradedVal": { 33 | "type": "number" 34 | }, 35 | "tradeDate": { 36 | "type": "number", 37 | "format":"date-time", 38 | "customDateTimePattern" : "dd-MMM-yyyy", 39 | "customTimezone": "IST" 40 | }, 41 | "totalTrades": { 42 | "type": "number" 43 | }, 44 | "isinCode": { 45 | "type": "string" 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /kstream-aggregate/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic employees -------------------------------------------------------------------------------- /kstream-aggregate/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /kstream-aggregate/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /kstream-aggregate/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /kstream-aggregate/scripts/start-producer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic employees --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /kstream-aggregate/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /kstream-aggregate/src/main/java/guru/learningjournal/kafka/examples/AppConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples; 17 | 18 | /** 19 | * Application Config 20 | * 21 | * @author prashant 22 | * @author www.learningjournal.guru 23 | */ 24 | class AppConfigs { 25 | 26 | final static String applicationID = "KStreamAggDemo"; 27 | final static String bootstrapServers = "localhost:9092"; 28 | final static String topicName = "employees"; 29 | final static String stateStoreName = "state-store"; 30 | } 31 | -------------------------------------------------------------------------------- /kstream-aggregate/src/main/resources/data/data.txt: -------------------------------------------------------------------------------- 1 | 101:{"id": "101", "name": "Prashant", "department": "engineering", "salary": 5000} 2 | 102:{"id": "102", "name": "John", "department": "accounts", "salary": 8000} 3 | 103:{"id": "103", "name": "Abdul", "department": "engineering", "salary": 3000} 4 | 104:{"id": "104", "name": "Melinda", "department": "support", "salary": 7000} 5 | 105:{"id": "105", "name": "Jimmy", "department": "support", "salary": 6000} 6 | 101:{"id": "101", "name": "Prashant", "department": "support", "salary": 5000} 7 | 104:{"id": "104", "name": "Melinda", "department": "engineering", "salary": 7000} -------------------------------------------------------------------------------- /kstream-aggregate/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /kstream-aggregate/src/main/resources/schema/department_aggregates.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.DepartmentAggregate", 4 | "properties": { 5 | "total_salary": { 6 | "type": "integer" 7 | }, 8 | "employee_count": { 9 | "type": "integer" 10 | }, 11 | "avg_salary": { 12 | "type": "number" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /kstream-aggregate/src/main/resources/schema/employees.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.Employee", 4 | "properties": { 5 | "id": { 6 | "type": "string" 7 | }, 8 | "name": { 9 | "type": "string" 10 | }, 11 | "department": { 12 | "type": "string" 13 | }, 14 | "salary": { 15 | "type": "integer" 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /ktable-aggregate-query/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 6 --topic employees -------------------------------------------------------------------------------- /ktable-aggregate-query/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /ktable-aggregate-query/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /ktable-aggregate-query/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /ktable-aggregate-query/scripts/start-producer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic employees --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /ktable-aggregate-query/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /ktable-aggregate-query/src/main/java/guru/learningjournal/kafka/examples/AppConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples; 17 | 18 | /** 19 | * Application Config 20 | * 21 | * @author prashant 22 | * @author www.learningjournal.guru 23 | */ 24 | 25 | class AppConfigs { 26 | final static String applicationID = "KTableQueryApp"; 27 | final static String bootstrapServers = "localhost:9092"; 28 | final static String topicName = "employees"; 29 | final static String stateStoreLocation = "logs/state-store"; 30 | final static String aggStateStoreName = "dept-agg-store"; 31 | } 32 | -------------------------------------------------------------------------------- /ktable-aggregate-query/src/main/resources/data/data.txt: -------------------------------------------------------------------------------- 1 | 101:{"id": "101", "name": "Prashant", "department": "engineering", "salary": 5000} 2 | 102:{"id": "102", "name": "John", "department": "accounts", "salary": 8000} 3 | 103:{"id": "103", "name": "Abdul", "department": "engineering", "salary": 3000} 4 | 104:{"id": "104", "name": "Melinda", "department": "support", "salary": 7000} 5 | 105:{"id": "105", "name": "Jimmy", "department": "support", "salary": 6000} 6 | 10001:{"id": "107", "name": "Jimmy", "department": "123456", "salary": 6000} 7 | 8 | 9 | 101:{"id": "101", "name": "Prashant", "department": "support", "salary": 5000} 10 | 104:{"id": "104", "name": "Melinda", "department": "engineering", "salary": 7000} 11 | -------------------------------------------------------------------------------- /ktable-aggregate-query/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /ktable-aggregate-query/src/main/resources/schema/department_aggregates.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.DepartmentAggregate", 4 | "properties": { 5 | "total_salary": { 6 | "type": "integer" 7 | }, 8 | "employee_count": { 9 | "type": "integer" 10 | }, 11 | "avg_salary": { 12 | "type": "number" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /ktable-aggregate-query/src/main/resources/schema/employees.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.Employee", 4 | "properties": { 5 | "id": { 6 | "type": "string" 7 | }, 8 | "name": { 9 | "type": "string" 10 | }, 11 | "department": { 12 | "type": "string" 13 | }, 14 | "salary": { 15 | "type": "integer" 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /ktable-aggregate/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic employees -------------------------------------------------------------------------------- /ktable-aggregate/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /ktable-aggregate/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /ktable-aggregate/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /ktable-aggregate/scripts/start-producer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic employees --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /ktable-aggregate/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /ktable-aggregate/src/main/java/guru/learningjournal/kafka/examples/AppConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples; 17 | 18 | /** 19 | * Application Config 20 | * 21 | * @author prashant 22 | * @author www.learningjournal.guru 23 | */ 24 | 25 | class AppConfigs { 26 | final static String applicationID = "KTableAggDemo"; 27 | final static String bootstrapServers = "localhost:9092"; 28 | final static String topicName = "employees"; 29 | final static String stateStoreName = "state-store"; 30 | } 31 | -------------------------------------------------------------------------------- /ktable-aggregate/src/main/resources/data/data.txt: -------------------------------------------------------------------------------- 1 | 101:{"id": "101", "name": "Prashant", "department": "engineering", "salary": 5000} 2 | 102:{"id": "102", "name": "John", "department": "accounts", "salary": 8000} 3 | 103:{"id": "103", "name": "Abdul", "department": "engineering", "salary": 3000} 4 | 104:{"id": "104", "name": "Melinda", "department": "support", "salary": 7000} 5 | 105:{"id": "105", "name": "Jimmy", "department": "support", "salary": 6000} 6 | 101:{"id": "101", "name": "Prashant", "department": "support", "salary": 5000} 7 | 104:{"id": "104", "name": "Melinda", "department": "engineering", "salary": 7000} -------------------------------------------------------------------------------- /ktable-aggregate/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /ktable-aggregate/src/main/resources/schema/department_aggregates.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.DepartmentAggregate", 4 | "properties": { 5 | "total_salary": { 6 | "type": "integer" 7 | }, 8 | "employee_count": { 9 | "type": "integer" 10 | }, 11 | "avg_salary": { 12 | "type": "number" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /ktable-aggregate/src/main/resources/schema/employees.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.Employee", 4 | "properties": { 5 | "id": { 6 | "type": "string" 7 | }, 8 | "name": { 9 | "type": "string" 10 | }, 11 | "department": { 12 | "type": "string" 13 | }, 14 | "salary": { 15 | "type": "integer" 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /last-login/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic user-master & kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic user-login -------------------------------------------------------------------------------- /last-login/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /last-login/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /last-login/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /last-login/scripts/start-producer-user-login.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic user-login --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /last-login/scripts/start-producer-user-master.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic user-master --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /last-login/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /last-login/src/main/java/guru/learningjournal/kafka/examples/common/AppConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples.common; 17 | 18 | /** 19 | * Application Config 20 | * 21 | * @author prashant 22 | * @author www.learningjournal.guru 23 | */ 24 | 25 | public class AppConfigs { 26 | public final static String applicationID = "LastLoginDemo"; 27 | public final static String bootstrapServers = "localhost:9092,localhost:9093"; 28 | public final static String userMasterTopic = "user-master"; 29 | public final static String lastLoginTopic = "user-login"; 30 | public final static String stateStoreName = "logs/state-store"; 31 | } 32 | -------------------------------------------------------------------------------- /last-login/src/main/java/guru/learningjournal/kafka/examples/types/UserLogin.java: -------------------------------------------------------------------------------- 1 | 2 | package guru.learningjournal.kafka.examples.types; 3 | 4 | import com.fasterxml.jackson.annotation.JsonInclude; 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | import com.fasterxml.jackson.annotation.JsonPropertyOrder; 7 | import org.apache.commons.lang.builder.ToStringBuilder; 8 | 9 | @JsonInclude(JsonInclude.Include.NON_NULL) 10 | @JsonPropertyOrder({ 11 | "LoginID", 12 | "CreatedTime" 13 | }) 14 | public class UserLogin { 15 | 16 | @JsonProperty("LoginID") 17 | private String loginID; 18 | @JsonProperty("CreatedTime") 19 | private Long createdTime; 20 | 21 | @JsonProperty("LoginID") 22 | public String getLoginID() { 23 | return loginID; 24 | } 25 | 26 | @JsonProperty("LoginID") 27 | public void setLoginID(String loginID) { 28 | this.loginID = loginID; 29 | } 30 | 31 | public UserLogin withLoginID(String loginID) { 32 | this.loginID = loginID; 33 | return this; 34 | } 35 | 36 | @JsonProperty("CreatedTime") 37 | public Long getCreatedTime() { 38 | return createdTime; 39 | } 40 | 41 | @JsonProperty("CreatedTime") 42 | public void setCreatedTime(Long createdTime) { 43 | this.createdTime = createdTime; 44 | } 45 | 46 | public UserLogin withCreatedTime(Long createdTime) { 47 | this.createdTime = createdTime; 48 | return this; 49 | } 50 | 51 | @Override 52 | public String toString() { 53 | return new ToStringBuilder(this).append("loginID", loginID).append("createdTime", createdTime).toString(); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /last-login/src/main/resources/data/sample.txt: -------------------------------------------------------------------------------- 1 | 100001:{"UserName": "Prashant", "LoginID": "100001", "LastLogin": 1550150109302} 2 | 100009:{"UserName": "Alisha", "LoginID": "100009", "LastLogin": 1550150280409} 3 | 100087:{"UserName": "Abdul", "LoginID": "100087", "LastLogin": 1550150290305} 4 | 5 | 100001:{"LoginID": "100001", "CreatedTime": 1550150100000} 6 | 100087:{"LoginID": "100087", "CreatedTime": 1550150280000} 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /last-login/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /last-login/src/main/resources/schema/UserDetails.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.UserDetails", 4 | "properties": { 5 | "UserName": { 6 | "type": "string" 7 | }, 8 | "LoginID": { 9 | "type": "string" 10 | }, 11 | "LastLogin": { 12 | "javaType": "java.lang.Long" 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /last-login/src/main/resources/schema/UserLogins.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.UserLogin", 4 | "properties": { 5 | "LoginID": { 6 | "type": "string" 7 | }, 8 | "CreatedTime": { 9 | "javaType": "java.lang.Long" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /partitioned-producer/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | This directory contains Kafka Producer example. 4 | 5 | The demo application takes following Program Arguments: 6 | 7 | `partitioned-producer 10` -------------------------------------------------------------------------------- /partitioned-producer/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 1 --partitions 2 --topic partitioned-producer -------------------------------------------------------------------------------- /partitioned-producer/scripts/start-consumer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic partitioned-producer --from-beginning -------------------------------------------------------------------------------- /partitioned-producer/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\server-0.properties -------------------------------------------------------------------------------- /partitioned-producer/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\server-1.properties -------------------------------------------------------------------------------- /partitioned-producer/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\server-2.properties -------------------------------------------------------------------------------- /partitioned-producer/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\zookeeper.properties -------------------------------------------------------------------------------- /partitioned-producer/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /pojo-from-avro/README.md: -------------------------------------------------------------------------------- 1 | Generate Java Objects from Avro Schema Definition -------------------------------------------------------------------------------- /pojo-from-avro/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /pojo-from-avro/src/main/resources/schema/notification.avsc: -------------------------------------------------------------------------------- 1 | {"namespace": "guru.learningjournal.kafka.examples.types", 2 | "type": "record", 3 | "name": "Notification", 4 | "fields": [ 5 | {"name": "InvoiceNumber", "type": ["null", "string"]}, 6 | {"name": "CustomerCardNo", "type":["null", "string"]}, 7 | {"name": "TotalAmount", "type": ["null", "double"]}, 8 | {"name": "EarnedLoyaltyPoints", "type": ["null", "int"]} 9 | ] 10 | } 11 | 12 | -------------------------------------------------------------------------------- /pojo-from-json/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /pojo-from-json/src/main/resources/schema/notification.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.Notification", 4 | "properties": { 5 | "InvoiceNumber": { 6 | "type": "string" 7 | }, 8 | "CustomerCardNo": { 9 | "type": "string" 10 | }, 11 | "TotalAmount": { 12 | "type": "number" 13 | }, 14 | "EarnedLoyaltyPoints": { 15 | "type": "number" 16 | } 17 | } 18 | } 19 | 20 | -------------------------------------------------------------------------------- /pos-fanout-avro/README.md: -------------------------------------------------------------------------------- 1 | This directory contains Kafka Streams example. 2 | 3 | The project uses avro-maven-plugin to generate classes. 4 | 5 | Do not manually modify classes in types directory -------------------------------------------------------------------------------- /pos-fanout-avro/scripts/start-hadoop-sink-consumer.cmd: -------------------------------------------------------------------------------- 1 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic hadoop-sink --from-beginning -------------------------------------------------------------------------------- /pos-fanout-avro/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\kafka-server-start.bat C:\Users\prashant\Downloads\confluent-5.0.1\etc\kafka\server-0.properties 2 | -------------------------------------------------------------------------------- /pos-fanout-avro/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\kafka-server-start.bat C:\Users\prashant\Downloads\confluent-5.0.1\etc\kafka\server-1.properties -------------------------------------------------------------------------------- /pos-fanout-avro/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\kafka-server-start.bat C:\Users\prashant\Downloads\confluent-5.0.1\etc\kafka\server-2.properties -------------------------------------------------------------------------------- /pos-fanout-avro/scripts/start-schema-registry.cmd: -------------------------------------------------------------------------------- 1 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\schema-registry-start.bat C:\Users\prashant\Downloads\confluent-5.0.1\etc\schema-registry\schema-registry.properties -------------------------------------------------------------------------------- /pos-fanout-avro/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\zookeeper-server-start.bat C:\Users\prashant\Downloads\confluent-5.0.1\etc\kafka\zookeeper.properties -------------------------------------------------------------------------------- /pos-fanout-avro/src/main/java/guru/learningjournal/kafka/examples/FanOutConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples; 17 | 18 | /** 19 | * Configurations for the PosFanOutApp 20 | * 21 | * @author prashant 22 | * @author www.learningjournal.guru 23 | */ 24 | 25 | class FanOutConfigs { 26 | final static String applicationID = "PosFanOutApp"; 27 | final static String bootstrapServers = "localhost:9092"; 28 | final static String posTopicName = "pos"; 29 | final static String shipmentTopicName = "shipment"; 30 | final static String notificationTopic = "loyalty"; 31 | final static String hadoopTopic = "hadoop-sink"; 32 | final static String DELIVERY_TYPE_HOME_DELIVERY = "HOME-DELIVERY"; 33 | final static String CUSTOMER_TYPE_PRIME = "PRIME"; 34 | final static Double LOYALTY_FACTOR = 0.02; 35 | } -------------------------------------------------------------------------------- /pos-fanout-avro/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /pos-fanout-avro/src/main/resources/schema/HadoopRecord.avsc: -------------------------------------------------------------------------------- 1 | {"namespace": "guru.learningjournal.kafka.examples.types", 2 | "type": "record", 3 | "name": "HadoopRecord", 4 | "fields": [ 5 | {"name": "InvoiceNumber", "type": ["null", "string"]}, 6 | {"name": "CreatedTime", "type":["null", "long"]}, 7 | {"name": "StoreID", "type": ["null", "string"]}, 8 | {"name": "PosID", "type": ["null", "string"]}, 9 | {"name": "CustomerType", "type": ["null", "string"]}, 10 | {"name": "PaymentMethod", "type": ["null", "string"]}, 11 | {"name": "DeliveryType", "type": ["null", "string"]}, 12 | {"name": "City", "type": ["null", "string"]}, 13 | {"name": "State", "type": ["null", "string"]}, 14 | {"name": "PinCode", "type": ["null", "string"]}, 15 | {"name": "ItemCode", "type": ["null", "string"]}, 16 | {"name": "ItemDescription", "type": ["null", "string"]}, 17 | {"name": "ItemPrice", "type": ["null", "double"]}, 18 | {"name": "ItemQty", "type": ["null", "int"]}, 19 | {"name": "TotalValue", "type": ["null", "double"]} 20 | ] 21 | } -------------------------------------------------------------------------------- /pos-fanout-avro/src/main/resources/schema/Notification.avsc: -------------------------------------------------------------------------------- 1 | {"namespace": "guru.learningjournal.kafka.examples.types", 2 | "type": "record", 3 | "name": "Notification", 4 | "fields": [ 5 | {"name": "InvoiceNumber", "type": ["null", "string"]}, 6 | {"name": "CustomerCardNo", "type":["null", "string"]}, 7 | {"name": "TotalAmount", "type": ["null", "double"]}, 8 | {"name": "EarnedLoyaltyPoints", "type": ["null", "double"]} 9 | ] 10 | } -------------------------------------------------------------------------------- /pos-fanout/README.md: -------------------------------------------------------------------------------- 1 | This directory contains Kafka Streams example. 2 | 3 | The project uses JsonSchema2Pojo maven plugin to generate classes. 4 | Do not manually modify classes in types directory -------------------------------------------------------------------------------- /pos-fanout/scripts/create-hadoop-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic hadoop-sink --config min.insync.replicas=2 -------------------------------------------------------------------------------- /pos-fanout/scripts/create-loyalty-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic loyalty --config min.insync.replicas=2 -------------------------------------------------------------------------------- /pos-fanout/scripts/create-shipment-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic shipment --config min.insync.replicas=2 -------------------------------------------------------------------------------- /pos-fanout/scripts/start-hadoop-sink-consumer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic hadoop-sink --from-beginning -------------------------------------------------------------------------------- /pos-fanout/scripts/start-loyalty-consumer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic loyalty --from-beginning -------------------------------------------------------------------------------- /pos-fanout/scripts/start-shipment-consumer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic shipment --from-beginning -------------------------------------------------------------------------------- /pos-fanout/src/main/java/guru/learningjournal/kafka/examples/FanOutConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples; 17 | 18 | /** 19 | * Configurations for the PosFanOutApp 20 | * 21 | * @author prashant 22 | * @author www.learningjournal.guru 23 | */ 24 | 25 | class FanOutConfigs { 26 | final static String applicationID = "PosFanOutApp"; 27 | final static String bootstrapServers = "localhost:9092"; 28 | final static String posTopicName = "pos"; 29 | final static String shipmentTopicName = "shipment"; 30 | final static String notificationTopic = "loyalty"; 31 | final static String hadoopTopic = "hadoop-sink"; 32 | final static String DELIVERY_TYPE_HOME_DELIVERY = "HOME-DELIVERY"; 33 | final static String CUSTOMER_TYPE_PRIME = "PRIME"; 34 | final static Double LOYALTY_FACTOR = 0.02; 35 | } 36 | -------------------------------------------------------------------------------- /pos-fanout/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /pos-fanout/src/main/resources/schema/HadoopRecord.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.HadoopRecord", 4 | "properties": { 5 | "InvoiceNumber": { 6 | "type": "string" 7 | }, 8 | "CreatedTime": { 9 | "javaType": "java.lang.Long", 10 | "type": "object" 11 | }, 12 | "StoreID": { 13 | "type": "string" 14 | }, 15 | "PosID": { 16 | "type": "string" 17 | }, 18 | "CustomerType": { 19 | "type": "string" 20 | }, 21 | "PaymentMethod": { 22 | "type": "string" 23 | }, 24 | "DeliveryType": { 25 | "type": "string" 26 | }, 27 | "City": { 28 | "type": "string" 29 | }, 30 | "State": { 31 | "type": "string" 32 | }, 33 | "PinCode": { 34 | "type": "string" 35 | }, 36 | "ItemCode": { 37 | "type": "string" 38 | }, 39 | "ItemDescription": { 40 | "type": "string" 41 | }, 42 | "ItemPrice": { 43 | "type": "number" 44 | }, 45 | "ItemQty": { 46 | "type": "integer" 47 | }, 48 | "TotalValue": { 49 | "type": "number" 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /pos-fanout/src/main/resources/schema/Notification.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.Notification", 4 | "properties": { 5 | "InvoiceNumber": { 6 | "type": "string" 7 | }, 8 | "CustomerCardNo": { 9 | "type": "string" 10 | }, 11 | "TotalAmount": { 12 | "type": "number" 13 | }, 14 | "EarnedLoyaltyPoints": { 15 | "type": "number" 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /pos-simulator-avro/README.md: -------------------------------------------------------------------------------- 1 | POS Simulator that generates random invoices in Avro 2 | 3 | The demo application takes following Program Arguments: 4 | 5 | pos 2 15000 -------------------------------------------------------------------------------- /pos-simulator-avro/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 3 --topic pos --config min.insync.replicas=2 -------------------------------------------------------------------------------- /pos-simulator-avro/scripts/start-consumer.cmd: -------------------------------------------------------------------------------- 1 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic pos --from-beginning 2 | -------------------------------------------------------------------------------- /pos-simulator-avro/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\kafka-server-start.bat C:\Users\prashant\Downloads\confluent-5.0.1\etc\kafka\server-0.properties 2 | -------------------------------------------------------------------------------- /pos-simulator-avro/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\kafka-server-start.bat C:\Users\prashant\Downloads\confluent-5.0.1\etc\kafka\server-1.properties -------------------------------------------------------------------------------- /pos-simulator-avro/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\kafka-server-start.bat C:\Users\prashant\Downloads\confluent-5.0.1\etc\kafka\server-2.properties -------------------------------------------------------------------------------- /pos-simulator-avro/scripts/start-schema-registry.cmd: -------------------------------------------------------------------------------- 1 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\schema-registry-start.bat C:\Users\prashant\Downloads\confluent-5.0.1\etc\schema-registry\schema-registry.properties -------------------------------------------------------------------------------- /pos-simulator-avro/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | C:\Users\prashant\Downloads\confluent-5.0.1\bin\windows\zookeeper-server-start.bat C:\Users\prashant\Downloads\confluent-5.0.1\etc\kafka\zookeeper.properties -------------------------------------------------------------------------------- /pos-simulator-avro/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /pos-simulator/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | This directory contains Kafka Producer example. 4 | 5 | The demo application takes following Program Arguments: 6 | 7 | `pos 2 1000` -------------------------------------------------------------------------------- /pos-simulator/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 3 --topic pos --config min.insync.replicas=2 -------------------------------------------------------------------------------- /pos-simulator/scripts/start-consumer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic pos --from-beginning -------------------------------------------------------------------------------- /pos-simulator/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /pos-simulator/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /pos-simulator/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /pos-simulator/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /pos-simulator/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /producer-threads/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic nse-bhav -------------------------------------------------------------------------------- /producer-threads/scripts/start-consumer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic nse-bhav --from-beginning -------------------------------------------------------------------------------- /producer-threads/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /producer-threads/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /producer-threads/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /producer-threads/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /producer-threads/src/main/resources/kafka.properties: -------------------------------------------------------------------------------- 1 | client.id=NSE_BHAV_DISPATCHER 2 | bootstrap.servers=localhost:9092,localhost:9093 -------------------------------------------------------------------------------- /producer-threads/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /rewards-in-memory/scripts/create-intermediate-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic rewards-intermediate --config min.insync.replicas=2 -------------------------------------------------------------------------------- /rewards-in-memory/scripts/create-loyalty-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic loyalty --config min.insync.replicas=2 -------------------------------------------------------------------------------- /rewards-in-memory/scripts/start-loyalty-consumer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic loyalty --from-beginning -------------------------------------------------------------------------------- /rewards-in-memory/src/main/java/guru/learningjournal/kafka/examples/AppConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples; 17 | 18 | /** 19 | * Configurations for the PosFanOutApp 20 | * 21 | * @author prashant 22 | * @author www.learningjournal.guru 23 | */ 24 | class AppConfigs { 25 | 26 | final static String applicationID = "RewardsApp"; 27 | final static String bootstrapServers = "localhost:9092"; 28 | final static String posTopicName = "pos"; 29 | final static String notificationTopic = "loyalty"; 30 | final static String CUSTOMER_TYPE_PRIME = "PRIME"; 31 | final static Double LOYALTY_FACTOR = 0.02; 32 | final static String REWARDS_STORE_NAME = "CustomerRewardsStore"; 33 | } 34 | -------------------------------------------------------------------------------- /rewards-in-memory/src/main/java/guru/learningjournal/kafka/examples/RecordBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples; 17 | 18 | import guru.learningjournal.kafka.examples.types.Notification; 19 | import guru.learningjournal.kafka.examples.types.PosInvoice; 20 | 21 | 22 | /** 23 | * Transform Invoice to other Objects 24 | * 25 | * @author prashant 26 | * @author www.learningjournal.guru 27 | */ 28 | 29 | class RecordBuilder { 30 | 31 | /** 32 | * Transform PosInvoice to Notification 33 | * 34 | * @param invoice PosInvoice Object 35 | * @return Notification Object 36 | */ 37 | static Notification getNotification(PosInvoice invoice) { 38 | return new Notification() 39 | .withInvoiceNumber(invoice.getInvoiceNumber()) 40 | .withCustomerCardNo(invoice.getCustomerCardNo()) 41 | .withTotalAmount(invoice.getTotalAmount()) 42 | .withEarnedLoyaltyPoints(invoice.getTotalAmount() * AppConfigs.LOYALTY_FACTOR) 43 | .withTotalLoyaltyPoints(0.0); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /rewards-in-memory/src/main/java/guru/learningjournal/kafka/examples/RewardsPartitioner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples; 17 | 18 | import guru.learningjournal.kafka.examples.types.PosInvoice; 19 | import org.apache.kafka.streams.processor.StreamPartitioner; 20 | 21 | /** 22 | * Partitioner for customer ID 23 | * 24 | * @author prashant 25 | * @author www.learningjournal.guru 26 | */ 27 | 28 | public class RewardsPartitioner implements StreamPartitioner { 29 | @Override 30 | public Integer partition(String topic, String key, 31 | PosInvoice value, int numPartitions) { 32 | 33 | return value.getCustomerCardNo().hashCode() % numPartitions; 34 | } 35 | } 36 | 37 | 38 | -------------------------------------------------------------------------------- /rewards-in-memory/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /rewards-in-memory/src/main/resources/schema/Notification.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.Notification", 4 | "properties": { 5 | "InvoiceNumber": { 6 | "type": "string" 7 | }, 8 | "CustomerCardNo": { 9 | "type": "string" 10 | }, 11 | "TotalAmount": { 12 | "type": "number" 13 | }, 14 | "EarnedLoyaltyPoints": { 15 | "type": "number" 16 | }, 17 | "TotalLoyaltyPoints": { 18 | "type": "number" 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /rewards-table/scripts/create-loyalty-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic loyalty --config min.insync.replicas=2 -------------------------------------------------------------------------------- /rewards-table/scripts/start-loyalty-consumer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic loyalty --from-beginning -------------------------------------------------------------------------------- /rewards-table/src/main/java/guru/learningjournal/kafka/examples/AppConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples; 17 | 18 | /** 19 | * Configurations for the PosFanOutApp 20 | * 21 | * @author prashant 22 | * @author www.learningjournal.guru 23 | */ 24 | class AppConfigs { 25 | 26 | final static String applicationID = "RewardsApp"; 27 | final static String bootstrapServers = "localhost:9092"; 28 | final static String posTopicName = "pos"; 29 | final static String notificationTopic = "loyalty"; 30 | final static String CUSTOMER_TYPE_PRIME = "PRIME"; 31 | final static Double LOYALTY_FACTOR = 0.02; 32 | } 33 | -------------------------------------------------------------------------------- /rewards-table/src/main/java/guru/learningjournal/kafka/examples/Notifications.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples; 17 | 18 | 19 | import guru.learningjournal.kafka.examples.types.Notification; 20 | import guru.learningjournal.kafka.examples.types.PosInvoice; 21 | 22 | /** 23 | * Utility class for Notification 24 | * 25 | * @author prashant 26 | * @author www.learningjournal.guru 27 | */ 28 | 29 | class Notifications { 30 | 31 | static Notification getNotificationFrom(PosInvoice invoice) { 32 | return new Notification() 33 | .withInvoiceNumber(invoice.getInvoiceNumber()) 34 | .withCustomerCardNo(invoice.getCustomerCardNo()) 35 | .withTotalAmount(invoice.getTotalAmount()) 36 | .withEarnedLoyaltyPoints(invoice.getTotalAmount() * AppConfigs.LOYALTY_FACTOR) 37 | .withTotalLoyaltyPoints(invoice.getTotalAmount() * AppConfigs.LOYALTY_FACTOR); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /rewards-table/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /rewards-table/src/main/resources/schema/Notification.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.Notification", 4 | "properties": { 5 | "InvoiceNumber": { 6 | "type": "string" 7 | }, 8 | "CustomerCardNo": { 9 | "type": "string" 10 | }, 11 | "TotalAmount": { 12 | "type": "number" 13 | }, 14 | "EarnedLoyaltyPoints": { 15 | "type": "number" 16 | }, 17 | "TotalLoyaltyPoints": { 18 | "type": "number" 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /stream-join/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic payment_request & kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic payment_confirmation -------------------------------------------------------------------------------- /stream-join/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /stream-join/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /stream-join/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /stream-join/scripts/start-producer_pay_confirm.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic payment_confirmation --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /stream-join/scripts/start-producer_pay_request.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic payment_request --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /stream-join/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /stream-join/src/main/java/guru/learningjournal/kafka/examples/common/AppConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples.common; 17 | 18 | /** 19 | * Application Config 20 | * 21 | * @author prashant 22 | * @author www.learningjournal.guru 23 | */ 24 | 25 | public class AppConfigs { 26 | public final static String applicationID = "StreamJoinDemo"; 27 | public final static String bootstrapServers = "localhost:9092,localhost:9093"; 28 | public final static String paymentRequestTopicName = "payment_request"; 29 | public final static String paymentConfirmationTopicName = "payment_confirmation"; 30 | public final static String stateStoreName = "logs/state-store"; 31 | } 32 | -------------------------------------------------------------------------------- /stream-join/src/main/java/guru/learningjournal/kafka/examples/types/TransactionStatus.java: -------------------------------------------------------------------------------- 1 | 2 | package guru.learningjournal.kafka.examples.types; 3 | 4 | import com.fasterxml.jackson.annotation.JsonInclude; 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | import com.fasterxml.jackson.annotation.JsonPropertyOrder; 7 | import org.apache.commons.lang.builder.ToStringBuilder; 8 | 9 | @JsonInclude(JsonInclude.Include.NON_NULL) 10 | @JsonPropertyOrder({ 11 | "TransactionID", 12 | "Status" 13 | }) 14 | public class TransactionStatus { 15 | 16 | @JsonProperty("TransactionID") 17 | private String transactionID; 18 | @JsonProperty("Status") 19 | private String status; 20 | 21 | @JsonProperty("TransactionID") 22 | public String getTransactionID() { 23 | return transactionID; 24 | } 25 | 26 | @JsonProperty("TransactionID") 27 | public void setTransactionID(String transactionID) { 28 | this.transactionID = transactionID; 29 | } 30 | 31 | public TransactionStatus withTransactionID(String transactionID) { 32 | this.transactionID = transactionID; 33 | return this; 34 | } 35 | 36 | @JsonProperty("Status") 37 | public String getStatus() { 38 | return status; 39 | } 40 | 41 | @JsonProperty("Status") 42 | public void setStatus(String status) { 43 | this.status = status; 44 | } 45 | 46 | public TransactionStatus withStatus(String status) { 47 | this.status = status; 48 | return this; 49 | } 50 | 51 | @Override 52 | public String toString() { 53 | return new ToStringBuilder(this).append("transactionID", transactionID).append("status", status).toString(); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /stream-join/src/main/resources/data/sample.txt: -------------------------------------------------------------------------------- 1 | 100001:{"TransactionID": "100001", "CreatedTime": 1550149860000, "SourceAccountID": "131100", "TargetAccountID": "151837", "Amount": 3000, "OTP": 852960} 2 | 100002:{"TransactionID": "100002", "CreatedTime": 1550149920000, "SourceAccountID": "131200", "TargetAccountID": "151837", "Amount": 2000, "OTP": 931749} 3 | 100003:{"TransactionID": "100003", "CreatedTime": 1550149980000, "SourceAccountID": "131300", "TargetAccountID": "151837", "Amount": 5000, "OTP": 591296} 4 | 100004:{"TransactionID": "100004", "CreatedTime": 1550150100000, "SourceAccountID": "131400", "TargetAccountID": "151837", "Amount": 1000, "OTP": 283084} 5 | 6 | 100001:{"TransactionID": "100001", "CreatedTime": 1550150100000, "OTP": 852960} 7 | 100002:{"TransactionID": "100001", "CreatedTime": 1550150280000, "OTP": 931749} 8 | 100004:{"TransactionID": "100001", "CreatedTime": 1550150040000, "OTP": 283086} -------------------------------------------------------------------------------- /stream-join/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /stream-join/src/main/resources/schema/PaymentConfirmation.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.PaymentConfirmation", 4 | "properties": { 5 | "TransactionID": { 6 | "type": "string" 7 | }, 8 | "CreatedTime": { 9 | "javaType": "java.lang.Long" 10 | }, 11 | "OTP": { 12 | "type": "string" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /stream-join/src/main/resources/schema/PaymentRequest.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.PaymentRequest", 4 | "properties": { 5 | "TransactionID": { 6 | "type": "string" 7 | }, 8 | "CreatedTime": { 9 | "javaType": "java.lang.Long" 10 | }, 11 | "SourceAccountID": { 12 | "type": "string" 13 | }, 14 | "TargetAccountID": { 15 | "type": "string" 16 | }, 17 | "Amount": { 18 | "type": "number" 19 | }, 20 | "OTP": { 21 | "type": "string" 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /stream-join/src/main/resources/schema/TransactionStatus.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.TransactionStatus", 4 | "properties": { 5 | "TransactionID": { 6 | "type": "string" 7 | }, 8 | "Status": { 9 | "type": "string" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /streaming-table/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic stock-tick -------------------------------------------------------------------------------- /streaming-table/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /streaming-table/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /streaming-table/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /streaming-table/scripts/start-producer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic stock-tick --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /streaming-table/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /streaming-table/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /streaming-word-count/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 3 --topic streaming-word-count --config min.insync.replicas=2 -------------------------------------------------------------------------------- /streaming-word-count/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /streaming-word-count/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /streaming-word-count/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /streaming-word-count/scripts/start-producer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic streaming-word-count -------------------------------------------------------------------------------- /streaming-word-count/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /streaming-word-count/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /sync-producer/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | This directory contains Kafka Producer example. 4 | 5 | The demo application takes following Program Arguments: 6 | 7 | `hello-producer 10` -------------------------------------------------------------------------------- /sync-producer/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic hello-producer -------------------------------------------------------------------------------- /sync-producer/scripts/start-consumer.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic hello-producer --from-beginning -------------------------------------------------------------------------------- /sync-producer/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\server-0.properties -------------------------------------------------------------------------------- /sync-producer/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\server-1.properties -------------------------------------------------------------------------------- /sync-producer/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\server-2.properties -------------------------------------------------------------------------------- /sync-producer/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\zookeeper.properties -------------------------------------------------------------------------------- /sync-producer/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /time-check/src/main/java/guru/learningjournal/kafka/examples/AppConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples; 17 | 18 | /** 19 | * Application Configs 20 | * 21 | * @author prashant 22 | * @author www.learningjournal.guru 23 | */ 24 | class AppConfigs { 25 | 26 | final static String applicationID = "TimeCheckDemo"; 27 | final static String bootstrapServers = "localhost:9092"; 28 | final static String posTopicName = "pos"; 29 | } 30 | -------------------------------------------------------------------------------- /time-check/src/main/java/guru/learningjournal/kafka/examples/InvoiceTimeExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples; 17 | 18 | import guru.learningjournal.kafka.examples.types.PosInvoice; 19 | import org.apache.kafka.clients.consumer.ConsumerRecord; 20 | import org.apache.kafka.streams.processor.TimestampExtractor; 21 | 22 | /** 23 | * Extract timestamp from Invoice field 24 | * 25 | * @author prashant 26 | * @author www.learningjournal.guru 27 | */ 28 | 29 | public class InvoiceTimeExtractor implements TimestampExtractor { 30 | 31 | @Override 32 | public long extract(ConsumerRecord consumerRecord, long prevTime) { 33 | PosInvoice invoice = (PosInvoice) consumerRecord.value(); 34 | return ((invoice.getCreatedTime() > 0) ? invoice.getCreatedTime() : prevTime); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /time-check/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /top3-inventories/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic active-inventories & kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic ad-clicks -------------------------------------------------------------------------------- /top3-inventories/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /top3-inventories/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /top3-inventories/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /top3-inventories/scripts/start-producer-inventories.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic active-inventories --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /top3-inventories/scripts/start-producer_ad-clicks.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic ad-clicks --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /top3-inventories/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /top3-inventories/src/main/java/guru/learningjournal/kafka/examples/common/AppConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples.common; 17 | 18 | /** 19 | * Application Config 20 | * 21 | * @author prashant 22 | * @author www.learningjournal.guru 23 | */ 24 | 25 | public class AppConfigs { 26 | public final static String applicationID = "Top3InventoriesDemo"; 27 | public final static String bootstrapServers = "localhost:9092,localhost:9093"; 28 | public final static String inventoryTopic = "active-inventories"; 29 | public final static String clicksTopic = "ad-clicks"; 30 | public final static String stateStoreName = "logs/state-store"; 31 | public final static String top3AggregateKey = "top3NewsTypes"; 32 | } 33 | -------------------------------------------------------------------------------- /top3-inventories/src/main/java/guru/learningjournal/kafka/examples/types/AdClick.java: -------------------------------------------------------------------------------- 1 | 2 | package guru.learningjournal.kafka.examples.types; 3 | 4 | import com.fasterxml.jackson.annotation.JsonInclude; 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | import com.fasterxml.jackson.annotation.JsonPropertyOrder; 7 | import org.apache.commons.lang.builder.ToStringBuilder; 8 | 9 | @JsonInclude(JsonInclude.Include.NON_NULL) 10 | @JsonPropertyOrder({ 11 | "InventoryID" 12 | }) 13 | public class AdClick { 14 | 15 | @JsonProperty("InventoryID") 16 | private String inventoryID; 17 | 18 | @JsonProperty("InventoryID") 19 | public String getInventoryID() { 20 | return inventoryID; 21 | } 22 | 23 | @JsonProperty("InventoryID") 24 | public void setInventoryID(String inventoryID) { 25 | this.inventoryID = inventoryID; 26 | } 27 | 28 | public AdClick withInventoryID(String inventoryID) { 29 | this.inventoryID = inventoryID; 30 | return this; 31 | } 32 | 33 | @Override 34 | public String toString() { 35 | return new ToStringBuilder(this).append("inventoryID", inventoryID).toString(); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /top3-inventories/src/main/java/guru/learningjournal/kafka/examples/types/AdInventories.java: -------------------------------------------------------------------------------- 1 | 2 | package guru.learningjournal.kafka.examples.types; 3 | 4 | import com.fasterxml.jackson.annotation.JsonInclude; 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | import com.fasterxml.jackson.annotation.JsonPropertyOrder; 7 | import org.apache.commons.lang.builder.ToStringBuilder; 8 | 9 | @JsonInclude(JsonInclude.Include.NON_NULL) 10 | @JsonPropertyOrder({ 11 | "InventoryID", 12 | "NewsType" 13 | }) 14 | public class AdInventories { 15 | 16 | @JsonProperty("InventoryID") 17 | private String inventoryID; 18 | @JsonProperty("NewsType") 19 | private String newsType; 20 | 21 | @JsonProperty("InventoryID") 22 | public String getInventoryID() { 23 | return inventoryID; 24 | } 25 | 26 | @JsonProperty("InventoryID") 27 | public void setInventoryID(String inventoryID) { 28 | this.inventoryID = inventoryID; 29 | } 30 | 31 | public AdInventories withInventoryID(String inventoryID) { 32 | this.inventoryID = inventoryID; 33 | return this; 34 | } 35 | 36 | @JsonProperty("NewsType") 37 | public String getNewsType() { 38 | return newsType; 39 | } 40 | 41 | @JsonProperty("NewsType") 42 | public void setNewsType(String newsType) { 43 | this.newsType = newsType; 44 | } 45 | 46 | public AdInventories withNewsType(String newsType) { 47 | this.newsType = newsType; 48 | return this; 49 | } 50 | 51 | @Override 52 | public String toString() { 53 | return new ToStringBuilder(this).append("inventoryID", inventoryID).append("newsType", newsType).toString(); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /top3-inventories/src/main/java/guru/learningjournal/kafka/examples/types/ClicksByNewsType.java: -------------------------------------------------------------------------------- 1 | 2 | package guru.learningjournal.kafka.examples.types; 3 | 4 | import com.fasterxml.jackson.annotation.JsonInclude; 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | import com.fasterxml.jackson.annotation.JsonPropertyOrder; 7 | import org.apache.commons.lang.builder.ToStringBuilder; 8 | 9 | @JsonInclude(JsonInclude.Include.NON_NULL) 10 | @JsonPropertyOrder({ 11 | "NewsType", 12 | "Clicks" 13 | }) 14 | public class ClicksByNewsType { 15 | 16 | @JsonProperty("NewsType") 17 | private String newsType; 18 | @JsonProperty("Clicks") 19 | private Long clicks; 20 | 21 | @JsonProperty("NewsType") 22 | public String getNewsType() { 23 | return newsType; 24 | } 25 | 26 | @JsonProperty("NewsType") 27 | public void setNewsType(String newsType) { 28 | this.newsType = newsType; 29 | } 30 | 31 | public ClicksByNewsType withNewsType(String newsType) { 32 | this.newsType = newsType; 33 | return this; 34 | } 35 | 36 | @JsonProperty("Clicks") 37 | public Long getClicks() { 38 | return clicks; 39 | } 40 | 41 | @JsonProperty("Clicks") 42 | public void setClicks(Long clicks) { 43 | this.clicks = clicks; 44 | } 45 | 46 | public ClicksByNewsType withClicks(Long clicks) { 47 | this.clicks = clicks; 48 | return this; 49 | } 50 | 51 | @Override 52 | public String toString() { 53 | return new ToStringBuilder(this).append("newsType", newsType).append("clicks", clicks).toString(); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /top3-inventories/src/main/resources/data/sample.txt: -------------------------------------------------------------------------------- 1 | 1001:{"InventoryID": "1001", "NewsType": "Sports"} 2 | 1002:{"InventoryID": "1002", "NewsType": "Politics"} 3 | 1003:{"InventoryID": "1003", "NewsType": "LocalNews"} 4 | 1004:{"InventoryID": "1004", "NewsType": "WorldNews"} 5 | 1005:{"InventoryID": "1005", "NewsType": "Health"} 6 | 1006:{"InventoryID": "1006", "NewsType": "Lifestyle"} 7 | 1007:{"InventoryID": "1007", "NewsType": "Literature"} 8 | 1008:{"InventoryID": "1008", "NewsType": "Education"} 9 | 1009:{"InventoryID": "1009", "NewsType": "Social"} 10 | 1010:{"InventoryID": "1010", "NewsType": "Business"} 11 | 12 | 13 | 1001:{"InventoryID": "1001"} 14 | 1002:{"InventoryID": "1002"} 15 | 1003:{"InventoryID": "1003"} 16 | 1004:{"InventoryID": "1004"} 17 | 1004:{"InventoryID": "1004"} 18 | 1005:{"InventoryID": "1005"} 19 | 1006:{"InventoryID": "1006"} 20 | 1007:{"InventoryID": "1007"} 21 | 1008:{"InventoryID": "1008"} 22 | 1009:{"InventoryID": "1009"} 23 | 1010:{"InventoryID": "1010"} 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /top3-inventories/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /top3-inventories/src/main/resources/schema/AdClick.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.AdClick", 4 | "properties": { 5 | "InventoryID": { 6 | "type": "string" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /top3-inventories/src/main/resources/schema/AdInventories.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.AdInventories", 4 | "properties": { 5 | "InventoryID": { 6 | "type": "string" 7 | }, 8 | "NewsType": { 9 | "type": "string" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /top3-inventories/src/main/resources/schema/ClicksByNewsType.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.ClicksByNewsType", 4 | "properties": { 5 | "NewsType": { 6 | "type": "string" 7 | }, 8 | "Clicks": { 9 | "javaType": "java.lang.Long" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /topology-test/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic ad-impressions & kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic ad-clicks & kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 2 --topic campaign-performance scripts -------------------------------------------------------------------------------- /topology-test/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-0.properties -------------------------------------------------------------------------------- /topology-test/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-1.properties -------------------------------------------------------------------------------- /topology-test/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat %KAFKA_HOME%\config\server-2.properties -------------------------------------------------------------------------------- /topology-test/scripts/start-producer_ad-clicks.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic ad-clicks --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /topology-test/scripts/start-producer_ad-impressions.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-producer.bat --broker-list localhost:9092 --topic ad-impressions --property parse.key=true --property key.separator=":" -------------------------------------------------------------------------------- /topology-test/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat %KAFKA_HOME%\config\zookeeper.properties -------------------------------------------------------------------------------- /topology-test/src/main/java/guru/learningjournal/kafka/examples/common/AppConfigs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019. Prashant Kumar Pandey 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, 11 | * software distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package guru.learningjournal.kafka.examples.common; 17 | 18 | /** 19 | * Application Config 20 | * 21 | * @author prashant 22 | * @author www.learningjournal.guru 23 | */ 24 | 25 | public class AppConfigs { 26 | public final static String applicationID = "AdCampaignsDemo"; 27 | public final static String bootstrapServers = "localhost:9092,localhost:9093"; 28 | public final static String impressionTopic = "ad-impressions"; 29 | public final static String clicksTopic = "ad-clicks"; 30 | public final static String outputTopic = "campaign-performance"; 31 | public final static String stateStoreLocation = "logs/state-store"; 32 | public final static String stateStoreLocationUT = "logs-ut/state-store"; 33 | public final static String stateStoreNameCP = "cp-store"; 34 | } 35 | -------------------------------------------------------------------------------- /topology-test/src/main/java/guru/learningjournal/kafka/examples/types/AdClick.java: -------------------------------------------------------------------------------- 1 | 2 | package guru.learningjournal.kafka.examples.types; 3 | 4 | import com.fasterxml.jackson.annotation.JsonInclude; 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | import com.fasterxml.jackson.annotation.JsonPropertyOrder; 7 | import org.apache.commons.lang.builder.ToStringBuilder; 8 | 9 | @JsonInclude(JsonInclude.Include.NON_NULL) 10 | @JsonPropertyOrder({ 11 | "ImpressionID", 12 | "Campaigner" 13 | }) 14 | public class AdClick { 15 | 16 | @JsonProperty("ImpressionID") 17 | private String impressionID; 18 | @JsonProperty("Campaigner") 19 | private String campaigner; 20 | 21 | @JsonProperty("ImpressionID") 22 | public String getImpressionID() { 23 | return impressionID; 24 | } 25 | 26 | @JsonProperty("ImpressionID") 27 | public void setImpressionID(String impressionID) { 28 | this.impressionID = impressionID; 29 | } 30 | 31 | public AdClick withImpressionID(String impressionID) { 32 | this.impressionID = impressionID; 33 | return this; 34 | } 35 | 36 | @JsonProperty("Campaigner") 37 | public String getCampaigner() { 38 | return campaigner; 39 | } 40 | 41 | @JsonProperty("Campaigner") 42 | public void setCampaigner(String campaigner) { 43 | this.campaigner = campaigner; 44 | } 45 | 46 | public AdClick withCampaigner(String campaigner) { 47 | this.campaigner = campaigner; 48 | return this; 49 | } 50 | 51 | @Override 52 | public String toString() { 53 | return new ToStringBuilder(this).append("impressionID", impressionID).append("campaigner", campaigner).toString(); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /topology-test/src/main/resources/data/sample.txt: -------------------------------------------------------------------------------- 1 | 100001:{"ImpressionID": "100001", "Campaigner": "ABC Ltd"} 2 | 100002:{"ImpressionID": "100002", "Campaigner": "XYZ Ltd"} 3 | 100003:{"ImpressionID": "100003", "Campaigner": "XYZ Ltd"} 4 | 100004:{"ImpressionID": "100004", "Campaigner": "XYZ Ltd"} 5 | 100005:{"ImpressionID": "100005", "Campaigner": "XYZ Ltd"} 6 | 100006:{"ImpressionID": "100006", "Campaigner": "ABC Ltd"} 7 | 100007:{"ImpressionID": "100007", "Campaigner": "ABC Ltd"} 8 | 100008:{"ImpressionID": "100008", "Campaigner": "ABC Ltd"} 9 | 100009:{"ImpressionID": "100009", "Campaigner": "ABC Ltd"} 10 | 100010:{"ImpressionID": "100010", "Campaigner": "ABC Ltd"} 11 | 12 | 100009:{"ImpressionID": "100009", "Campaigner": "ABC Ltd"} 13 | 100010:{"ImpressionID": "100010", "Campaigner": "ABC Ltd"} 14 | 100002:{"ImpressionID": "100002", "Campaigner": "XYZ Ltd"} -------------------------------------------------------------------------------- /topology-test/src/main/resources/schema/AdClick.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.AdClick", 4 | "properties": { 5 | "ImpressionID": { 6 | "type": "string" 7 | }, 8 | "Campaigner": { 9 | "type": "string" 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /topology-test/src/main/resources/schema/AdImpression.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.AdImpression", 4 | "properties": { 5 | "ImpressionID": { 6 | "type": "string" 7 | }, 8 | "Campaigner": { 9 | "type": "string" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /topology-test/src/main/resources/schema/CampaignPerformance.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "javaType": "guru.learningjournal.kafka.examples.types.CampaignPerformance", 4 | "properties": { 5 | "Campaigner": { 6 | "type": "string" 7 | }, 8 | "AdImpressions": { 9 | "javaType": "java.lang.Long" 10 | }, 11 | "AdClicks": { 12 | "javaType": "java.lang.Long" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /transactional-producer/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | This directory contains Kafka Producer example. 4 | 5 | The demo application takes following Program Arguments: 6 | 7 | `nse05 nse06 /data/NSE05NOV2018BHAV.csv /data/NSE06NOV2018BHAV.csv` -------------------------------------------------------------------------------- /transactional-producer/scripts/create-topic.cmd: -------------------------------------------------------------------------------- 1 | kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 1 --topic nse05 --config min.insync.replicas=2 & kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 3 --partitions 1 --topic nse06 --config min.insync.replicas=2 -------------------------------------------------------------------------------- /transactional-producer/scripts/start-consumer1.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic nse05 --from-beginning --isolation-level read_committed -------------------------------------------------------------------------------- /transactional-producer/scripts/start-consumer2.cmd: -------------------------------------------------------------------------------- 1 | kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic nse06 --from-beginning --isolation-level read_committed -------------------------------------------------------------------------------- /transactional-producer/scripts/start-kafka-server-0.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\server-0.properties -------------------------------------------------------------------------------- /transactional-producer/scripts/start-kafka-server-1.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\server-1.properties -------------------------------------------------------------------------------- /transactional-producer/scripts/start-kafka-server-2.cmd: -------------------------------------------------------------------------------- 1 | kafka-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\server-2.properties -------------------------------------------------------------------------------- /transactional-producer/scripts/start-zookeeper.cmd: -------------------------------------------------------------------------------- 1 | zookeeper-server-start.bat C:\Users\prashant\Downloads\kafka_2.12-2.0.0\config\zookeeper.properties -------------------------------------------------------------------------------- /transactional-producer/src/main/resources/kafka.properties: -------------------------------------------------------------------------------- 1 | client.id=test_client 2 | bootstrap.servers=localhost:9092,localhost:9093 -------------------------------------------------------------------------------- /transactional-producer/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /transactional-producer/src/main/resources/schema/stock-data.json: -------------------------------------------------------------------------------- 1 | { 2 | "type":"object", 3 | "javaType" : "guru.learningjournal.kafka.examples.StockData", 4 | "properties": { 5 | "symbol": { 6 | "type": "string" 7 | }, 8 | "series": { 9 | "type": "string" 10 | }, 11 | "open": { 12 | "type": "string" 13 | }, 14 | "high": { 15 | "type": "number" 16 | }, 17 | "low": { 18 | "type": "number" 19 | }, 20 | "close": { 21 | "type": "number" 22 | }, 23 | "last": { 24 | "type": "number" 25 | }, 26 | "previousClose": { 27 | "type": "number" 28 | }, 29 | "totalTradedQty": { 30 | "type": "number" 31 | }, 32 | "totalTradedVal": { 33 | "type": "number" 34 | }, 35 | "tradeDate": { 36 | "type": "number", 37 | "format":"date-time", 38 | "customDateTimePattern" : "dd-MMM-yyyy", 39 | "customTimezone": "IST" 40 | }, 41 | "totalTrades": { 42 | "type": "number" 43 | }, 44 | "isinCode": { 45 | "type": "string" 46 | } 47 | } 48 | } --------------------------------------------------------------------------------