├── .circleci
└── config.yml
├── README.md
├── order-service
├── pom.xml
└── src
│ ├── main
│ ├── java
│ │ └── pl
│ │ │ └── piomin
│ │ │ └── samples
│ │ │ └── kafka
│ │ │ └── order
│ │ │ ├── OrderService.java
│ │ │ └── model
│ │ │ ├── Order.java
│ │ │ ├── OrderType.java
│ │ │ └── Transaction.java
│ └── resources
│ │ └── application.yml
│ └── test
│ └── java
│ └── pl
│ └── piomin
│ └── samples
│ └── kafka
│ └── order
│ ├── KafkaContainerDevMode.java
│ ├── OrderServiceAppTest.java
│ └── TestOrderService.java
├── pom.xml
├── renovate.json
└── stock-service
├── pom.xml
└── src
├── main
├── java
│ └── pl
│ │ └── piomin
│ │ └── samples
│ │ └── kafka
│ │ └── stock
│ │ ├── StockService.java
│ │ ├── controller
│ │ └── TransactionController.java
│ │ ├── logic
│ │ └── OrderLogic.java
│ │ ├── model
│ │ ├── Order.java
│ │ ├── OrderType.java
│ │ ├── OrdersSellBuy.java
│ │ ├── Transaction.java
│ │ ├── TransactionTotal.java
│ │ └── TransactionTotalWithProduct.java
│ │ └── repository
│ │ └── OrderRepository.java
└── resources
│ └── application.yml
└── test
└── java
└── pl
└── piomin
└── samples
└── kafka
└── stock
├── KafkaContainerDevMode.java
├── StockServiceAppTest.java
└── TestStockService.java
/.circleci/config.yml:
--------------------------------------------------------------------------------
1 | version: 2.1
2 |
3 | jobs:
4 | analyze:
5 | docker:
6 | - image: 'cimg/openjdk:21.0.6'
7 | steps:
8 | - checkout
9 | - run:
10 | name: Analyze on SonarCloud
11 | command: mvn verify sonar:sonar -DskipTests
12 | test:
13 | executor: machine_executor_amd64
14 | steps:
15 | - checkout
16 | - run:
17 | name: Install OpenJDK 21
18 | command: |
19 | java -version
20 | sudo apt-get update && sudo apt-get install openjdk-21-jdk
21 | sudo update-alternatives --set java /usr/lib/jvm/java-21-openjdk-amd64/bin/java
22 | sudo update-alternatives --set javac /usr/lib/jvm/java-21-openjdk-amd64/bin/javac
23 | java -version
24 | export JAVA_HOME=/usr/lib/jvm/java-21-openjdk-amd64
25 | - run:
26 | name: Maven Tests
27 | command: mvn test
28 |
29 | orbs:
30 | maven: circleci/maven@2.0.0
31 |
32 | executors:
33 | machine_executor_amd64:
34 | machine:
35 | image: ubuntu-2204:2023.10.1
36 | environment:
37 | architecture: "amd64"
38 | platform: "linux/amd64"
39 |
40 | workflows:
41 | maven_test:
42 | jobs:
43 | - test
44 | - analyze:
45 | context: SonarCloud
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Spring Cloud Stream Kafka Streams Advanced Demo Project [](https://twitter.com/piotr_minkowski)
2 |
3 | [](https://circleci.com/gh/piomin/sample-spring-cloud-stream-kafka)
4 |
5 | [](https://sonarcloud.io/dashboard?id=piomin_sample-spring-cloud-stream-kafka)
6 | [](https://sonarcloud.io/dashboard?id=piomin_sample-spring-cloud-stream-kafka)
7 | [](https://sonarcloud.io/dashboard?id=piomin_sample-spring-cloud-stream-kafka)
8 | [](https://sonarcloud.io/dashboard?id=piomin_sample-spring-cloud-stream-kafka)
9 |
10 | In this project I'm demonstrating you the most interesting features of [Spring Cloud Project](https://spring.io/projects/spring-cloud) for building event-driven, microservice-based architecture.
11 |
12 | Want to find out more details? Here's the article about it: [Kafka Streams with Spring Cloud Stream](https://piotrminkowski.com/2021/11/11/kafka-streams-with-spring-cloud-stream/). I'm describing there how to use Kafka Streams with Spring Cloud Stream to create stock market app.
--------------------------------------------------------------------------------
/order-service/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
5 |
6 | pl.piomin
7 | sample-spring-cloud-stream-kafka
8 | 1.0-SNAPSHOT
9 |
10 | 4.0.0
11 |
12 | order-service
13 |
14 |
15 | ${artifactId}
16 |
17 |
18 |
19 |
20 | org.springframework.cloud
21 | spring-cloud-starter-stream-kafka
22 |
23 |
24 | org.projectlombok
25 | lombok
26 |
27 |
28 | com.fasterxml.jackson.datatype
29 | jackson-datatype-jsr310
30 |
31 |
32 | org.springframework.boot
33 | spring-boot-starter-test
34 | test
35 |
36 |
37 | org.testcontainers
38 | kafka
39 | 1.21.1
40 | test
41 |
42 |
43 | org.testcontainers
44 | junit-jupiter
45 | 1.21.1
46 | test
47 |
48 |
49 | org.springframework.boot
50 | spring-boot-testcontainers
51 | test
52 |
53 |
54 |
55 |
56 |
57 |
58 | org.springframework.boot
59 | spring-boot-maven-plugin
60 |
61 |
62 |
63 |
--------------------------------------------------------------------------------
/order-service/src/main/java/pl/piomin/samples/kafka/order/OrderService.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.order;
2 |
3 | import lombok.extern.slf4j.Slf4j;
4 | import org.springframework.boot.SpringApplication;
5 | import org.springframework.boot.autoconfigure.SpringBootApplication;
6 | import org.springframework.context.annotation.Bean;
7 | import org.springframework.kafka.support.KafkaHeaders;
8 | import org.springframework.messaging.Message;
9 | import org.springframework.messaging.support.MessageBuilder;
10 | import pl.piomin.samples.kafka.order.model.Order;
11 | import pl.piomin.samples.kafka.order.model.OrderType;
12 |
13 | import java.time.LocalDateTime;
14 | import java.util.*;
15 | import java.util.function.Supplier;
16 |
17 | @SpringBootApplication
18 | @Slf4j
19 | public class OrderService {
20 |
21 | private static long orderId = 0;
22 | private static final Random r = new Random();
23 |
24 | private final Map prices = Map.of(
25 | 1, 1000,
26 | 2, 2000,
27 | 3, 5000,
28 | 4, 1500,
29 | 5, 2500,
30 | 6, 1000,
31 | 7, 2000,
32 | 8, 5000,
33 | 9, 1500,
34 | 10, 2500);
35 |
36 | LinkedList buyOrders = new LinkedList<>(List.of(
37 | new Order(++orderId, 1, 1, 100, LocalDateTime.now(), OrderType.BUY, 1000),
38 | new Order(++orderId, 2, 1, 200, LocalDateTime.now(), OrderType.BUY, 1050),
39 | new Order(++orderId, 3, 1, 100, LocalDateTime.now(), OrderType.BUY, 1030),
40 | new Order(++orderId, 4, 1, 200, LocalDateTime.now(), OrderType.BUY, 1050),
41 | new Order(++orderId, 5, 1, 200, LocalDateTime.now(), OrderType.BUY, 1000),
42 | new Order(++orderId, 11, 1, 100, LocalDateTime.now(), OrderType.BUY, 1050)
43 | ));
44 |
45 | LinkedList sellOrders = new LinkedList<>(List.of(
46 | new Order(++orderId, 6, 1, 200, LocalDateTime.now(), OrderType.SELL, 950),
47 | new Order(++orderId, 7, 1, 100, LocalDateTime.now(), OrderType.SELL, 1000),
48 | new Order(++orderId, 8, 1, 100, LocalDateTime.now(), OrderType.SELL, 1050),
49 | new Order(++orderId, 9, 1, 300, LocalDateTime.now(), OrderType.SELL, 1000),
50 | new Order(++orderId, 10, 1, 200, LocalDateTime.now(), OrderType.SELL, 1020)
51 | ));
52 |
53 | public static void main(String[] args) {
54 | SpringApplication.run(OrderService.class, args);
55 | }
56 |
57 | @Bean
58 | public Supplier> orderBuySupplier() {
59 | return () -> {
60 | if (buyOrders.peek() != null) {
61 | Message o = MessageBuilder
62 | .withPayload(buyOrders.peek())
63 | .setHeader(KafkaHeaders.KEY, Objects.requireNonNull(buyOrders.poll()).getId())
64 | .build();
65 | log.info("Order: {}", o.getPayload());
66 | return o;
67 | } else {
68 | return null;
69 | }
70 | };
71 | }
72 |
73 | @Bean
74 | public Supplier> orderSellSupplier() {
75 | return () -> {
76 | if (sellOrders.peek() != null) {
77 | Message o = MessageBuilder
78 | .withPayload(sellOrders.peek())
79 | .setHeader(KafkaHeaders.KEY, Objects.requireNonNull(sellOrders.poll()).getId())
80 | .build();
81 | log.info("Order: {}", o.getPayload());
82 | return o;
83 | } else {
84 | return null;
85 | }
86 | };
87 | }
88 |
89 | // @Bean
90 | // public Supplier> orderBuySupplier() {
91 | // return () -> {
92 | // Integer productId = r.nextInt(1, 6);
93 | // int price = prices.get(productId) + r.nextInt(-100,100);
94 | // Order o = new Order(
95 | // ++orderId,
96 | // r.nextInt(1, 11),
97 | // productId,
98 | // 100,
99 | // LocalDateTime.now(),
100 | // OrderType.BUY,
101 | // price);
102 | // log.info("Order: {}", o);
103 | // return MessageBuilder
104 | // .withPayload(o)
105 | // .setHeader(KafkaHeaders.MESSAGE_KEY, orderId)
106 | // .build();
107 | // };
108 | // }
109 |
110 | // @Bean
111 | // public Supplier> orderSellSupplier() {
112 | // return () -> {
113 | // Integer productId = r.nextInt(1, 6);
114 | // int price = prices.get(productId) + r.nextInt(-100,100);
115 | // Order o = new Order(
116 | // ++orderId,
117 | // r.nextInt(1, 11),
118 | // productId,
119 | // 100,
120 | // LocalDateTime.now(),
121 | // OrderType.SELL,
122 | // price);
123 | // log.info("Order: {}", o);
124 | // return MessageBuilder
125 | // .withPayload(o)
126 | // .setHeader(KafkaHeaders.MESSAGE_KEY, orderId)
127 | // .build();
128 | // };
129 | // }
130 |
131 | }
132 |
--------------------------------------------------------------------------------
/order-service/src/main/java/pl/piomin/samples/kafka/order/model/Order.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.order.model;
2 |
3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
4 | import com.fasterxml.jackson.databind.annotation.JsonSerialize;
5 | import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer;
6 | import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer;
7 | import lombok.*;
8 |
9 | import java.time.LocalDateTime;
10 |
11 | @Getter
12 | @Setter
13 | @ToString
14 | @AllArgsConstructor
15 | @NoArgsConstructor
16 | public class Order {
17 | private Long id;
18 | private Integer customerId;
19 | private Integer productId;
20 | private int productCount;
21 | @JsonDeserialize(using = LocalDateTimeDeserializer.class)
22 | @JsonSerialize(using = LocalDateTimeSerializer.class)
23 | private LocalDateTime creationDate;
24 | private OrderType type;
25 | private int amount;
26 | }
27 |
--------------------------------------------------------------------------------
/order-service/src/main/java/pl/piomin/samples/kafka/order/model/OrderType.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.order.model;
2 |
3 | public enum OrderType {
4 | SELL, BUY;
5 | }
6 |
--------------------------------------------------------------------------------
/order-service/src/main/java/pl/piomin/samples/kafka/order/model/Transaction.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.order.model;
2 |
3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
4 | import com.fasterxml.jackson.databind.annotation.JsonSerialize;
5 | import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer;
6 | import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer;
7 | import lombok.*;
8 |
9 | import java.time.LocalDateTime;
10 |
11 | @Getter
12 | @Setter
13 | @ToString
14 | @AllArgsConstructor
15 | @NoArgsConstructor
16 | public class Transaction {
17 | private Long id;
18 | private Long buyOrderId;
19 | private Long sellOrderId;
20 | private int amount;
21 | private int price;
22 | @JsonDeserialize(using = LocalDateTimeDeserializer.class)
23 | @JsonSerialize(using = LocalDateTimeSerializer.class)
24 | private LocalDateTime creationTime;
25 | }
26 |
--------------------------------------------------------------------------------
/order-service/src/main/resources/application.yml:
--------------------------------------------------------------------------------
1 | spring.application.name: order-service
2 |
3 | spring.kafka.bootstrap-servers: ${KAFKA_URL}
4 |
5 | spring.cloud.stream.function.definition: orderBuySupplier;orderSellSupplier
6 | spring.cloud.function.definition: orderBuySupplier;orderSellSupplier
7 |
8 | spring.cloud.stream.bindings.orderBuySupplier-out-0.destination: orders.buy
9 | spring.cloud.stream.kafka.bindings.orderBuySupplier-out-0.producer.configuration.key.serializer: org.apache.kafka.common.serialization.LongSerializer
10 |
11 | spring.cloud.stream.bindings.orderSellSupplier-out-0.destination: orders.sell
12 | spring.cloud.stream.kafka.bindings.orderSellSupplier-out-0.producer.configuration.key.serializer: org.apache.kafka.common.serialization.LongSerializer
13 |
14 |
15 | spring.output.ansi.enabled: ALWAYS
16 |
17 | logging.pattern.console: "%clr(%d{HH:mm:ss.SSS}){blue} %clr(---){faint} %clr([%15.15t]){yellow} %clr(:){red} %clr(%m){faint}%n"
--------------------------------------------------------------------------------
/order-service/src/test/java/pl/piomin/samples/kafka/order/KafkaContainerDevMode.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.order;
2 |
3 | import org.springframework.boot.test.context.TestConfiguration;
4 | import org.springframework.boot.testcontainers.service.connection.ServiceConnection;
5 | import org.springframework.context.annotation.Bean;
6 | import org.testcontainers.containers.KafkaContainer;
7 | import org.testcontainers.utility.DockerImageName;
8 |
9 | @TestConfiguration
10 | public class KafkaContainerDevMode {
11 |
12 | @Bean
13 | @ServiceConnection
14 | public KafkaContainer mongodbContainer() {
15 | return new KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:6.2.1"));
16 | }
17 |
18 | }
19 |
--------------------------------------------------------------------------------
/order-service/src/test/java/pl/piomin/samples/kafka/order/OrderServiceAppTest.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.order;
2 |
3 | import org.junit.jupiter.api.Test;
4 | import org.springframework.boot.test.context.SpringBootTest;
5 | import org.springframework.test.context.DynamicPropertyRegistry;
6 | import org.springframework.test.context.DynamicPropertySource;
7 | import org.testcontainers.containers.KafkaContainer;
8 | import org.testcontainers.junit.jupiter.Container;
9 | import org.testcontainers.junit.jupiter.Testcontainers;
10 | import org.testcontainers.utility.DockerImageName;
11 |
12 | @SpringBootTest
13 | @Testcontainers
14 | public class OrderServiceAppTest {
15 |
16 | @Container
17 | static KafkaContainer kafka = new KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:6.2.1"));
18 |
19 |
20 | @DynamicPropertySource
21 | static void kafkaProperties(DynamicPropertyRegistry registry) {
22 | registry.add("spring.kafka.bootstrap-servers", kafka::getBootstrapServers);
23 | }
24 |
25 | @Test
26 | void startup() {
27 |
28 | }
29 |
30 | }
31 |
--------------------------------------------------------------------------------
/order-service/src/test/java/pl/piomin/samples/kafka/order/TestOrderService.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.order;
2 |
3 | import org.springframework.boot.SpringApplication;
4 |
5 | public class TestOrderService {
6 |
7 | public static void main(String[] args) {
8 | SpringApplication.from(OrderService::main)
9 | .with(KafkaContainerDevMode.class)
10 | .run(args);
11 | }
12 |
13 | }
14 |
--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
5 | 4.0.0
6 |
7 |
8 | org.springframework.boot
9 | spring-boot-starter-parent
10 | 3.5.0
11 |
12 |
13 |
14 | pl.piomin
15 | sample-spring-cloud-stream-kafka
16 | 1.0-SNAPSHOT
17 |
18 | order-service
19 | stock-service
20 |
21 | pom
22 |
23 |
24 | 2025.0.0
25 | 21
26 | piomin_sample-spring-cloud-stream-kafka
27 | piomin
28 | https://sonarcloud.io
29 |
30 |
31 |
32 |
33 |
34 | org.jacoco
35 | jacoco-maven-plugin
36 | 0.8.13
37 |
38 |
39 |
40 | prepare-agent
41 |
42 |
43 |
44 | report
45 | test
46 |
47 | report
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 | org.springframework.cloud
59 | spring-cloud-dependencies
60 | ${spring-cloud.version}
61 | pom
62 | import
63 |
64 |
65 |
66 |
67 |
--------------------------------------------------------------------------------
/renovate.json:
--------------------------------------------------------------------------------
1 | {
2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json",
3 | "extends": [
4 | "config:base",":dependencyDashboard"
5 | ],
6 | "packageRules": [
7 | {
8 | "matchUpdateTypes": ["minor", "patch", "pin", "digest"],
9 | "automerge": true
10 | }
11 | ],
12 | "prCreation": "not-pending"
13 | }
--------------------------------------------------------------------------------
/stock-service/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
5 |
6 | pl.piomin
7 | sample-spring-cloud-stream-kafka
8 | 1.0-SNAPSHOT
9 |
10 | 4.0.0
11 |
12 | stock-service
13 |
14 |
15 | ${artifactId}
16 |
17 |
18 |
19 |
20 | org.springframework.cloud
21 | spring-cloud-stream-binder-kafka-streams
22 |
23 |
24 | org.springframework.boot
25 | spring-boot-starter-data-jpa
26 |
27 |
28 | com.h2database
29 | h2
30 |
31 |
32 | org.springframework.boot
33 | spring-boot-starter-web
34 |
35 |
36 | com.fasterxml.jackson.datatype
37 | jackson-datatype-jsr310
38 |
39 |
40 | org.springframework.boot
41 | spring-boot-starter-test
42 | test
43 |
44 |
45 | org.testcontainers
46 | kafka
47 | 1.21.1
48 | test
49 |
50 |
51 | org.testcontainers
52 | junit-jupiter
53 | 1.21.1
54 | test
55 |
56 |
57 | org.springframework.boot
58 | spring-boot-testcontainers
59 | test
60 |
61 |
62 |
63 |
64 |
65 |
66 | org.springframework.boot
67 | spring-boot-maven-plugin
68 |
69 |
70 |
71 |
72 |
--------------------------------------------------------------------------------
/stock-service/src/main/java/pl/piomin/samples/kafka/stock/StockService.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.stock;
2 |
3 | import org.apache.kafka.common.serialization.Serdes;
4 | import org.apache.kafka.streams.KeyValue;
5 | import org.apache.kafka.streams.kstream.*;
6 | import org.apache.kafka.streams.state.KeyValueBytesStoreSupplier;
7 | import org.apache.kafka.streams.state.Stores;
8 | import org.apache.kafka.streams.state.WindowBytesStoreSupplier;
9 | import org.slf4j.Logger;
10 | import org.slf4j.LoggerFactory;
11 | import org.springframework.beans.factory.annotation.Autowired;
12 | import org.springframework.boot.SpringApplication;
13 | import org.springframework.boot.autoconfigure.SpringBootApplication;
14 | import org.springframework.context.annotation.Bean;
15 | import org.springframework.kafka.support.serializer.JsonSerde;
16 | import pl.piomin.samples.kafka.stock.logic.OrderLogic;
17 | import pl.piomin.samples.kafka.stock.model.Order;
18 | import pl.piomin.samples.kafka.stock.model.Transaction;
19 | import pl.piomin.samples.kafka.stock.model.TransactionTotal;
20 | import pl.piomin.samples.kafka.stock.model.TransactionTotalWithProduct;
21 |
22 | import java.time.Duration;
23 | import java.time.LocalDateTime;
24 | import java.util.function.BiConsumer;
25 | import java.util.function.BiFunction;
26 | import java.util.function.Consumer;
27 |
28 | @SpringBootApplication
29 | public class StockService {
30 |
31 | private static final Logger log = LoggerFactory.getLogger(StockService.class);
32 | private static long transactionId = 0;
33 |
34 | public static void main(String[] args) {
35 | SpringApplication.run(StockService.class, args);
36 | }
37 |
38 | @Autowired
39 | OrderLogic logic;
40 |
41 | @Bean
42 | public BiConsumer, KStream> orders() {
43 | return (orderBuy, orderSell) -> orderBuy
44 | .merge(orderSell)
45 | .peek((k, v) -> {
46 | log.info("New({}): {}", k, v);
47 | logic.add(v);
48 | });
49 | }
50 |
51 | @Bean
52 | public BiFunction, KStream, KStream> transactions() {
53 | return (orderBuy, orderSell) -> orderBuy
54 | .selectKey((k, v) -> v.getProductId())
55 | .join(orderSell.selectKey((k, v) -> v.getProductId()),
56 | this::execute,
57 | JoinWindows.of(Duration.ofSeconds(10)),
58 | StreamJoined.with(Serdes.Integer(), new JsonSerde<>(Order.class), new JsonSerde<>(Order.class)))
59 | .filterNot((k, v) -> v == null)
60 | .map((k, v) -> new KeyValue<>(v.getId(), v))
61 | .peek((k, v) -> log.info("Done -> {}", v));
62 | }
63 |
64 | @Bean
65 | public Consumer> total() {
66 | KeyValueBytesStoreSupplier storeSupplier = Stores.persistentKeyValueStore(
67 | "all-transactions-store");
68 | return transactions -> transactions
69 | .groupBy((k, v) -> v.getStatus(),
70 | Grouped.with(Serdes.String(), new JsonSerde<>(Transaction.class)))
71 | .aggregate(
72 | TransactionTotal::new,
73 | (k, v, a) -> {
74 | a.setCount(a.getCount() + 1);
75 | a.setProductCount(a.getProductCount() + v.getAmount());
76 | a.setAmount(a.getAmount() + (v.getPrice() * v.getAmount()));
77 | return a;
78 | },
79 | Materialized. as(storeSupplier)
80 | .withKeySerde(Serdes.String())
81 | .withValueSerde(new JsonSerde<>(TransactionTotal.class)))
82 | .toStream()
83 | .peek((k, v) -> log.info("Total: {}", v));
84 | }
85 |
86 | @Bean
87 | public BiConsumer, KStream> totalPerProduct() {
88 | KeyValueBytesStoreSupplier storeSupplier = Stores.persistentKeyValueStore(
89 | "transactions-per-product-store");
90 | return (transactions, orders) -> transactions
91 | .selectKey((k, v) -> v.getSellOrderId())
92 | .join(orders.selectKey((k, v) -> v.getId()),
93 | (t, o) -> new TransactionTotalWithProduct(t, o.getProductId()),
94 | JoinWindows.of(Duration.ofSeconds(10)),
95 | StreamJoined.with(Serdes.Long(),
96 | new JsonSerde<>(Transaction.class),
97 | new JsonSerde<>(Order.class)))
98 | .groupBy((k, v) -> v.getProductId(),
99 | Grouped.with(Serdes.Integer(), new JsonSerde<>(TransactionTotalWithProduct.class)))
100 | .aggregate(
101 | TransactionTotal::new,
102 | (k, v, a) -> {
103 | a.setCount(a.getCount() + 1);
104 | a.setProductCount(a.getProductCount() + v.getTransaction().getAmount());
105 | a.setAmount(a.getAmount() + (v.getTransaction().getPrice() * v.getTransaction().getAmount()));
106 | return a;
107 | },
108 | Materialized. as(storeSupplier)
109 | .withKeySerde(Serdes.Integer())
110 | .withValueSerde(new JsonSerde<>(TransactionTotal.class)))
111 | .toStream()
112 | .peek((k, v) -> log.info("Total per product({}): {}", k, v));
113 | }
114 |
115 | @Bean
116 | public BiConsumer, KStream> latestPerProduct() {
117 | WindowBytesStoreSupplier storeSupplier = Stores.persistentWindowStore(
118 | "latest-transactions-per-product-store", Duration.ofSeconds(30), Duration.ofSeconds(30), false);
119 | return (transactions, orders) -> transactions
120 | .selectKey((k, v) -> v.getSellOrderId())
121 | .join(orders.selectKey((k, v) -> v.getId()),
122 | (t, o) -> new TransactionTotalWithProduct(t, o.getProductId()),
123 | JoinWindows.of(Duration.ofSeconds(10)),
124 | StreamJoined.with(Serdes.Long(), new JsonSerde<>(Transaction.class), new JsonSerde<>(Order.class)))
125 | .groupBy((k, v) -> v.getProductId(), Grouped.with(Serdes.Integer(), new JsonSerde<>(TransactionTotalWithProduct.class)))
126 | .windowedBy(TimeWindows.of(Duration.ofSeconds(30)))
127 | .aggregate(
128 | TransactionTotal::new,
129 | (k, v, a) -> {
130 | a.setCount(a.getCount() + 1);
131 | a.setAmount(a.getAmount() + v.getTransaction().getAmount());
132 | return a;
133 | },
134 | Materialized. as(storeSupplier)
135 | .withKeySerde(Serdes.Integer())
136 | .withValueSerde(new JsonSerde<>(TransactionTotal.class)))
137 | .toStream()
138 | .peek((k, v) -> log.info("Total per product last 30s({}): {}", k, v));
139 | }
140 |
141 | private Transaction execute(Order orderBuy, Order orderSell) {
142 | if (orderBuy.getAmount() >= orderSell.getAmount()) {
143 | int count = Math.min(orderBuy.getProductCount(), orderSell.getProductCount());
144 | // log.info("Executed: orderBuy={}, orderSell={}", orderBuy.getId(), orderSell.getId());
145 | boolean allowed = logic.performUpdate(orderBuy.getId(), orderSell.getId(), count);
146 | if (!allowed)
147 | return null;
148 | else
149 | return new Transaction(
150 | ++transactionId,
151 | orderBuy.getId(),
152 | orderSell.getId(),
153 | Math.min(orderBuy.getProductCount(), orderSell.getProductCount()),
154 | (orderBuy.getAmount() + orderSell.getAmount()) / 2,
155 | LocalDateTime.now(),
156 | "NEW");
157 | } else {
158 | return null;
159 | }
160 | }
161 |
162 | }
163 |
--------------------------------------------------------------------------------
/stock-service/src/main/java/pl/piomin/samples/kafka/stock/controller/TransactionController.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.stock.controller;
2 |
3 | import org.apache.kafka.streams.KeyValue;
4 | import org.apache.kafka.streams.state.KeyValueIterator;
5 | import org.apache.kafka.streams.state.QueryableStoreTypes;
6 | import org.apache.kafka.streams.state.ReadOnlyKeyValueStore;
7 | import org.springframework.cloud.stream.binder.kafka.streams.InteractiveQueryService;
8 | import org.springframework.web.bind.annotation.GetMapping;
9 | import org.springframework.web.bind.annotation.PathVariable;
10 | import org.springframework.web.bind.annotation.RequestMapping;
11 | import org.springframework.web.bind.annotation.RestController;
12 | import pl.piomin.samples.kafka.stock.model.TransactionTotal;
13 |
14 | import java.util.HashMap;
15 | import java.util.Map;
16 |
17 | @RestController
18 | @RequestMapping("/transactions")
19 | public class TransactionController {
20 |
21 | private InteractiveQueryService queryService;
22 |
23 | public TransactionController(InteractiveQueryService queryService) {
24 | this.queryService = queryService;
25 | }
26 |
27 | @GetMapping("/all")
28 | public TransactionTotal getAllTransactionsSummary() {
29 | ReadOnlyKeyValueStore keyValueStore =
30 | queryService.getQueryableStore("all-transactions-store",
31 | QueryableStoreTypes.keyValueStore());
32 | return keyValueStore.get("NEW");
33 | }
34 |
35 | @GetMapping("/product/{productId}")
36 | public TransactionTotal getSummaryByProductId(@PathVariable("productId") Integer productId) {
37 | ReadOnlyKeyValueStore keyValueStore =
38 | queryService.getQueryableStore("transactions-per-product-store",
39 | QueryableStoreTypes.keyValueStore());
40 | return keyValueStore.get(productId);
41 | }
42 |
43 | @GetMapping("/product/latest/{productId}")
44 | public TransactionTotal getLatestSummaryByProductId(@PathVariable("productId") Integer productId) {
45 | ReadOnlyKeyValueStore keyValueStore =
46 | queryService.getQueryableStore("latest-transactions-per-product-store",
47 | QueryableStoreTypes.keyValueStore());
48 | return keyValueStore.get(productId);
49 | }
50 |
51 | @GetMapping("/product")
52 | public Map getSummaryByAllProducts() {
53 | Map m = new HashMap<>();
54 | ReadOnlyKeyValueStore keyValueStore =
55 | queryService.getQueryableStore("transactions-per-product-store",
56 | QueryableStoreTypes.keyValueStore());
57 | KeyValueIterator it = keyValueStore.all();
58 | while (it.hasNext()) {
59 | KeyValue kv = it.next();
60 | m.put(kv.key, kv.value);
61 | }
62 | return m;
63 | }
64 |
65 | }
66 |
--------------------------------------------------------------------------------
/stock-service/src/main/java/pl/piomin/samples/kafka/stock/logic/OrderLogic.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.stock.logic;
2 |
3 | import org.springframework.stereotype.Service;
4 | import org.springframework.transaction.annotation.Transactional;
5 | import pl.piomin.samples.kafka.stock.model.Order;
6 | import pl.piomin.samples.kafka.stock.repository.OrderRepository;
7 |
8 | @Service
9 | public class OrderLogic {
10 |
11 | private OrderRepository repository;
12 |
13 | public OrderLogic(OrderRepository repository) {
14 | this.repository = repository;
15 | }
16 |
17 | public Order add(Order order) {
18 | return repository.save(order);
19 | }
20 |
21 | @Transactional
22 | public boolean performUpdate(Long buyOrderId, Long sellOrderId, int amount) {
23 | Order buyOrder = repository.findById(buyOrderId).orElseThrow();
24 | Order sellOrder = repository.findById(sellOrderId).orElseThrow();
25 | int buyAvailableCount = buyOrder.getProductCount() - buyOrder.getRealizedCount();
26 | int sellAvailableCount = sellOrder.getProductCount() - sellOrder.getRealizedCount();
27 | if (buyAvailableCount >= amount && sellAvailableCount >= amount) {
28 | buyOrder.setRealizedCount(buyOrder.getRealizedCount() + amount);
29 | sellOrder.setRealizedCount(sellOrder.getRealizedCount() + amount);
30 | repository.save(buyOrder);
31 | repository.save(sellOrder);
32 | return true;
33 | } else {
34 | return false;
35 | }
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/stock-service/src/main/java/pl/piomin/samples/kafka/stock/model/Order.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.stock.model;
2 |
3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
4 | import com.fasterxml.jackson.databind.annotation.JsonSerialize;
5 | import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer;
6 | import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer;
7 | import jakarta.persistence.Entity;
8 | import jakarta.persistence.Id;
9 | import jakarta.persistence.Table;
10 |
11 | import java.time.LocalDateTime;
12 |
13 | @Entity
14 | @Table(name = "orders")
15 | public class Order {
16 | @Id
17 | private Long id;
18 | private Integer customerId;
19 | private Integer productId;
20 | private int productCount;
21 | private int realizedCount;
22 | @JsonDeserialize(using = LocalDateTimeDeserializer.class)
23 | @JsonSerialize(using = LocalDateTimeSerializer.class)
24 | private LocalDateTime creationDate;
25 | private OrderType type;
26 | private int amount;
27 |
28 | public Order() {
29 | }
30 |
31 | public Order(Long id, Integer customerId, Integer productId, int productCount, int realizedCount, LocalDateTime creationDate, OrderType type, int amount) {
32 | this.id = id;
33 | this.customerId = customerId;
34 | this.productId = productId;
35 | this.productCount = productCount;
36 | this.realizedCount = realizedCount;
37 | this.creationDate = creationDate;
38 | this.type = type;
39 | this.amount = amount;
40 | }
41 |
42 | public Long getId() {
43 | return id;
44 | }
45 |
46 | public void setId(Long id) {
47 | this.id = id;
48 | }
49 |
50 | public Integer getCustomerId() {
51 | return customerId;
52 | }
53 |
54 | public void setCustomerId(Integer customerId) {
55 | this.customerId = customerId;
56 | }
57 |
58 | public Integer getProductId() {
59 | return productId;
60 | }
61 |
62 | public void setProductId(Integer productId) {
63 | this.productId = productId;
64 | }
65 |
66 | public int getProductCount() {
67 | return productCount;
68 | }
69 |
70 | public void setProductCount(int productCount) {
71 | this.productCount = productCount;
72 | }
73 |
74 | public int getRealizedCount() {
75 | return realizedCount;
76 | }
77 |
78 | public void setRealizedCount(int realizedCount) {
79 | this.realizedCount = realizedCount;
80 | }
81 |
82 | public LocalDateTime getCreationDate() {
83 | return creationDate;
84 | }
85 |
86 | public void setCreationDate(LocalDateTime creationDate) {
87 | this.creationDate = creationDate;
88 | }
89 |
90 | public OrderType getType() {
91 | return type;
92 | }
93 |
94 | public void setType(OrderType type) {
95 | this.type = type;
96 | }
97 |
98 | public int getAmount() {
99 | return amount;
100 | }
101 |
102 | public void setAmount(int amount) {
103 | this.amount = amount;
104 | }
105 |
106 | @Override
107 | public String toString() {
108 | return "Order{" +
109 | "id=" + id +
110 | ", customerId=" + customerId +
111 | ", productId=" + productId +
112 | ", productCount=" + productCount +
113 | ", realizedCount=" + realizedCount +
114 | ", creationDate=" + creationDate +
115 | ", type=" + type +
116 | ", amount=" + amount +
117 | '}';
118 | }
119 | }
120 |
--------------------------------------------------------------------------------
/stock-service/src/main/java/pl/piomin/samples/kafka/stock/model/OrderType.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.stock.model;
2 |
3 | public enum OrderType {
4 | SELL, BUY;
5 | }
6 |
--------------------------------------------------------------------------------
/stock-service/src/main/java/pl/piomin/samples/kafka/stock/model/OrdersSellBuy.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.stock.model;
2 |
3 | public class OrdersSellBuy {
4 |
5 | private int sellCount;
6 | private int buyCount;
7 |
8 | public OrdersSellBuy() {
9 | }
10 |
11 | public OrdersSellBuy(int sellCount, int buyCount) {
12 | this.sellCount = sellCount;
13 | this.buyCount = buyCount;
14 | }
15 |
16 | public OrdersSellBuy addSell(int sellCount) {
17 | this.sellCount += sellCount;
18 | return this;
19 | }
20 |
21 | public OrdersSellBuy addBuy(int buyCount) {
22 | this.buyCount += buyCount;
23 | return this;
24 | }
25 |
26 | public int getSellCount() {
27 | return sellCount;
28 | }
29 |
30 | public void setSellCount(int sellCount) {
31 | this.sellCount = sellCount;
32 | }
33 |
34 | public int getBuyCount() {
35 | return buyCount;
36 | }
37 |
38 | public void setBuyCount(int buyCount) {
39 | this.buyCount = buyCount;
40 | }
41 |
42 | @Override
43 | public String toString() {
44 | return "OrdersSellBuy{" +
45 | "sellCount=" + sellCount +
46 | ", buyCount=" + buyCount +
47 | '}';
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/stock-service/src/main/java/pl/piomin/samples/kafka/stock/model/Transaction.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.stock.model;
2 |
3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
4 | import com.fasterxml.jackson.databind.annotation.JsonSerialize;
5 | import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer;
6 | import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer;
7 |
8 | import java.time.LocalDateTime;
9 |
10 | public class Transaction {
11 |
12 | private Long id;
13 | private Long buyOrderId;
14 | private Long sellOrderId;
15 | private int amount;
16 | private int price;
17 | @JsonDeserialize(using = LocalDateTimeDeserializer.class)
18 | @JsonSerialize(using = LocalDateTimeSerializer.class)
19 | private LocalDateTime creationTime;
20 | private String status;
21 |
22 | public Transaction() {
23 | }
24 |
25 | public Transaction(Long id, Long buyOrderId, Long sellOrderId, int amount, int price, LocalDateTime creationTime, String status) {
26 | this.id = id;
27 | this.buyOrderId = buyOrderId;
28 | this.sellOrderId = sellOrderId;
29 | this.amount = amount;
30 | this.price = price;
31 | this.creationTime = creationTime;
32 | this.status = status;
33 | }
34 |
35 | public Long getId() {
36 | return id;
37 | }
38 |
39 | public void setId(Long id) {
40 | this.id = id;
41 | }
42 |
43 | public Long getBuyOrderId() {
44 | return buyOrderId;
45 | }
46 |
47 | public void setBuyOrderId(Long buyOrderId) {
48 | this.buyOrderId = buyOrderId;
49 | }
50 |
51 | public Long getSellOrderId() {
52 | return sellOrderId;
53 | }
54 |
55 | public void setSellOrderId(Long sellOrderId) {
56 | this.sellOrderId = sellOrderId;
57 | }
58 |
59 | public int getAmount() {
60 | return amount;
61 | }
62 |
63 | public void setAmount(int amount) {
64 | this.amount = amount;
65 | }
66 |
67 | public int getPrice() {
68 | return price;
69 | }
70 |
71 | public void setPrice(int price) {
72 | this.price = price;
73 | }
74 |
75 | public LocalDateTime getCreationTime() {
76 | return creationTime;
77 | }
78 |
79 | public void setCreationTime(LocalDateTime creationTime) {
80 | this.creationTime = creationTime;
81 | }
82 |
83 | public String getStatus() {
84 | return status;
85 | }
86 |
87 | public void setStatus(String status) {
88 | this.status = status;
89 | }
90 |
91 | @Override
92 | public String toString() {
93 | return "Transaction{" +
94 | "id=" + id +
95 | ", buyOrderId=" + buyOrderId +
96 | ", sellOrderId=" + sellOrderId +
97 | ", amount=" + amount +
98 | ", price=" + price +
99 | ", creationTime=" + creationTime +
100 | ", status='" + status + '\'' +
101 | '}';
102 | }
103 | }
104 |
--------------------------------------------------------------------------------
/stock-service/src/main/java/pl/piomin/samples/kafka/stock/model/TransactionTotal.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.stock.model;
2 |
3 | public class TransactionTotal {
4 | private int count;
5 | private int productCount;
6 | private long amount;
7 |
8 | public TransactionTotal() {
9 | }
10 |
11 | public TransactionTotal(int count, int productCount, long amount) {
12 | this.count = count;
13 | this.productCount = productCount;
14 | this.amount = amount;
15 | }
16 |
17 | public int getCount() {
18 | return count;
19 | }
20 |
21 | public void setCount(int count) {
22 | this.count = count;
23 | }
24 |
25 | public int getProductCount() {
26 | return productCount;
27 | }
28 |
29 | public void setProductCount(int productCount) {
30 | this.productCount = productCount;
31 | }
32 |
33 | public long getAmount() {
34 | return amount;
35 | }
36 |
37 | public void setAmount(long amount) {
38 | this.amount = amount;
39 | }
40 |
41 | @Override
42 | public String toString() {
43 | return "TransactionTotal{" +
44 | "count=" + count +
45 | ", productCount=" + productCount +
46 | ", amount=" + amount +
47 | '}';
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/stock-service/src/main/java/pl/piomin/samples/kafka/stock/model/TransactionTotalWithProduct.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.stock.model;
2 |
3 | public class TransactionTotalWithProduct {
4 | private Transaction transaction;
5 | private Integer productId;
6 |
7 | public TransactionTotalWithProduct() {
8 | }
9 |
10 | public TransactionTotalWithProduct(Transaction transaction, Integer productId) {
11 | this.transaction = transaction;
12 | this.productId = productId;
13 | }
14 |
15 | public Transaction getTransaction() {
16 | return transaction;
17 | }
18 |
19 | public void setTransaction(Transaction transaction) {
20 | this.transaction = transaction;
21 | }
22 |
23 | public Integer getProductId() {
24 | return productId;
25 | }
26 |
27 | public void setProductId(Integer productId) {
28 | this.productId = productId;
29 | }
30 |
31 | @Override
32 | public String toString() {
33 | return "TransactionTotalWithProduct{" +
34 | "transaction=" + transaction +
35 | ", productId=" + productId +
36 | '}';
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/stock-service/src/main/java/pl/piomin/samples/kafka/stock/repository/OrderRepository.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.stock.repository;
2 |
3 | import org.springframework.data.jpa.repository.Lock;
4 | import org.springframework.data.repository.CrudRepository;
5 | import pl.piomin.samples.kafka.stock.model.Order;
6 |
7 | import jakarta.persistence.LockModeType;
8 | import java.util.Optional;
9 |
10 | public interface OrderRepository extends CrudRepository {
11 |
12 | @Lock(LockModeType.PESSIMISTIC_WRITE)
13 | Optional findById(Long id);
14 |
15 | }
16 |
--------------------------------------------------------------------------------
/stock-service/src/main/resources/application.yml:
--------------------------------------------------------------------------------
1 | spring.application.name: stock-service
2 |
3 | spring.kafka.bootstrap-servers: ${KAFKA_URL}
4 |
5 | spring.cloud.stream.function.definition: orders;transactions;total;totalPerProduct;latestPerProduct
6 | spring.cloud.function.definition: orders;transactions;total;totalPerProduct;latestPerProduct
7 |
8 | spring.cloud.stream.bindings.orders-in-0.destination: orders.buy
9 | spring.cloud.stream.bindings.orders-in-1.destination: orders.sell
10 | spring.cloud.stream.kafka.streams.binder.functions.orders.applicationId: orders
11 |
12 | spring.cloud.stream.bindings.transactions-in-0.destination: orders.buy
13 | spring.cloud.stream.bindings.transactions-in-1.destination: orders.sell
14 | spring.cloud.stream.bindings.transactions-out-0.destination: transactions
15 | spring.cloud.stream.kafka.streams.binder.functions.transactions.applicationId: transactions
16 |
17 | spring.cloud.stream.bindings.total-in-0.destination: transactions
18 | spring.cloud.stream.kafka.streams.binder.functions.total.applicationId: total
19 |
20 | spring.cloud.stream.bindings.totalPerProduct-in-0.destination: transactions
21 | spring.cloud.stream.bindings.totalPerProduct-in-1.destination: orders.sell
22 | spring.cloud.stream.kafka.streams.binder.functions.totalPerProduct.applicationId: totalPerProduct
23 |
24 | spring.cloud.stream.bindings.latestPerProduct-in-0.destination: transactions
25 | spring.cloud.stream.bindings.latestPerProduct-in-1.destination: orders.sell
26 | spring.cloud.stream.kafka.streams.binder.functions.latestPerProduct.applicationId: latestPerProduct
27 |
28 | spring.output.ansi.enabled: ALWAYS
29 |
30 | logging.pattern.console: "%clr(%d{HH:mm:ss.SSS}){blue} %clr(---){faint} %clr([%15.15t]){yellow} %clr(:){red} %clr(%m){faint}%n"
31 |
32 | management.endpoints.web.exposure.include: "*"
--------------------------------------------------------------------------------
/stock-service/src/test/java/pl/piomin/samples/kafka/stock/KafkaContainerDevMode.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.stock;
2 |
3 | import org.springframework.boot.test.context.TestConfiguration;
4 | import org.springframework.boot.testcontainers.service.connection.ServiceConnection;
5 | import org.springframework.context.annotation.Bean;
6 | import org.testcontainers.containers.KafkaContainer;
7 | import org.testcontainers.utility.DockerImageName;
8 |
9 | @TestConfiguration
10 | public class KafkaContainerDevMode {
11 |
12 | @Bean
13 | @ServiceConnection
14 | public KafkaContainer mongodbContainer() {
15 | return new KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:6.2.1"));
16 | }
17 |
18 | }
19 |
--------------------------------------------------------------------------------
/stock-service/src/test/java/pl/piomin/samples/kafka/stock/StockServiceAppTest.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.stock;
2 |
3 | import org.junit.jupiter.api.Test;
4 | import org.springframework.boot.test.context.SpringBootTest;
5 | import org.springframework.test.context.DynamicPropertyRegistry;
6 | import org.springframework.test.context.DynamicPropertySource;
7 | import org.testcontainers.containers.KafkaContainer;
8 | import org.testcontainers.junit.jupiter.Container;
9 | import org.testcontainers.junit.jupiter.Testcontainers;
10 | import org.testcontainers.utility.DockerImageName;
11 |
12 | @SpringBootTest
13 | @Testcontainers
14 | public class StockServiceAppTest {
15 |
16 | @Container
17 | static KafkaContainer kafka = new KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:6.2.1"));
18 |
19 |
20 | @DynamicPropertySource
21 | static void kafkaProperties(DynamicPropertyRegistry registry) {
22 | registry.add("spring.kafka.bootstrap-servers", kafka::getBootstrapServers);
23 | }
24 |
25 | @Test
26 | void startup() {
27 |
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/stock-service/src/test/java/pl/piomin/samples/kafka/stock/TestStockService.java:
--------------------------------------------------------------------------------
1 | package pl.piomin.samples.kafka.stock;
2 |
3 | import org.springframework.boot.SpringApplication;
4 |
5 | public class TestStockService {
6 |
7 | public static void main(String[] args) {
8 | SpringApplication.from(StockService::main)
9 | .with(KafkaContainerDevMode.class)
10 | .run(args);
11 | }
12 |
13 | }
14 |
--------------------------------------------------------------------------------