├── .github └── FUNDING.yml ├── .gitignore ├── .mvn └── wrapper │ └── maven-wrapper.properties ├── README.md ├── avro-2-de-serialization ├── README.md ├── avro-2-consumer-service │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── ivanfranchin │ │ │ │ ├── avro2consumerservice │ │ │ │ ├── Avro2ConsumerServiceApplication.java │ │ │ │ └── news │ │ │ │ │ ├── NewsListener.java │ │ │ │ │ └── NewsListenerConfig.java │ │ │ │ └── commons │ │ │ │ └── avroserialization │ │ │ │ └── avro │ │ │ │ └── NewsMessage.java │ │ └── resources │ │ │ ├── application.properties │ │ │ ├── avro │ │ │ └── news-message.avsc │ │ │ └── banner.txt │ │ └── test │ │ └── java │ │ └── com │ │ └── ivanfranchin │ │ └── avro2consumerservice │ │ └── Avro2ConsumerServiceApplicationTests.java ├── avro-2-producer-service │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── ivanfranchin │ │ │ │ ├── avro2producerservice │ │ │ │ ├── Avro2ProducerServiceApplication.java │ │ │ │ ├── news │ │ │ │ │ ├── News.java │ │ │ │ │ ├── NewsEmitter.java │ │ │ │ │ └── NewsEmitterConfig.java │ │ │ │ └── simulation │ │ │ │ │ └── SimulationRunner.java │ │ │ │ └── commons │ │ │ │ └── avroserialization │ │ │ │ └── avro │ │ │ │ └── NewsMessage.java │ │ └── resources │ │ │ ├── application.properties │ │ │ ├── avro │ │ │ └── news-message.avsc │ │ │ └── banner.txt │ │ └── test │ │ └── java │ │ └── com │ │ └── ivanfranchin │ │ └── avro2producerservice │ │ └── Avro2ProducerServiceApplicationTests.java └── pom.xml ├── avro-3-de-serialization ├── README.md ├── avro-3-consumer-service │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── ivanfranchin │ │ │ │ └── avro3consumerservice │ │ │ │ ├── Avro3ConsumerServiceApplication.java │ │ │ │ ├── avro │ │ │ │ └── NewsMessage.java │ │ │ │ └── news │ │ │ │ ├── NewsListener.java │ │ │ │ ├── NewsListenerConfig.java │ │ │ │ └── SpecificAvroWithSchemaDeserializer.java │ │ └── resources │ │ │ ├── application.properties │ │ │ ├── avro │ │ │ └── news-message.avsc │ │ │ └── banner.txt │ │ └── test │ │ └── java │ │ └── com │ │ └── ivanfranchin │ │ └── avro3consumerservice │ │ └── Avro3ConsumerServiceApplicationTests.java ├── avro-3-producer-service │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── ivanfranchin │ │ │ │ └── avro3producerservice │ │ │ │ ├── Avro3ProducerServiceApplication.java │ │ │ │ ├── avro │ │ │ │ └── NewsMessage.java │ │ │ │ ├── news │ │ │ │ ├── News.java │ │ │ │ ├── NewsEmitter.java │ │ │ │ └── NewsEmitterConfig.java │ │ │ │ └── simulation │ │ │ │ └── SimulationRunner.java │ │ └── resources │ │ │ ├── application.properties │ │ │ ├── avro │ │ │ └── news-message.avsc │ │ │ └── banner.txt │ │ └── test │ │ └── java │ │ └── com │ │ └── ivanfranchin │ │ └── avro3producerservice │ │ └── Avro3ProducerServiceApplicationTests.java └── pom.xml ├── avro-de-serialization ├── README.md ├── avro-consumer-service │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── ivanfranchin │ │ │ │ └── avroconsumerservice │ │ │ │ ├── AvroConsumerServiceApplication.java │ │ │ │ ├── avro │ │ │ │ └── NewsMessage.java │ │ │ │ └── news │ │ │ │ ├── AvroDeserializer.java │ │ │ │ ├── NewsListener.java │ │ │ │ └── NewsListenerConfig.java │ │ └── resources │ │ │ ├── application.properties │ │ │ ├── avro │ │ │ └── news-message.avsc │ │ │ └── banner.txt │ │ └── test │ │ └── java │ │ └── com │ │ └── ivanfranchin │ │ └── avroconsumerservice │ │ └── AvroConsumerServiceApplicationTests.java ├── avro-producer-service │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── ivanfranchin │ │ │ │ └── avroproducerservice │ │ │ │ ├── AvroProducerServiceApplication.java │ │ │ │ ├── avro │ │ │ │ └── NewsMessage.java │ │ │ │ ├── news │ │ │ │ ├── AvroSerializer.java │ │ │ │ ├── News.java │ │ │ │ ├── NewsEmitter.java │ │ │ │ └── NewsEmitterConfig.java │ │ │ │ └── simulation │ │ │ │ └── SimulationRunner.java │ │ └── resources │ │ │ ├── application.properties │ │ │ ├── avro │ │ │ └── news-message.avsc │ │ │ └── banner.txt │ │ └── test │ │ └── java │ │ └── com │ │ └── ivanfranchin │ │ └── avroproducerservice │ │ └── AvroProducerServiceApplicationTests.java └── pom.xml ├── build-docker-images.sh ├── docker-compose.yml ├── documentation ├── avro-2-de-serialization.excalidraw ├── avro-2-de-serialization.jpeg ├── avro-3-de-serialization.excalidraw ├── avro-3-de-serialization.jpeg ├── avro-de-serialization.excalidraw ├── avro-de-serialization.jpeg ├── json-de-serialization.excalidraw ├── json-de-serialization.jpeg ├── string-de-serialization.excalidraw └── string-de-serialization.jpeg ├── json-de-serialization ├── README.md ├── json-consumer-service │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── ivanfranchin │ │ │ │ └── jsonconsumerservice │ │ │ │ ├── JsonConsumerServiceApplication.java │ │ │ │ └── news │ │ │ │ ├── News.java │ │ │ │ ├── NewsListener.java │ │ │ │ └── NewsListenerConfig.java │ │ └── resources │ │ │ ├── application.properties │ │ │ └── banner.txt │ │ └── test │ │ └── java │ │ └── com │ │ └── ivanfranchin │ │ └── jsonconsumerservice │ │ └── JsonConsumerServiceApplicationTests.java ├── json-producer-service │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── ivanfranchin │ │ │ │ └── jsonproducerservice │ │ │ │ ├── JsonProducerServiceApplication.java │ │ │ │ ├── news │ │ │ │ ├── News.java │ │ │ │ ├── NewsEmitter.java │ │ │ │ └── NewsEmitterConfig.java │ │ │ │ └── simulation │ │ │ │ └── SimulationRunner.java │ │ └── resources │ │ │ ├── application.properties │ │ │ └── banner.txt │ │ └── test │ │ └── java │ │ └── com │ │ └── ivanfranchin │ │ └── jsonproducerservice │ │ └── JsonProducerServiceApplicationTests.java └── pom.xml ├── mvnw ├── mvnw.cmd ├── my-functions.sh ├── pom.xml ├── remove-docker-images.sh └── string-de-serialization ├── README.md ├── pom.xml ├── string-consumer-service ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── ivanfranchin │ │ │ └── stringconsumerservice │ │ │ ├── StringConsumerServiceApplication.java │ │ │ └── news │ │ │ ├── NewsListener.java │ │ │ └── NewsListenerConfig.java │ └── resources │ │ ├── application.properties │ │ └── banner.txt │ └── test │ └── java │ └── com │ └── ivanfranchin │ └── jsonconsumerservice │ └── StringConsumerServiceApplicationTests.java └── string-producer-service ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── ivanfranchin │ │ └── stringproducerservice │ │ ├── StringProducerServiceApplication.java │ │ ├── news │ │ ├── News.java │ │ ├── NewsEmitter.java │ │ └── NewsEmitterConfig.java │ │ └── simulation │ │ └── SimulationRunner.java └── resources │ ├── application.properties │ └── banner.txt └── test └── java └── com └── ivanfranchin └── jsonproducerservice └── StringProducerServiceApplicationTests.java /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: ivangfr 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | !.mvn/wrapper/maven-wrapper.jar 3 | !**/src/main/**/target/ 4 | !**/src/test/**/target/ 5 | 6 | ### STS ### 7 | .apt_generated 8 | .classpath 9 | .factorypath 10 | .project 11 | .settings 12 | .springBeans 13 | .sts4-cache 14 | 15 | ### IntelliJ IDEA ### 16 | .idea 17 | *.iws 18 | *.iml 19 | *.ipr 20 | 21 | ### NetBeans ### 22 | /nbproject/private/ 23 | /nbbuild/ 24 | /dist/ 25 | /nbdist/ 26 | /.nb-gradle/ 27 | build/ 28 | !**/src/main/**/build/ 29 | !**/src/test/**/build/ 30 | 31 | ### VS Code ### 32 | .vscode/ 33 | 34 | ### MAC OS ### 35 | *.DS_Store 36 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | wrapperVersion=3.3.2 18 | distributionType=only-script 19 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # spring-kafka-de-serialization-types 2 | 3 | The goal of this project is to play with [`Spring for Apache Kafka`](https://docs.spring.io/spring-kafka/reference/index.html). We've implemented 5 examples of `producer` and `consumer` services that exchanges messages through [`Apache Kafka`](https://kafka.apache.org/) using different types of serialization and approaches. 4 | 5 | ## Proof-of-Concepts & Articles 6 | 7 | On [ivangfr.github.io](https://ivangfr.github.io), I have compiled my Proof-of-Concepts (PoCs) and articles. You can easily search for the technology you are interested in by using the filter. Who knows, perhaps I have already implemented a PoC or written an article about what you are looking for. 8 | 9 | ## Examples 10 | 11 | The following examples demonstrate a **producer** that pushes _"News"_ messages to a topic in `Kafka` and a **consumer** that listens those messages from `Kafka`. 12 | 13 | | Example | Diagram | 14 | |--------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------| 15 | | [string-de-serialization](https://github.com/ivangfr/spring-kafka-de-serialization-types/tree/master/string-de-serialization#spring-kafka-de-serialization-types) | ![project-diagram-samples](documentation/string-de-serialization.jpeg) | 16 | | [json-de-serialization](https://github.com/ivangfr/spring-kafka-de-serialization-types/tree/master/json-de-serialization#spring-kafka-de-serialization-types) | ![project-diagram-samples](documentation/json-de-serialization.jpeg) | 17 | | [avro-de-serialization](https://github.com/ivangfr/spring-kafka-de-serialization-types/tree/master/avro-de-serialization#spring-kafka-de-serialization-types) | ![project-diagram-samples](documentation/avro-de-serialization.jpeg) | 18 | | [avro-2-de-serialization](https://github.com/ivangfr/spring-kafka-de-serialization-types/tree/master/avro-2-de-serialization#spring-kafka-de-serialization-types) | ![project-diagram-samples](documentation/avro-2-de-serialization.jpeg) | 19 | | [avro-3-de-serialization](https://github.com/ivangfr/spring-kafka-de-serialization-types/tree/master/avro-3-de-serialization#spring-kafka-de-serialization-types) | ![project-diagram-samples](documentation/avro-3-de-serialization.jpeg) | 20 | 21 | ## Prerequisites 22 | 23 | - [`Java 21+`](https://www.oracle.com/java/technologies/downloads/#java21) 24 | - Some containerization tool [`Docker`](https://www.docker.com), [`Podman`](https://podman.io), etc. 25 | 26 | ## Start Environment 27 | 28 | - Open a terminal and inside the `spring-kafka-de-serialization-types` root folder run: 29 | ``` 30 | docker compose up -d 31 | ``` 32 | 33 | - Wait for Docker containers to be up and running. To check it, run: 34 | ``` 35 | docker ps -a 36 | ``` 37 | 38 | ## Useful Links 39 | 40 | - **Schema Registry UI** 41 | 42 | `Schema Registry UI` can be accessed at http://localhost:8001 43 | 44 | - **Kafka Topics UI** 45 | 46 | `Kafka Topics UI` can be accessed at http://localhost:8085 47 | 48 | - **Kafka Manager** 49 | 50 | `Kafka Manager` can be accessed at http://localhost:9000 51 | 52 | _Configuration_ 53 | - First, you must create a new cluster. Click on `Cluster` (dropdown on the header) and then on `Add Cluster`; 54 | - Type the name of your cluster in `Cluster Name` field, for example: `MyCluster`; 55 | - Type `zookeeper:2181` in `Cluster Zookeeper Hosts` field; 56 | - Enable checkbox `Poll consumer information (Not recommended for large # of consumers if ZK is used for offsets tracking on older Kafka versions)`; 57 | - Click on `Save` button at the bottom of the page. 58 | 59 | ## Shutdown 60 | 61 | To stop and remove docker compose containers, network and volumes, go to a terminal and, inside the `spring-kafka-de-serialization-types` root folder, run the command below: 62 | ``` 63 | docker compose down -v 64 | ``` 65 | 66 | ## Cleanup 67 | 68 | To remove the Docker created by this project, go to a terminal and, inside the `spring-kafka-de-serialization-types` root folder, run the following script: 69 | ``` 70 | ./remove-docker-images.sh all 71 | ``` 72 | 73 | ## References 74 | 75 | - https://codenotfound.com/spring-kafka-consumer-producer-example.html 76 | - https://codenotfound.com/spring-kafka-json-serializer-deserializer-example.html 77 | - https://codenotfound.com/spring-kafka-apache-avro-serializer-deserializer-example.html -------------------------------------------------------------------------------- /avro-2-de-serialization/README.md: -------------------------------------------------------------------------------- 1 | # spring-kafka-de-serialization-types 2 | ## `> avro-2-de-serialization` 3 | 4 | ![avro-2-de-serialization](../documentation/avro-2-de-serialization.jpeg) 5 | 6 | This sample demonstrates a **producer** that pushes `News` messages to a topic in `Kafka` and a **consumer** that listens those messages from `Kafka`: 7 | - **Producer** serializes the message `key` using `StringSerializer` and the message `value` using `KafkaAvroSerializer`; 8 | - **Consumer** deserializes the message `key` using `StringDeserializer` and the message `value` using `KafkaAvroDeserializer`; 9 | - This type of serialization uses `Schema Registry`; 10 | - The Java class generated from the `Avro` schema **MUST** have the same `name` and `package (or namespace)` in the producer and the consumer; 11 | - **Producer** creates the Kafka topics and **Consumer** doesn't. 12 | 13 | ## Start Environment 14 | 15 | Before starting producer and consumer, the services present in `docker-compose.yml` file must be up and running as explained in [Start Environment](https://github.com/ivangfr/spring-kafka-de-serialization-types#start-environment) section of the main README: 16 | 17 | ## Running applications using Maven 18 | 19 | > **Note**: run `avro-2-producer-service` first so that it can create the `Kafka` topics. 20 | 21 | - **avro-2-producer-service** 22 | 23 | - Open a terminal and navigate to the `spring-kafka-de-serialization-types` root folder; 24 | - Run application: 25 | ``` 26 | ./mvnw clean spring-boot:run --projects avro-2-de-serialization/avro-2-producer-service 27 | ``` 28 | > The Java class `com.ivanfranchin.commons.avroserialization.avro.NewsMessage` is generated by the Avro file `news-message.avsc` present in `src/main/resources/avro` by running the command: 29 | > ``` 30 | > ./mvnw generate-sources --projects avro-2-de-serialization/avro-2-producer-service 31 | > ``` 32 | - As soon as the producer is up and running, it will start pushing automatically and randomly `News` messages to `Kafka` topic `avro-2-de-serialization-news`. The default `interval` between messages is `3 seconds`. 33 | 34 | - **avro-2-consumer-service** 35 | 36 | - Open another terminal and make sure you are in `spring-kafka-de-serialization-types` root folder; 37 | - Run application: 38 | ``` 39 | ./mvnw clean spring-boot:run --projects avro-2-de-serialization/avro-2-consumer-service 40 | ``` 41 | > The Java class `com.ivanfranchin.commons.avroserialization.avro.NewsMessage` is generated by the Avro file `news-message.avsc` present in `src/main/resources/avro` by running the command: 42 | > ``` 43 | > ./mvnw generate-sources --projects avro-2-de-serialization/avro-2-consumer-service 44 | > ``` 45 | - Once the consumer is up and running, it will start listening `News` messages from the `Kafka` topic `avro-2-de-serialization-news`. 46 | 47 | ## Running applications as Docker containers 48 | 49 | - ### Build Docker images 50 | 51 | In a terminal and, inside the `spring-kafka-de-serialization-types` root folder, run: 52 | ``` 53 | ./build-docker-images.sh avro-2-de-serialization 54 | ``` 55 | 56 | - ### Environment variables 57 | 58 | **avro-2-producer-service** and **avro-2-consumer-service** 59 | 60 | | Environment Variable | Description | 61 | |------------------------|-------------------------------------------------------------------------| 62 | | `KAFKA_HOST` | Specify host of the `Kafka` message broker to use (default `localhost`) | 63 | | `KAFKA_PORT` | Specify port of the `Kafka` message broker to use (default `29092`) | 64 | | `SCHEMA_REGISTRY_HOST` | Specify host of the `Schema Registry` to use (default `localhost`) | 65 | | `SCHEMA_REGISTRY_PORT` | Specify port of the `Schema Registry` to use (default `8081`) | 66 | 67 | - ### Run Docker containers 68 | 69 | > **Note**: run `avro-2-producer-service` first so that it can create the `Kafka` topics. 70 | 71 | - **avro-2-producer-service** 72 | 73 | In a terminal, run the following Docker command: 74 | ``` 75 | docker run --rm --name avro-2-producer-service -p 9086:9086 \ 76 | -e KAFKA_HOST=kafka -e KAFKA_PORT=9092 -e SCHEMA_REGISTRY_HOST=schema-registry \ 77 | --network=spring-kafka-de-serialization-types_default \ 78 | ivanfranchin/avro-2-producer-service:1.0.0 79 | ``` 80 | 81 | - **avro-2-consumer-service** 82 | 83 | In another terminal, run the Docker command below: 84 | ``` 85 | docker run --rm --name avro-2-consumer-service -p 9087:9087 \ 86 | -e KAFKA_HOST=kafka -e KAFKA_PORT=9092 -e SCHEMA_REGISTRY_HOST=schema-registry \ 87 | --network=spring-kafka-de-serialization-types_default \ 88 | ivanfranchin/avro-2-consumer-service:1.0.0 89 | ``` 90 | 91 | ## Shutdown 92 | 93 | - Go to the terminals where the applications are running and press `Ctrl+C`; 94 | - Stop the services present in `docker-compose.yml` as explained in [Shutdown](https://github.com/ivangfr/spring-kafka-de-serialization-types#shutdown) section of the main README. 95 | 96 | ## Cleanup 97 | 98 | To remove the Docker images created by this example, go to a terminal and, inside the `spring-kafka-de-serialization-types` root folder, run the following script: 99 | ``` 100 | ./remove-docker-images.sh avro-2-de-serialization 101 | ``` 102 | -------------------------------------------------------------------------------- /avro-2-de-serialization/avro-2-consumer-service/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.ivanfranchin 7 | avro-2-de-serialization 8 | 1.0.0 9 | ../pom.xml 10 | 11 | avro-2-consumer-service 12 | avro-2-consumer-service 13 | Demo project for Spring Boot 14 | 15 | 16 | org.springframework.boot 17 | spring-boot-starter-web 18 | 19 | 20 | org.springframework.kafka 21 | spring-kafka 22 | 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-starter-test 27 | test 28 | 29 | 30 | org.springframework.kafka 31 | spring-kafka-test 32 | test 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /avro-2-de-serialization/avro-2-consumer-service/src/main/java/com/ivanfranchin/avro2consumerservice/Avro2ConsumerServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro2consumerservice; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class Avro2ConsumerServiceApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(Avro2ConsumerServiceApplication.class, args); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /avro-2-de-serialization/avro-2-consumer-service/src/main/java/com/ivanfranchin/avro2consumerservice/news/NewsListener.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro2consumerservice.news; 2 | 3 | import com.ivanfranchin.commons.avroserialization.avro.NewsMessage; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.kafka.annotation.KafkaListener; 7 | import org.springframework.kafka.listener.adapter.ConsumerRecordMetadata; 8 | import org.springframework.messaging.handler.annotation.Payload; 9 | import org.springframework.stereotype.Service; 10 | 11 | @Service 12 | public class NewsListener { 13 | 14 | private static final Logger log = LoggerFactory.getLogger(NewsListener.class); 15 | 16 | @KafkaListener(topics = "${spring.kafka.consumer.topic}", groupId = "${spring.kafka.consumer.group-id}") 17 | public void listen(@Payload NewsMessage newsMessage, ConsumerRecordMetadata metadata) { 18 | log.info("Received message\n---\nTOPIC: {}; PARTITION: {}; OFFSET: {};\nPAYLOAD: {}\n---", 19 | metadata.topic(), metadata.partition(), metadata.offset(), newsMessage); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /avro-2-de-serialization/avro-2-consumer-service/src/main/java/com/ivanfranchin/avro2consumerservice/news/NewsListenerConfig.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro2consumerservice.news; 2 | 3 | import com.ivanfranchin.commons.avroserialization.avro.NewsMessage; 4 | import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; 5 | import io.confluent.kafka.serializers.KafkaAvroDeserializer; 6 | import io.confluent.kafka.serializers.KafkaAvroDeserializerConfig; 7 | import org.apache.kafka.clients.consumer.ConsumerConfig; 8 | import org.apache.kafka.common.serialization.StringDeserializer; 9 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 10 | import org.springframework.context.annotation.Bean; 11 | import org.springframework.context.annotation.Configuration; 12 | import org.springframework.kafka.annotation.EnableKafka; 13 | import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; 14 | import org.springframework.kafka.core.ConsumerFactory; 15 | import org.springframework.kafka.core.DefaultKafkaConsumerFactory; 16 | 17 | import java.util.Map; 18 | 19 | @EnableKafka 20 | @Configuration 21 | public class NewsListenerConfig { 22 | 23 | private final KafkaProperties kafkaProperties; 24 | 25 | public NewsListenerConfig(KafkaProperties kafkaProperties) { 26 | this.kafkaProperties = kafkaProperties; 27 | } 28 | 29 | @Bean 30 | ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory() { 31 | ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); 32 | factory.setConsumerFactory(consumerFactory()); 33 | factory.setConcurrency(kafkaProperties.getListener().getConcurrency()); 34 | return factory; 35 | } 36 | 37 | @Bean 38 | ConsumerFactory consumerFactory() { 39 | Map props = kafkaProperties.buildConsumerProperties(null); 40 | props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); 41 | props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, KafkaAvroDeserializer.class); 42 | props.put(ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, false); 43 | 44 | props.put(AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, kafkaProperties.getProperties().get("schema-registry-url")); 45 | props.put(KafkaAvroDeserializerConfig.SPECIFIC_AVRO_READER_CONFIG, true); 46 | return new DefaultKafkaConsumerFactory<>(props); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /avro-2-de-serialization/avro-2-consumer-service/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=9087 2 | 3 | spring.application.name=avro-2-consumer-service 4 | 5 | spring.kafka.bootstrap-servers=${KAFKA_HOST:localhost}:${KAFKA_PORT:29092} 6 | spring.kafka.properties.schema-registry-url=http://${SCHEMA_REGISTRY_HOST:localhost}:${SCHEMA_REGISTRY_PORT:8081} 7 | spring.kafka.consumer.group-id=avro-2-consumer-service-group 8 | spring.kafka.consumer.topic=avro-2-de-serialization-news 9 | spring.kafka.consumer.auto-offset-reset=earliest 10 | spring.kafka.listener.concurrency=2 -------------------------------------------------------------------------------- /avro-2-de-serialization/avro-2-consumer-service/src/main/resources/avro/news-message.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "com.ivanfranchin.commons.avroserialization.avro", 3 | "type": "record", 4 | "name": "NewsMessage", 5 | "fields": [ 6 | {"name": "id", "type": "string"}, 7 | {"name": "fromId", "type": "int"}, 8 | {"name": "fromName", "type": "string"}, 9 | {"name": "title", "type": "string"} 10 | ] 11 | } -------------------------------------------------------------------------------- /avro-2-de-serialization/avro-2-consumer-service/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | ____ _ 2 | __ ___ ___ __ ___ |___ \ ___ ___ _ __ ___ _ _ _ __ ___ ___ _ __ ___ ___ _ ____ _(_) ___ ___ 3 | / _` \ \ / / '__/ _ \ _____ __) |____ / __/ _ \| '_ \/ __| | | | '_ ` _ \ / _ \ '__|____/ __|/ _ \ '__\ \ / / |/ __/ _ \ 4 | | (_| |\ V /| | | (_) |_____/ __/_____| (_| (_) | | | \__ \ |_| | | | | | | __/ | |_____\__ \ __/ | \ V /| | (_| __/ 5 | \__,_| \_/ |_| \___/ |_____| \___\___/|_| |_|___/\__,_|_| |_| |_|\___|_| |___/\___|_| \_/ |_|\___\___| 6 | :: Spring Boot :: ${spring-boot.formatted-version} 7 | -------------------------------------------------------------------------------- /avro-2-de-serialization/avro-2-consumer-service/src/test/java/com/ivanfranchin/avro2consumerservice/Avro2ConsumerServiceApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro2consumerservice; 2 | 3 | import org.junit.jupiter.api.Disabled; 4 | import org.junit.jupiter.api.Test; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | 7 | @Disabled 8 | @SpringBootTest 9 | class Avro2ConsumerServiceApplicationTests { 10 | 11 | @Test 12 | void contextLoads() { 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /avro-2-de-serialization/avro-2-producer-service/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.ivanfranchin 7 | avro-2-de-serialization 8 | 1.0.0 9 | ../pom.xml 10 | 11 | avro-2-producer-service 12 | avro-2-producer-service 13 | Demo project for Spring Boot 14 | 15 | 16 | org.springframework.boot 17 | spring-boot-starter-web 18 | 19 | 20 | org.springframework.kafka 21 | spring-kafka 22 | 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-starter-test 27 | test 28 | 29 | 30 | org.springframework.kafka 31 | spring-kafka-test 32 | test 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /avro-2-de-serialization/avro-2-producer-service/src/main/java/com/ivanfranchin/avro2producerservice/Avro2ProducerServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro2producerservice; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class Avro2ProducerServiceApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(Avro2ProducerServiceApplication.class, args); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /avro-2-de-serialization/avro-2-producer-service/src/main/java/com/ivanfranchin/avro2producerservice/news/News.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro2producerservice.news; 2 | 3 | public record News(String id, Integer fromId, String fromName, String title) { 4 | } 5 | -------------------------------------------------------------------------------- /avro-2-de-serialization/avro-2-producer-service/src/main/java/com/ivanfranchin/avro2producerservice/news/NewsEmitter.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro2producerservice.news; 2 | 3 | import com.ivanfranchin.commons.avroserialization.avro.NewsMessage; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 7 | import org.springframework.kafka.core.KafkaTemplate; 8 | import org.springframework.stereotype.Component; 9 | 10 | @Component 11 | public class NewsEmitter { 12 | 13 | private static final Logger log = LoggerFactory.getLogger(NewsEmitter.class); 14 | 15 | private final KafkaTemplate kafkaTemplate; 16 | private final KafkaProperties kafkaProperties; 17 | 18 | public NewsEmitter(KafkaTemplate kafkaTemplate, KafkaProperties kafkaProperties) { 19 | this.kafkaTemplate = kafkaTemplate; 20 | this.kafkaProperties = kafkaProperties; 21 | } 22 | 23 | public void send(NewsMessage newsMessage) { 24 | String kafkaTopic = kafkaProperties.getProducer().getProperties().get("topic"); 25 | log.info("Sending News '{}' to topic '{}'", newsMessage, kafkaTopic); 26 | kafkaTemplate.send(kafkaTopic, String.valueOf(newsMessage.getFromId()), newsMessage); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /avro-2-de-serialization/avro-2-producer-service/src/main/java/com/ivanfranchin/avro2producerservice/news/NewsEmitterConfig.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro2producerservice.news; 2 | 3 | import com.ivanfranchin.commons.avroserialization.avro.NewsMessage; 4 | import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; 5 | import io.confluent.kafka.serializers.KafkaAvroSerializer; 6 | import org.apache.kafka.clients.admin.AdminClientConfig; 7 | import org.apache.kafka.clients.admin.NewTopic; 8 | import org.apache.kafka.clients.producer.ProducerConfig; 9 | import org.apache.kafka.common.serialization.StringSerializer; 10 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 11 | import org.springframework.context.annotation.Bean; 12 | import org.springframework.context.annotation.Configuration; 13 | import org.springframework.kafka.core.DefaultKafkaProducerFactory; 14 | import org.springframework.kafka.core.KafkaAdmin; 15 | import org.springframework.kafka.core.KafkaTemplate; 16 | import org.springframework.kafka.core.ProducerFactory; 17 | 18 | import java.util.HashMap; 19 | import java.util.Map; 20 | 21 | @Configuration 22 | public class NewsEmitterConfig { 23 | 24 | private final KafkaProperties kafkaProperties; 25 | 26 | public NewsEmitterConfig(KafkaProperties kafkaProperties) { 27 | this.kafkaProperties = kafkaProperties; 28 | } 29 | 30 | @Bean 31 | KafkaTemplate kafkaTemplate() { 32 | return new KafkaTemplate<>(producerFactory()); 33 | } 34 | 35 | @Bean 36 | ProducerFactory producerFactory() { 37 | Map props = kafkaProperties.buildProducerProperties(null); 38 | props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); 39 | props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class); 40 | props.put(AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, kafkaProperties.getProperties().get("schema-registry-url")); 41 | return new DefaultKafkaProducerFactory<>(props); 42 | } 43 | 44 | // As the application will create a topic in Kafka, it is advisable to update the BOOTSTRAP_SERVERS_CONFIG 45 | // property of KafkaAdmin with the configured bootstrap server URLs. Otherwise, it will default to 'localhost:9082'. 46 | @Bean 47 | KafkaAdmin kafkaAdmin() { 48 | Map configs = new HashMap<>(); 49 | configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBootstrapServers()); 50 | return new KafkaAdmin(configs); 51 | } 52 | 53 | @Bean 54 | NewTopic newTopic() { 55 | Map producerProperties = kafkaProperties.getProducer().getProperties(); 56 | return new NewTopic(producerProperties.get("topic"), Integer.parseInt(producerProperties.get("num-partitions")), (short) 1); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /avro-2-de-serialization/avro-2-producer-service/src/main/java/com/ivanfranchin/avro2producerservice/simulation/SimulationRunner.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro2producerservice.simulation; 2 | 3 | import com.ivanfranchin.avro2producerservice.news.News; 4 | import com.ivanfranchin.avro2producerservice.news.NewsEmitter; 5 | import com.ivanfranchin.commons.avroserialization.avro.NewsMessage; 6 | import org.springframework.boot.CommandLineRunner; 7 | import org.springframework.stereotype.Component; 8 | 9 | import java.util.UUID; 10 | import java.util.concurrent.Executors; 11 | import java.util.concurrent.TimeUnit; 12 | 13 | @Component 14 | public class SimulationRunner implements CommandLineRunner { 15 | 16 | private final NewsEmitter newsEmitter; 17 | 18 | public SimulationRunner(NewsEmitter newsEmitter) { 19 | this.newsEmitter = newsEmitter; 20 | } 21 | 22 | @Override 23 | public void run(String... args) { 24 | Executors.newSingleThreadScheduledExecutor() 25 | .scheduleAtFixedRate(this::generateAndSendNews, 0, 3, TimeUnit.SECONDS); 26 | } 27 | 28 | private void generateAndSendNews() { 29 | newsEmitter.send(toNewsMessage(generate())); 30 | } 31 | 32 | private NewsMessage toNewsMessage(News news) { 33 | return new NewsMessage(news.id(), news.fromId(), news.fromName(), news.title()); 34 | } 35 | 36 | private News generate() { 37 | return new News(UUID.randomUUID().toString(), 1, "Channel", "Title"); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /avro-2-de-serialization/avro-2-producer-service/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=9086 2 | 3 | spring.application.name=avro-2-producer-service 4 | 5 | spring.kafka.bootstrap-servers=${KAFKA_HOST:localhost}:${KAFKA_PORT:29092} 6 | spring.kafka.properties.schema-registry-url=http://${SCHEMA_REGISTRY_HOST:localhost}:${SCHEMA_REGISTRY_PORT:8081} 7 | spring.kafka.producer.properties.topic=avro-2-de-serialization-news 8 | spring.kafka.producer.properties.num-partitions=4 9 | -------------------------------------------------------------------------------- /avro-2-de-serialization/avro-2-producer-service/src/main/resources/avro/news-message.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "com.ivanfranchin.commons.avroserialization.avro", 3 | "type": "record", 4 | "name": "NewsMessage", 5 | "fields": [ 6 | {"name": "id", "type": "string"}, 7 | {"name": "fromId", "type": "int"}, 8 | {"name": "fromName", "type": "string"}, 9 | {"name": "title", "type": "string"} 10 | ] 11 | } -------------------------------------------------------------------------------- /avro-2-de-serialization/avro-2-producer-service/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | ____ _ _ 2 | __ ___ ___ __ ___ |___ \ _ __ _ __ ___ __| |_ _ ___ ___ _ __ ___ ___ _ ____ _(_) ___ ___ 3 | / _` \ \ / / '__/ _ \ _____ __) |____| '_ \| '__/ _ \ / _` | | | |/ __/ _ \ '__|____/ __|/ _ \ '__\ \ / / |/ __/ _ \ 4 | | (_| |\ V /| | | (_) |_____/ __/_____| |_) | | | (_) | (_| | |_| | (_| __/ | |_____\__ \ __/ | \ V /| | (_| __/ 5 | \__,_| \_/ |_| \___/ |_____| | .__/|_| \___/ \__,_|\__,_|\___\___|_| |___/\___|_| \_/ |_|\___\___| 6 | |_| 7 | :: Spring Boot :: ${spring-boot.formatted-version} 8 | -------------------------------------------------------------------------------- /avro-2-de-serialization/avro-2-producer-service/src/test/java/com/ivanfranchin/avro2producerservice/Avro2ProducerServiceApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro2producerservice; 2 | 3 | import org.junit.jupiter.api.Disabled; 4 | import org.junit.jupiter.api.Test; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | 7 | @Disabled 8 | @SpringBootTest 9 | class Avro2ProducerServiceApplicationTests { 10 | 11 | @Test 12 | void contextLoads() { 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /avro-2-de-serialization/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.ivanfranchin 7 | spring-kafka-de-serialization-types 8 | 1.0.0 9 | ../pom.xml 10 | 11 | avro-2-de-serialization 12 | pom 13 | avro-2-de-serialization 14 | Demo project for Spring Boot 15 | 16 | 1.12.0 17 | 7.8.0 18 | 19 | 20 | 21 | 22 | org.apache.avro 23 | avro 24 | ${avro.version} 25 | 26 | 27 | 28 | 29 | io.confluent 30 | kafka-avro-serializer 31 | ${confluent-kafka-avro-serializer.version} 32 | 33 | 34 | 35 | 36 | 37 | 38 | org.springframework.boot 39 | spring-boot-maven-plugin 40 | 41 | 42 | org.apache.avro 43 | avro-maven-plugin 44 | ${avro.version} 45 | 46 | 47 | generate-sources 48 | 49 | schema 50 | 51 | 52 | ${project.basedir}/src/main/resources/avro/ 53 | ${project.basedir}/src/main/java 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | confluent 64 | https://packages.confluent.io/maven/ 65 | 66 | 67 | 68 | 69 | avro-2-producer-service 70 | avro-2-consumer-service 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /avro-3-de-serialization/README.md: -------------------------------------------------------------------------------- 1 | # spring-kafka-de-serialization-types 2 | ## `> avro-3-de-serialization` 3 | 4 | ![avro-3-de-serialization](../documentation/avro-3-de-serialization.jpeg) 5 | 6 | This sample demonstrates a **producer** that pushes `News` messages to a topic in `Kafka` and a **consumer** that listens those messages from `Kafka`: 7 | - **Producer** serializes the message `key` using `StringSerializer` and the message `value` using `KafkaAvroSerializer`; 8 | - **Consumer** deserializes message `key` using `StringSerializer` to the message `value` using `SpecificAvroWithSchemaDeserializer`; 9 | - This type of serialization/deserialization uses `Schema Registry`; 10 | - The Java class generated from the `Avro` schema **DOES NOT** need to have the same `name` and `package (or namespace)` in the producer and the consumer; 11 | - We needed to implement the `SpecificAvroWithSchemaDeserializer` class; 12 | - **Producer** creates the Kafka topics and **Consumer** doesn't. 13 | 14 | ## Start Environment 15 | 16 | Before starting producer and consumer, the services present in `docker-compose.yml` file must be up and running as explained in [Start Environment](https://github.com/ivangfr/spring-kafka-de-serialization-types#start-environment) section of the main README. 17 | 18 | ## Running applications using Maven 19 | 20 | > **Note**: run `avro-3-producer-service` first so that it can create the `Kafka` topics. 21 | 22 | - **avro-3-producer-service** 23 | 24 | - Open a terminal and navigate to the `spring-kafka-de-serialization-types` root folder; 25 | - Run application: 26 | ``` 27 | ./mvnw clean spring-boot:run --projects avro-3-de-serialization/avro-3-producer-service 28 | ``` 29 | > The Java class `com.ivanfranchin.avro3producerservice.avro.NewsMessage` is generated by the Avro file `news-message.avsc` present in `src/main/resources/avro` by running the command: 30 | > ``` 31 | > ./mvnw generate-sources --projects avro-3-de-serialization/avro-3-producer-service 32 | > ``` 33 | - As soon as the producer is up and running, it will start pushing automatically and randomly `News` messages to `Kafka` topic `avro-3-de-serialization-news`. The default `interval` between messages is `3 seconds`. 34 | 35 | - **avro-3-consumer-service** 36 | 37 | - Open another terminal and make sure you are in `spring-kafka-de-serialization-types` root folder; 38 | - Run application: 39 | ``` 40 | ./mvnw clean spring-boot:run --projects avro-3-de-serialization/avro-3-consumer-service 41 | ``` 42 | > The Java class `com.ivanfranchin.avro3consumerservice.avro.NewsMessage` is generated by the Avro file `news-message.avsc` present in `src/main/resources/avro` by running the command: 43 | > ``` 44 | > ./mvnw generate-sources --projects avro-3-de-serialization/avro-3-consumer-service 45 | > ``` 46 | - Once the consumer is up and running, it will start listening `News` messages from the `Kafka` topic `avro-3-de-serialization-news`. 47 | 48 | ## Running applications as Docker containers 49 | 50 | - ### Build Docker images 51 | 52 | In a terminal and, inside the `spring-kafka-de-serialization-types` root folder, run: 53 | ``` 54 | ./build-docker-images.sh avro-3-de-serialization 55 | ``` 56 | 57 | - ### Environment variables 58 | 59 | **avro-3-producer-service** and **avro-3-consumer-service** 60 | 61 | | Environment Variable | Description | 62 | |------------------------|-------------------------------------------------------------------------| 63 | | `KAFKA_HOST` | Specify host of the `Kafka` message broker to use (default `localhost`) | 64 | | `KAFKA_PORT` | Specify port of the `Kafka` message broker to use (default `29092`) | 65 | | `SCHEMA_REGISTRY_HOST` | Specify host of the `Schema Registry` to use (default `localhost`) | 66 | | `SCHEMA_REGISTRY_PORT` | Specify port of the `Schema Registry` to use (default `8081`) | 67 | 68 | - ### Run Docker containers 69 | 70 | > **Note**: run `avro-3-producer-service` first so that it can create the `Kafka` topics. 71 | 72 | - **avro-3-producer-service** 73 | 74 | In a terminal, run the following Docker command: 75 | ``` 76 | docker run --rm --name avro-3-producer-service -p 9088:9088 \ 77 | -e KAFKA_HOST=kafka -e KAFKA_PORT=9092 -e SCHEMA_REGISTRY_HOST=schema-registry \ 78 | --network=spring-kafka-de-serialization-types_default \ 79 | ivanfranchin/avro-3-producer-service:1.0.0 80 | ``` 81 | 82 | - **avro-3-consumer-service** 83 | 84 | In another terminal, run the Docker command below: 85 | ``` 86 | docker run --rm --name avro-3-consumer-service -p 9089:9089 \ 87 | -e KAFKA_HOST=kafka -e KAFKA_PORT=9092 -e SCHEMA_REGISTRY_HOST=schema-registry \ 88 | --network=spring-kafka-de-serialization-types_default \ 89 | ivanfranchin/avro-3-consumer-service:1.0.0 90 | ``` 91 | 92 | ## Shutdown 93 | 94 | - Go to the terminals where the applications are running and press `Ctrl+C`; 95 | - Stop the services present in `docker-compose.yml` as explained in [Shutdown](https://github.com/ivangfr/spring-kafka-de-serialization-types#shutdown) section of the main README. 96 | 97 | ## Cleanup 98 | 99 | To remove the Docker images created by this example, go to a terminal and, inside the `spring-kafka-de-serialization-types` root folder, run the following script: 100 | ``` 101 | ./remove-docker-images.sh avro-3-de-serialization 102 | ``` 103 | -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-consumer-service/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.ivanfranchin 7 | avro-3-de-serialization 8 | 1.0.0 9 | ../pom.xml 10 | 11 | avro-3-consumer-service 12 | avro-3-consumer-service 13 | Demo project for Spring Boot 14 | 15 | 16 | org.springframework.boot 17 | spring-boot-starter-web 18 | 19 | 20 | org.springframework.kafka 21 | spring-kafka 22 | 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-starter-test 27 | test 28 | 29 | 30 | org.springframework.kafka 31 | spring-kafka-test 32 | test 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-consumer-service/src/main/java/com/ivanfranchin/avro3consumerservice/Avro3ConsumerServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro3consumerservice; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class Avro3ConsumerServiceApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(Avro3ConsumerServiceApplication.class, args); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-consumer-service/src/main/java/com/ivanfranchin/avro3consumerservice/news/NewsListener.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro3consumerservice.news; 2 | 3 | import com.ivanfranchin.avro3consumerservice.avro.NewsMessage; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.kafka.annotation.KafkaListener; 7 | import org.springframework.kafka.listener.adapter.ConsumerRecordMetadata; 8 | import org.springframework.messaging.handler.annotation.Payload; 9 | import org.springframework.stereotype.Service; 10 | 11 | @Service 12 | public class NewsListener { 13 | 14 | private static final Logger log = LoggerFactory.getLogger(NewsListener.class); 15 | 16 | @KafkaListener(topics = "${spring.kafka.consumer.topic}", groupId = "${spring.kafka.consumer.group-id}") 17 | public void listen(@Payload NewsMessage newsMessage, ConsumerRecordMetadata metadata) { 18 | log.info("Received message\n---\nTOPIC: {}; PARTITION: {}; OFFSET: {};\nPAYLOAD: {}\n---", 19 | metadata.topic(), metadata.partition(), metadata.offset(), newsMessage); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-consumer-service/src/main/java/com/ivanfranchin/avro3consumerservice/news/NewsListenerConfig.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro3consumerservice.news; 2 | 3 | import com.ivanfranchin.avro3consumerservice.avro.NewsMessage; 4 | import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; 5 | import io.confluent.kafka.serializers.KafkaAvroDeserializerConfig; 6 | import org.apache.kafka.clients.consumer.ConsumerConfig; 7 | import org.apache.kafka.common.serialization.StringDeserializer; 8 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 9 | import org.springframework.context.annotation.Bean; 10 | import org.springframework.context.annotation.Configuration; 11 | import org.springframework.kafka.annotation.EnableKafka; 12 | import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; 13 | import org.springframework.kafka.core.ConsumerFactory; 14 | import org.springframework.kafka.core.DefaultKafkaConsumerFactory; 15 | 16 | import java.util.Map; 17 | 18 | @EnableKafka 19 | @Configuration 20 | public class NewsListenerConfig { 21 | 22 | private final KafkaProperties kafkaProperties; 23 | 24 | public NewsListenerConfig(KafkaProperties kafkaProperties) { 25 | this.kafkaProperties = kafkaProperties; 26 | } 27 | 28 | @Bean 29 | ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory() { 30 | ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); 31 | factory.setConsumerFactory(consumerFactory()); 32 | factory.setConcurrency(kafkaProperties.getListener().getConcurrency()); 33 | return factory; 34 | } 35 | 36 | @Bean 37 | ConsumerFactory consumerFactory() { 38 | Map props = kafkaProperties.buildConsumerProperties(null); 39 | props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); 40 | props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, SpecificAvroWithSchemaDeserializer.class); 41 | props.put(ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, false); 42 | 43 | props.put(AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, kafkaProperties.getProperties().get("schema-registry-url")); 44 | props.put(KafkaAvroDeserializerConfig.SPECIFIC_AVRO_READER_CONFIG, true); 45 | props.put(SpecificAvroWithSchemaDeserializer.AVRO_VALUE_RECORD_TYPE, NewsMessage.class); 46 | return new DefaultKafkaConsumerFactory<>(props); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-consumer-service/src/main/java/com/ivanfranchin/avro3consumerservice/news/SpecificAvroWithSchemaDeserializer.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro3consumerservice.news; 2 | 3 | import io.confluent.kafka.serializers.AbstractKafkaAvroDeserializer; 4 | import io.confluent.kafka.serializers.KafkaAvroDeserializerConfig; 5 | import org.apache.avro.Schema; 6 | import org.apache.kafka.common.serialization.Deserializer; 7 | 8 | import java.lang.reflect.Field; 9 | import java.util.Map; 10 | 11 | public class SpecificAvroWithSchemaDeserializer extends AbstractKafkaAvroDeserializer implements Deserializer { 12 | 13 | public static final String AVRO_KEY_RECORD_TYPE = "avro.key.record.type"; 14 | public static final String AVRO_VALUE_RECORD_TYPE = "avro.value.record.type"; 15 | 16 | private Schema readerSchema; 17 | 18 | public SpecificAvroWithSchemaDeserializer() { 19 | } 20 | 21 | @Override 22 | public void configure(Map props, boolean isKey) { 23 | this.configure(new KafkaAvroDeserializerConfig(props)); 24 | readerSchema = getSchema(getRecordClass(props, isKey)); 25 | } 26 | 27 | @Override 28 | public Object deserialize(String s, byte[] bytes) { 29 | return super.deserialize(bytes, readerSchema); 30 | } 31 | 32 | @Override 33 | public void close() { 34 | } 35 | 36 | private Class getRecordClass(Map configs, boolean isKey) { 37 | String configsKey = isKey ? AVRO_KEY_RECORD_TYPE : AVRO_VALUE_RECORD_TYPE; 38 | Object configsValue = configs.get(configsKey); 39 | 40 | if (configsValue instanceof Class) { 41 | return (Class) configsValue; 42 | } else if (configsValue instanceof String) { 43 | String recordClassName = (String) configsValue; 44 | try { 45 | return Class.forName(recordClassName); 46 | } catch (ClassNotFoundException e) { 47 | throw new IllegalArgumentException(String.format("Unable to find the class '%s'", recordClassName)); 48 | } 49 | } else { 50 | throw new IllegalArgumentException( 51 | String.format("A class or a string must be informed into ConsumerConfig properties: '%s' and/or '%s'", 52 | AVRO_KEY_RECORD_TYPE, AVRO_VALUE_RECORD_TYPE)); 53 | } 54 | } 55 | 56 | private Schema getSchema(Class targetType) { 57 | try { 58 | Field field = targetType.getDeclaredField("SCHEMA$"); 59 | return (Schema) field.get(null); 60 | } catch (NoSuchFieldException | IllegalAccessException e) { 61 | throw new IllegalArgumentException( 62 | String.format("Unable to get Avro Schema from the class '%s'", targetType.getName()), e); 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-consumer-service/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=9089 2 | 3 | spring.application.name=avro-3-consumer-service 4 | 5 | spring.kafka.bootstrap-servers=${KAFKA_HOST:localhost}:${KAFKA_PORT:29092} 6 | spring.kafka.properties.schema-registry-url=http://${SCHEMA_REGISTRY_HOST:localhost}:${SCHEMA_REGISTRY_PORT:8081} 7 | spring.kafka.consumer.group-id=avro-3-consumer-service-group 8 | spring.kafka.consumer.topic=avro-3-de-serialization-news 9 | spring.kafka.consumer.auto-offset-reset=earliest 10 | spring.kafka.listener.concurrency=2 -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-consumer-service/src/main/resources/avro/news-message.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "com.ivanfranchin.avro3consumerservice.avro", 3 | "type": "record", 4 | "name": "NewsMessage", 5 | "fields": [ 6 | {"name": "id", "type": "string"}, 7 | {"name": "fromId", "type": "int"}, 8 | {"name": "fromName", "type": "string"}, 9 | {"name": "title", "type": "string"} 10 | ] 11 | } -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-consumer-service/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _____ _ 2 | __ ___ ___ __ ___ |___ / ___ ___ _ __ ___ _ _ _ __ ___ ___ _ __ ___ ___ _ ____ _(_) ___ ___ 3 | / _` \ \ / / '__/ _ \ _____ |_ \ _____ / __/ _ \| '_ \/ __| | | | '_ ` _ \ / _ \ '__|____/ __|/ _ \ '__\ \ / / |/ __/ _ \ 4 | | (_| |\ V /| | | (_) |_____|__) |_____| (_| (_) | | | \__ \ |_| | | | | | | __/ | |_____\__ \ __/ | \ V /| | (_| __/ 5 | \__,_| \_/ |_| \___/ |____/ \___\___/|_| |_|___/\__,_|_| |_| |_|\___|_| |___/\___|_| \_/ |_|\___\___| 6 | :: Spring Boot :: ${spring-boot.formatted-version} 7 | -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-consumer-service/src/test/java/com/ivanfranchin/avro3consumerservice/Avro3ConsumerServiceApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro3consumerservice; 2 | 3 | import org.junit.jupiter.api.Disabled; 4 | import org.junit.jupiter.api.Test; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | 7 | @Disabled 8 | @SpringBootTest 9 | class Avro3ConsumerServiceApplicationTests { 10 | 11 | @Test 12 | void contextLoads() { 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-producer-service/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.ivanfranchin 7 | avro-3-de-serialization 8 | 1.0.0 9 | ../pom.xml 10 | 11 | avro-3-producer-service 12 | avro-3-producer-service 13 | Demo project for Spring Boot 14 | 15 | 16 | org.springframework.boot 17 | spring-boot-starter-web 18 | 19 | 20 | org.springframework.kafka 21 | spring-kafka 22 | 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-starter-test 27 | test 28 | 29 | 30 | org.springframework.kafka 31 | spring-kafka-test 32 | test 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-producer-service/src/main/java/com/ivanfranchin/avro3producerservice/Avro3ProducerServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro3producerservice; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class Avro3ProducerServiceApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(Avro3ProducerServiceApplication.class, args); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-producer-service/src/main/java/com/ivanfranchin/avro3producerservice/news/News.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro3producerservice.news; 2 | 3 | public record News(String id, Integer fromId, String fromName, String title) { 4 | } 5 | -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-producer-service/src/main/java/com/ivanfranchin/avro3producerservice/news/NewsEmitter.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro3producerservice.news; 2 | 3 | import com.ivanfranchin.avro3producerservice.avro.NewsMessage; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 7 | import org.springframework.kafka.core.KafkaTemplate; 8 | import org.springframework.stereotype.Component; 9 | 10 | @Component 11 | public class NewsEmitter { 12 | 13 | private static final Logger log = LoggerFactory.getLogger(NewsEmitter.class); 14 | 15 | private final KafkaTemplate kafkaTemplate; 16 | private final KafkaProperties kafkaProperties; 17 | 18 | public NewsEmitter(KafkaTemplate kafkaTemplate, KafkaProperties kafkaProperties) { 19 | this.kafkaTemplate = kafkaTemplate; 20 | this.kafkaProperties = kafkaProperties; 21 | } 22 | 23 | public void send(NewsMessage newsMessage) { 24 | String kafkaTopic = kafkaProperties.getProducer().getProperties().get("topic"); 25 | log.info("Sending News '{}' to topic '{}'", newsMessage, kafkaTopic); 26 | kafkaTemplate.send(kafkaTopic, String.valueOf(newsMessage.getFromId()), newsMessage); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-producer-service/src/main/java/com/ivanfranchin/avro3producerservice/news/NewsEmitterConfig.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro3producerservice.news; 2 | 3 | import com.ivanfranchin.avro3producerservice.avro.NewsMessage; 4 | import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; 5 | import io.confluent.kafka.serializers.KafkaAvroSerializer; 6 | import org.apache.kafka.clients.admin.AdminClientConfig; 7 | import org.apache.kafka.clients.admin.NewTopic; 8 | import org.apache.kafka.clients.producer.ProducerConfig; 9 | import org.apache.kafka.common.serialization.StringSerializer; 10 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 11 | import org.springframework.context.annotation.Bean; 12 | import org.springframework.context.annotation.Configuration; 13 | import org.springframework.kafka.core.DefaultKafkaProducerFactory; 14 | import org.springframework.kafka.core.KafkaAdmin; 15 | import org.springframework.kafka.core.KafkaTemplate; 16 | import org.springframework.kafka.core.ProducerFactory; 17 | 18 | import java.util.HashMap; 19 | import java.util.Map; 20 | 21 | @Configuration 22 | public class NewsEmitterConfig { 23 | 24 | private final KafkaProperties kafkaProperties; 25 | 26 | public NewsEmitterConfig(KafkaProperties kafkaProperties) { 27 | this.kafkaProperties = kafkaProperties; 28 | } 29 | 30 | @Bean 31 | KafkaTemplate kafkaTemplate() { 32 | return new KafkaTemplate<>(producerFactory()); 33 | } 34 | 35 | @Bean 36 | ProducerFactory producerFactory() { 37 | Map props = kafkaProperties.buildProducerProperties(null); 38 | props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); 39 | props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class); 40 | props.put(AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, kafkaProperties.getProperties().get("schema-registry-url")); 41 | return new DefaultKafkaProducerFactory<>(props); 42 | } 43 | 44 | // As the application will create a topic in Kafka, it is advisable to update the BOOTSTRAP_SERVERS_CONFIG 45 | // property of KafkaAdmin with the configured bootstrap server URLs. Otherwise, it will default to 'localhost:9082'. 46 | @Bean 47 | KafkaAdmin kafkaAdmin() { 48 | Map configs = new HashMap<>(); 49 | configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBootstrapServers()); 50 | return new KafkaAdmin(configs); 51 | } 52 | 53 | @Bean 54 | NewTopic newTopic() { 55 | Map producerProperties = kafkaProperties.getProducer().getProperties(); 56 | return new NewTopic(producerProperties.get("topic"), Integer.parseInt(producerProperties.get("num-partitions")), (short) 1); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-producer-service/src/main/java/com/ivanfranchin/avro3producerservice/simulation/SimulationRunner.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro3producerservice.simulation; 2 | 3 | import com.ivanfranchin.avro3producerservice.avro.NewsMessage; 4 | import com.ivanfranchin.avro3producerservice.news.News; 5 | import com.ivanfranchin.avro3producerservice.news.NewsEmitter; 6 | import org.springframework.boot.CommandLineRunner; 7 | import org.springframework.stereotype.Component; 8 | 9 | import java.util.UUID; 10 | import java.util.concurrent.Executors; 11 | import java.util.concurrent.TimeUnit; 12 | 13 | @Component 14 | public class SimulationRunner implements CommandLineRunner { 15 | 16 | private final NewsEmitter newsEmitter; 17 | 18 | public SimulationRunner(NewsEmitter newsEmitter) { 19 | this.newsEmitter = newsEmitter; 20 | } 21 | 22 | @Override 23 | public void run(String... args) { 24 | Executors.newSingleThreadScheduledExecutor() 25 | .scheduleAtFixedRate(this::generateAndSendNews, 0, 3, TimeUnit.SECONDS); 26 | } 27 | 28 | private void generateAndSendNews() { 29 | newsEmitter.send(toNewsMessage(generate())); 30 | } 31 | 32 | private NewsMessage toNewsMessage(News news) { 33 | return new NewsMessage(news.id(), news.fromId(), news.fromName(), news.title()); 34 | } 35 | 36 | private News generate() { 37 | return new News(UUID.randomUUID().toString(), 1, "Channel", "Title"); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-producer-service/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=9088 2 | 3 | spring.application.name=avro-3-producer-service 4 | 5 | spring.kafka.bootstrap-servers=${KAFKA_HOST:localhost}:${KAFKA_PORT:29092} 6 | spring.kafka.properties.schema-registry-url=http://${SCHEMA_REGISTRY_HOST:localhost}:${SCHEMA_REGISTRY_PORT:8081} 7 | spring.kafka.producer.properties.topic=avro-3-de-serialization-news 8 | spring.kafka.producer.properties.num-partitions=4 9 | -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-producer-service/src/main/resources/avro/news-message.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "com.ivanfranchin.avro3producerservice.avro", 3 | "type": "record", 4 | "name": "NewsMessage", 5 | "fields": [ 6 | {"name": "id", "type": "string"}, 7 | {"name": "fromId", "type": "int"}, 8 | {"name": "fromName", "type": "string"}, 9 | {"name": "title", "type": "string"} 10 | ] 11 | } -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-producer-service/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _____ _ _ 2 | __ ___ ___ __ ___ |___ / _ __ _ __ ___ __| |_ _ ___ ___ _ __ ___ ___ _ ____ _(_) ___ ___ 3 | / _` \ \ / / '__/ _ \ _____ |_ \ _____| '_ \| '__/ _ \ / _` | | | |/ __/ _ \ '__|____/ __|/ _ \ '__\ \ / / |/ __/ _ \ 4 | | (_| |\ V /| | | (_) |_____|__) |_____| |_) | | | (_) | (_| | |_| | (_| __/ | |_____\__ \ __/ | \ V /| | (_| __/ 5 | \__,_| \_/ |_| \___/ |____/ | .__/|_| \___/ \__,_|\__,_|\___\___|_| |___/\___|_| \_/ |_|\___\___| 6 | |_| 7 | :: Spring Boot :: ${spring-boot.formatted-version} 8 | -------------------------------------------------------------------------------- /avro-3-de-serialization/avro-3-producer-service/src/test/java/com/ivanfranchin/avro3producerservice/Avro3ProducerServiceApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avro3producerservice; 2 | 3 | import org.junit.jupiter.api.Disabled; 4 | import org.junit.jupiter.api.Test; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | 7 | @Disabled 8 | @SpringBootTest 9 | class Avro3ProducerServiceApplicationTests { 10 | 11 | @Test 12 | void contextLoads() { 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /avro-3-de-serialization/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.ivanfranchin 7 | spring-kafka-de-serialization-types 8 | 1.0.0 9 | ../pom.xml 10 | 11 | avro-3-de-serialization 12 | pom 13 | avro-3-de-serialization 14 | Demo project for Spring Boot 15 | 16 | 1.12.0 17 | 7.8.0 18 | 19 | 20 | 21 | 22 | org.apache.avro 23 | avro 24 | ${avro.version} 25 | 26 | 27 | 28 | 29 | io.confluent 30 | kafka-avro-serializer 31 | ${confluent-kafka-avro-serializer.version} 32 | 33 | 34 | 35 | 36 | 37 | 38 | org.springframework.boot 39 | spring-boot-maven-plugin 40 | 41 | 42 | org.apache.avro 43 | avro-maven-plugin 44 | ${avro.version} 45 | 46 | 47 | generate-sources 48 | 49 | schema 50 | 51 | 52 | ${project.basedir}/src/main/resources/avro/ 53 | ${project.basedir}/src/main/java 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | confluent 64 | https://packages.confluent.io/maven/ 65 | 66 | 67 | 68 | 69 | avro-3-producer-service 70 | avro-3-consumer-service 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /avro-de-serialization/README.md: -------------------------------------------------------------------------------- 1 | # spring-kafka-de-serialization-types 2 | ## `> avro-de-serialization` 3 | 4 | ![avro-de-serialization](../documentation/avro-de-serialization.jpeg) 5 | 6 | This sample demonstrates a **producer** that pushes `News` messages to a topic in `Kafka` and a **consumer** that listens those messages from `Kafka`: 7 | - **Producer** serializes the message `key` using `StringSerializer` and the message `value` using `AvroSerializer`; 8 | - **Consumer** deserializes the message `key` using `StringDeserializer` and the message `value` using `AvroDeserializer`; 9 | - We needed to implement `AvroSerializer` and `AvroDeserializer` classes; 10 | - **Producer** creates the Kafka topics and **Consumer** doesn't. 11 | 12 | ## Start Environment 13 | 14 | Before starting producer and consumer, the services present in `docker-compose.yml` file must be up and running as explained in [Start Environment](https://github.com/ivangfr/spring-kafka-de-serialization-types#start-environment) section of the main README. 15 | 16 | ## Running applications using Maven 17 | 18 | > **Note**: run `avro-producer-service` first so that it can create the `Kafka` topics. 19 | 20 | - **avro-producer-service** 21 | 22 | - Open a terminal and navigate to the `spring-kafka-de-serialization-types` root folder; 23 | - Run application: 24 | ``` 25 | ./mvnw clean spring-boot:run --projects avro-de-serialization/avro-producer-service 26 | ``` 27 | > The Java class `com.ivanfranchin.avroproducerservice.avro.NewsMessage` is generated by the Avro file `news-message.avsc` present in `src/main/resources/avro` by running the command: 28 | > ``` 29 | > ./mvnw generate-sources --projects avro-de-serialization/avro-producer-service 30 | > ``` 31 | - As soon as the producer is up and running, it will start pushing automatically and randomly `News` messages to `Kafka` topic `avro-de-serialization-news`. The default `interval` between messages is `3 seconds`. 32 | 33 | - **avro-consumer-service** 34 | 35 | - Open another terminal and make sure you are in `spring-kafka-de-serialization-types` root folder; 36 | - Run application: 37 | ``` 38 | ./mvnw clean spring-boot:run --projects avro-de-serialization/avro-consumer-service 39 | ``` 40 | > The Java class `com.ivanfranchin.avroconsumerservice.avro.NewsMessage` is generated by the Avro file `news-message.avsc` present in `src/main/resources/avro` by running the command: 41 | > ``` 42 | > ./mvnw generate-sources --projects avro-de-serialization/avro-consumer-service 43 | > ``` 44 | - Once the consumer is up and running, it will start listening `News` messages from the `Kafka` topic `avro-de-serialization-news`. 45 | 46 | ## Running applications as Docker containers 47 | 48 | - ### Build Docker images 49 | 50 | In a terminal and, inside the `spring-kafka-de-serialization-types` root folder, run: 51 | ``` 52 | ./build-docker-images.sh avro-de-serialization 53 | ``` 54 | 55 | - ### Environment variables 56 | 57 | **avro-producer-service** and **avro-consumer-service** 58 | 59 | | Environment Variable | Description | 60 | |----------------------|-------------------------------------------------------------------------| 61 | | `KAFKA_HOST` | Specify host of the `Kafka` message broker to use (default `localhost`) | 62 | | `KAFKA_PORT` | Specify port of the `Kafka` message broker to use (default `29092`) | 63 | 64 | - ### Run Docker containers 65 | 66 | > **Note**: run `avro-producer-service` first so that it can create the `Kafka` topics. 67 | 68 | - **avro-producer-service** 69 | 70 | In a terminal, run the following Docker command: 71 | ``` 72 | docker run --rm --name avro-producer-service -p 9084:9084 \ 73 | -e KAFKA_HOST=kafka -e KAFKA_PORT=9092 \ 74 | --network=spring-kafka-de-serialization-types_default \ 75 | ivanfranchin/avro-producer-service:1.0.0 76 | ``` 77 | 78 | - **avro-consumer-service** 79 | 80 | In another terminal, run the Docker command below: 81 | ``` 82 | docker run --rm --name avro-consumer-service -p 9085:9085 \ 83 | -e KAFKA_HOST=kafka -e KAFKA_PORT=9092 \ 84 | --network=spring-kafka-de-serialization-types_default \ 85 | ivanfranchin/avro-consumer-service:1.0.0 86 | ``` 87 | 88 | ## Shutdown 89 | 90 | - Go to the terminals where the applications are running and press `Ctrl+C`; 91 | - Stop the services present in `docker-compose.yml` as explained in [Shutdown](https://github.com/ivangfr/spring-kafka-de-serialization-types#shutdown) section of the main README. 92 | 93 | ## Cleanup 94 | 95 | To remove the Docker images created by this example, go to a terminal and, inside the `spring-kafka-de-serialization-types` root folder, run the following script. 96 | ``` 97 | ./remove-docker-images.sh avro-de-serialization 98 | ``` 99 | -------------------------------------------------------------------------------- /avro-de-serialization/avro-consumer-service/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.ivanfranchin 7 | avro-de-serialization 8 | 1.0.0 9 | ../pom.xml 10 | 11 | avro-consumer-service 12 | avro-consumer-service 13 | Demo project for Spring Boot 14 | 15 | 16 | org.springframework.boot 17 | spring-boot-starter-web 18 | 19 | 20 | org.springframework.kafka 21 | spring-kafka 22 | 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-starter-test 27 | test 28 | 29 | 30 | org.springframework.kafka 31 | spring-kafka-test 32 | test 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /avro-de-serialization/avro-consumer-service/src/main/java/com/ivanfranchin/avroconsumerservice/AvroConsumerServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avroconsumerservice; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class AvroConsumerServiceApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(AvroConsumerServiceApplication.class, args); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /avro-de-serialization/avro-consumer-service/src/main/java/com/ivanfranchin/avroconsumerservice/news/AvroDeserializer.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avroconsumerservice.news; 2 | 3 | import jakarta.xml.bind.DatatypeConverter; 4 | import org.apache.avro.generic.GenericRecord; 5 | import org.apache.avro.io.DatumReader; 6 | import org.apache.avro.io.Decoder; 7 | import org.apache.avro.io.DecoderFactory; 8 | import org.apache.avro.specific.SpecificDatumReader; 9 | import org.apache.avro.specific.SpecificRecordBase; 10 | import org.apache.kafka.common.errors.SerializationException; 11 | import org.apache.kafka.common.serialization.Deserializer; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import java.util.Arrays; 16 | import java.util.Map; 17 | 18 | public class AvroDeserializer implements Deserializer { 19 | 20 | private static final Logger log = LoggerFactory.getLogger(AvroDeserializer.class); 21 | 22 | protected final Class targetType; 23 | 24 | public AvroDeserializer(Class targetType) { 25 | this.targetType = targetType; 26 | } 27 | 28 | @Override 29 | public void close() { 30 | } 31 | 32 | @Override 33 | public void configure(Map arg0, boolean arg1) { 34 | } 35 | 36 | @SuppressWarnings("unchecked") 37 | @Override 38 | public T deserialize(String topic, byte[] data) { 39 | try { 40 | T result = null; 41 | 42 | if (data != null) { 43 | log.debug("data='{}'", DatatypeConverter.printHexBinary(data)); 44 | 45 | DatumReader datumReader = 46 | new SpecificDatumReader<>(targetType.getDeclaredConstructor().newInstance().getSchema()); 47 | Decoder decoder = DecoderFactory.get().binaryDecoder(data, null); 48 | 49 | result = (T) datumReader.read(null, decoder); 50 | log.debug("deserialized data='{}'", result); 51 | } 52 | return result; 53 | } catch (Exception ex) { 54 | throw new SerializationException("Can't deserialize data '" + Arrays.toString(data) + "' from topic '" + topic + "'", ex); 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /avro-de-serialization/avro-consumer-service/src/main/java/com/ivanfranchin/avroconsumerservice/news/NewsListener.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avroconsumerservice.news; 2 | 3 | import com.ivanfranchin.avroconsumerservice.avro.NewsMessage; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.kafka.annotation.KafkaListener; 7 | import org.springframework.kafka.listener.adapter.ConsumerRecordMetadata; 8 | import org.springframework.messaging.handler.annotation.Payload; 9 | import org.springframework.stereotype.Service; 10 | 11 | @Service 12 | public class NewsListener { 13 | 14 | private static final Logger log = LoggerFactory.getLogger(NewsListener.class); 15 | 16 | @KafkaListener(topics = "${spring.kafka.consumer.topic}", groupId = "${spring.kafka.consumer.group-id}") 17 | public void listen(@Payload NewsMessage newsMessage, ConsumerRecordMetadata metadata) { 18 | log.info("Received message\n---\nTOPIC: {}; PARTITION: {}; OFFSET: {};\nPAYLOAD: {}\n---", 19 | metadata.topic(), metadata.partition(), metadata.offset(), newsMessage); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /avro-de-serialization/avro-consumer-service/src/main/java/com/ivanfranchin/avroconsumerservice/news/NewsListenerConfig.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avroconsumerservice.news; 2 | 3 | import com.ivanfranchin.avroconsumerservice.avro.NewsMessage; 4 | import org.apache.kafka.clients.consumer.ConsumerConfig; 5 | import org.apache.kafka.common.serialization.StringDeserializer; 6 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | import org.springframework.kafka.annotation.EnableKafka; 10 | import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; 11 | import org.springframework.kafka.core.ConsumerFactory; 12 | import org.springframework.kafka.core.DefaultKafkaConsumerFactory; 13 | 14 | import java.util.Map; 15 | 16 | @EnableKafka 17 | @Configuration 18 | public class NewsListenerConfig { 19 | 20 | private final KafkaProperties kafkaProperties; 21 | 22 | public NewsListenerConfig(KafkaProperties kafkaProperties) { 23 | this.kafkaProperties = kafkaProperties; 24 | } 25 | 26 | @Bean 27 | ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory() { 28 | ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); 29 | factory.setConsumerFactory(consumerFactory()); 30 | factory.setConcurrency(kafkaProperties.getListener().getConcurrency()); 31 | return factory; 32 | } 33 | 34 | @Bean 35 | ConsumerFactory consumerFactory() { 36 | Map props = kafkaProperties.buildConsumerProperties(null); 37 | props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); 38 | props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, AvroDeserializer.class); 39 | props.put(ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, false); 40 | return new DefaultKafkaConsumerFactory<>(props, new StringDeserializer(), new AvroDeserializer<>(NewsMessage.class)); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /avro-de-serialization/avro-consumer-service/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=9085 2 | 3 | spring.application.name=avro-consumer-service 4 | 5 | spring.kafka.bootstrap-servers=${KAFKA_HOST:localhost}:${KAFKA_PORT:29092} 6 | spring.kafka.consumer.group-id=avro-consumer-service-group 7 | spring.kafka.consumer.topic=avro-de-serialization-news 8 | spring.kafka.consumer.auto-offset-reset=earliest 9 | spring.kafka.listener.concurrency=2 -------------------------------------------------------------------------------- /avro-de-serialization/avro-consumer-service/src/main/resources/avro/news-message.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "com.ivanfranchin.avroconsumerservice.avro", 3 | "type": "record", 4 | "name": "NewsMessage", 5 | "fields": [ 6 | {"name": "id", "type": "string"}, 7 | {"name": "fromId", "type": "int"}, 8 | {"name": "fromName", "type": "string"}, 9 | {"name": "title", "type": "string"} 10 | ] 11 | } -------------------------------------------------------------------------------- /avro-de-serialization/avro-consumer-service/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _ 2 | __ ___ ___ __ ___ ___ ___ _ __ ___ _ _ _ __ ___ ___ _ __ ___ ___ _ ____ _(_) ___ ___ 3 | / _` \ \ / / '__/ _ \ _____ / __/ _ \| '_ \/ __| | | | '_ ` _ \ / _ \ '__|____/ __|/ _ \ '__\ \ / / |/ __/ _ \ 4 | | (_| |\ V /| | | (_) |_____| (_| (_) | | | \__ \ |_| | | | | | | __/ | |_____\__ \ __/ | \ V /| | (_| __/ 5 | \__,_| \_/ |_| \___/ \___\___/|_| |_|___/\__,_|_| |_| |_|\___|_| |___/\___|_| \_/ |_|\___\___| 6 | :: Spring Boot :: ${spring-boot.formatted-version} 7 | -------------------------------------------------------------------------------- /avro-de-serialization/avro-consumer-service/src/test/java/com/ivanfranchin/avroconsumerservice/AvroConsumerServiceApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avroconsumerservice; 2 | 3 | import org.junit.jupiter.api.Disabled; 4 | import org.junit.jupiter.api.Test; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | 7 | @Disabled 8 | @SpringBootTest 9 | class AvroConsumerServiceApplicationTests { 10 | 11 | @Test 12 | void contextLoads() { 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /avro-de-serialization/avro-producer-service/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.ivanfranchin 7 | avro-de-serialization 8 | 1.0.0 9 | ../pom.xml 10 | 11 | avro-producer-service 12 | avro-producer-service 13 | Demo project for Spring Boot 14 | 15 | 16 | org.springframework.boot 17 | spring-boot-starter-web 18 | 19 | 20 | org.springframework.kafka 21 | spring-kafka 22 | 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-starter-test 27 | test 28 | 29 | 30 | org.springframework.kafka 31 | spring-kafka-test 32 | test 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /avro-de-serialization/avro-producer-service/src/main/java/com/ivanfranchin/avroproducerservice/AvroProducerServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avroproducerservice; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class AvroProducerServiceApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(AvroProducerServiceApplication.class, args); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /avro-de-serialization/avro-producer-service/src/main/java/com/ivanfranchin/avroproducerservice/news/AvroSerializer.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avroproducerservice.news; 2 | 3 | import jakarta.xml.bind.DatatypeConverter; 4 | import org.apache.avro.generic.GenericDatumWriter; 5 | import org.apache.avro.generic.GenericRecord; 6 | import org.apache.avro.io.BinaryEncoder; 7 | import org.apache.avro.io.DatumWriter; 8 | import org.apache.avro.io.EncoderFactory; 9 | import org.apache.avro.specific.SpecificRecordBase; 10 | import org.apache.kafka.common.errors.SerializationException; 11 | import org.apache.kafka.common.serialization.Serializer; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import java.io.ByteArrayOutputStream; 16 | import java.io.IOException; 17 | import java.util.Map; 18 | 19 | public class AvroSerializer implements Serializer { 20 | 21 | private static final Logger log = LoggerFactory.getLogger(AvroSerializer.class); 22 | 23 | @Override 24 | public void close() { 25 | } 26 | 27 | @Override 28 | public void configure(Map arg0, boolean arg1) { 29 | } 30 | 31 | @Override 32 | public byte[] serialize(String topic, T data) { 33 | try { 34 | byte[] result = null; 35 | 36 | if (data != null) { 37 | log.debug("data='{}'", data); 38 | 39 | ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); 40 | BinaryEncoder binaryEncoder = 41 | EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null); 42 | 43 | DatumWriter datumWriter = new GenericDatumWriter<>(data.getSchema()); 44 | datumWriter.write(data, binaryEncoder); 45 | 46 | binaryEncoder.flush(); 47 | byteArrayOutputStream.close(); 48 | 49 | result = byteArrayOutputStream.toByteArray(); 50 | log.debug("serialized data='{}'", DatatypeConverter.printHexBinary(result)); 51 | } 52 | return result; 53 | } catch (IOException ex) { 54 | throw new SerializationException("Can't serialize data='" + data + "' for topic='" + topic + "'", ex); 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /avro-de-serialization/avro-producer-service/src/main/java/com/ivanfranchin/avroproducerservice/news/News.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avroproducerservice.news; 2 | 3 | public record News(String id, Integer fromId, String fromName, String title) { 4 | } 5 | -------------------------------------------------------------------------------- /avro-de-serialization/avro-producer-service/src/main/java/com/ivanfranchin/avroproducerservice/news/NewsEmitter.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avroproducerservice.news; 2 | 3 | import com.ivanfranchin.avroproducerservice.avro.NewsMessage; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 7 | import org.springframework.kafka.core.KafkaTemplate; 8 | import org.springframework.stereotype.Component; 9 | 10 | @Component 11 | public class NewsEmitter { 12 | 13 | private static final Logger log = LoggerFactory.getLogger(NewsEmitter.class); 14 | 15 | private final KafkaProperties kafkaProperties; 16 | private final KafkaTemplate kafkaTemplate; 17 | 18 | public NewsEmitter(KafkaProperties kafkaProperties, KafkaTemplate kafkaTemplate) { 19 | this.kafkaProperties = kafkaProperties; 20 | this.kafkaTemplate = kafkaTemplate; 21 | } 22 | 23 | public void send(NewsMessage newsMessage) { 24 | String kafkaTopic = kafkaProperties.getProducer().getProperties().get("topic"); 25 | log.info("Sending News '{}' to topic '{}'", newsMessage, kafkaTopic); 26 | kafkaTemplate.send(kafkaTopic, String.valueOf(newsMessage.getFromId()), newsMessage); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /avro-de-serialization/avro-producer-service/src/main/java/com/ivanfranchin/avroproducerservice/news/NewsEmitterConfig.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avroproducerservice.news; 2 | 3 | import com.ivanfranchin.avroproducerservice.avro.NewsMessage; 4 | import org.apache.kafka.clients.admin.AdminClientConfig; 5 | import org.apache.kafka.clients.admin.NewTopic; 6 | import org.apache.kafka.clients.producer.ProducerConfig; 7 | import org.apache.kafka.common.serialization.StringSerializer; 8 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 9 | import org.springframework.context.annotation.Bean; 10 | import org.springframework.context.annotation.Configuration; 11 | import org.springframework.kafka.core.DefaultKafkaProducerFactory; 12 | import org.springframework.kafka.core.KafkaAdmin; 13 | import org.springframework.kafka.core.KafkaTemplate; 14 | import org.springframework.kafka.core.ProducerFactory; 15 | 16 | import java.util.HashMap; 17 | import java.util.Map; 18 | 19 | @Configuration 20 | public class NewsEmitterConfig { 21 | 22 | private final KafkaProperties kafkaProperties; 23 | 24 | public NewsEmitterConfig(KafkaProperties kafkaProperties) { 25 | this.kafkaProperties = kafkaProperties; 26 | } 27 | 28 | @Bean 29 | KafkaTemplate kafkaTemplate() { 30 | return new KafkaTemplate<>(producerFactory()); 31 | } 32 | 33 | @Bean 34 | ProducerFactory producerFactory() { 35 | Map props = kafkaProperties.buildProducerProperties(null); 36 | props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); 37 | props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, AvroSerializer.class); 38 | return new DefaultKafkaProducerFactory<>(props); 39 | } 40 | 41 | // As the application will create a topic in Kafka, it is advisable to update the BOOTSTRAP_SERVERS_CONFIG 42 | // property of KafkaAdmin with the configured bootstrap server URLs. Otherwise, it will default to 'localhost:9082'. 43 | @Bean 44 | KafkaAdmin kafkaAdmin() { 45 | Map configs = new HashMap<>(); 46 | configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBootstrapServers()); 47 | return new KafkaAdmin(configs); 48 | } 49 | 50 | @Bean 51 | NewTopic newTopic() { 52 | Map producerProperties = kafkaProperties.getProducer().getProperties(); 53 | return new NewTopic(producerProperties.get("topic"), Integer.parseInt(producerProperties.get("num-partitions")), (short) 1); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /avro-de-serialization/avro-producer-service/src/main/java/com/ivanfranchin/avroproducerservice/simulation/SimulationRunner.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avroproducerservice.simulation; 2 | 3 | import com.ivanfranchin.avroproducerservice.avro.NewsMessage; 4 | import com.ivanfranchin.avroproducerservice.news.News; 5 | import com.ivanfranchin.avroproducerservice.news.NewsEmitter; 6 | import org.springframework.boot.CommandLineRunner; 7 | import org.springframework.stereotype.Component; 8 | 9 | import java.util.UUID; 10 | import java.util.concurrent.Executors; 11 | import java.util.concurrent.TimeUnit; 12 | 13 | @Component 14 | public class SimulationRunner implements CommandLineRunner { 15 | 16 | private final NewsEmitter newsEmitter; 17 | 18 | public SimulationRunner(NewsEmitter newsEmitter) { 19 | this.newsEmitter = newsEmitter; 20 | } 21 | 22 | @Override 23 | public void run(String... args) { 24 | Executors.newSingleThreadScheduledExecutor() 25 | .scheduleAtFixedRate(this::generateAndSendNews, 0, 3, TimeUnit.SECONDS); 26 | } 27 | 28 | private void generateAndSendNews() { 29 | newsEmitter.send(toNewsMessage(generate())); 30 | } 31 | 32 | private NewsMessage toNewsMessage(News news) { 33 | return new NewsMessage(news.id(), news.fromId(), news.fromName(), news.title()); 34 | } 35 | 36 | private News generate() { 37 | return new News(UUID.randomUUID().toString(), 1, "Channel", "Title"); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /avro-de-serialization/avro-producer-service/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=9084 2 | 3 | spring.application.name=avro-producer-service 4 | 5 | spring.kafka.bootstrap-servers=${KAFKA_HOST:localhost}:${KAFKA_PORT:29092} 6 | spring.kafka.producer.properties.topic=avro-de-serialization-news 7 | spring.kafka.producer.properties.num-partitions=4 8 | -------------------------------------------------------------------------------- /avro-de-serialization/avro-producer-service/src/main/resources/avro/news-message.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "com.ivanfranchin.avroproducerservice.avro", 3 | "type": "record", 4 | "name": "NewsMessage", 5 | "fields": [ 6 | {"name": "id", "type": "string"}, 7 | {"name": "fromId", "type": "int"}, 8 | {"name": "fromName", "type": "string"}, 9 | {"name": "title", "type": "string"} 10 | ] 11 | } -------------------------------------------------------------------------------- /avro-de-serialization/avro-producer-service/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _ _ 2 | __ ___ ___ __ ___ _ __ _ __ ___ __| |_ _ ___ ___ _ __ ___ ___ _ ____ _(_) ___ ___ 3 | / _` \ \ / / '__/ _ \ _____| '_ \| '__/ _ \ / _` | | | |/ __/ _ \ '__|____/ __|/ _ \ '__\ \ / / |/ __/ _ \ 4 | | (_| |\ V /| | | (_) |_____| |_) | | | (_) | (_| | |_| | (_| __/ | |_____\__ \ __/ | \ V /| | (_| __/ 5 | \__,_| \_/ |_| \___/ | .__/|_| \___/ \__,_|\__,_|\___\___|_| |___/\___|_| \_/ |_|\___\___| 6 | |_| 7 | :: Spring Boot :: ${spring-boot.formatted-version} 8 | -------------------------------------------------------------------------------- /avro-de-serialization/avro-producer-service/src/test/java/com/ivanfranchin/avroproducerservice/AvroProducerServiceApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.avroproducerservice; 2 | 3 | import org.junit.jupiter.api.Disabled; 4 | import org.junit.jupiter.api.Test; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | 7 | @Disabled 8 | @SpringBootTest 9 | class AvroProducerServiceApplicationTests { 10 | 11 | @Test 12 | void contextLoads() { 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /avro-de-serialization/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.ivanfranchin 7 | spring-kafka-de-serialization-types 8 | 1.0.0 9 | ../pom.xml 10 | 11 | avro-de-serialization 12 | pom 13 | avro-de-serialization 14 | Demo project for Spring Boot 15 | 16 | 1.12.0 17 | 4.0.0 18 | 19 | 20 | 21 | 22 | org.apache.avro 23 | avro 24 | ${avro.version} 25 | 26 | 27 | 28 | 29 | jakarta.xml.bind 30 | jakarta.xml.bind-api 31 | ${jakarta-xml-bind-api.version} 32 | 33 | 34 | 35 | 36 | 37 | 38 | org.springframework.boot 39 | spring-boot-maven-plugin 40 | 41 | 42 | org.apache.avro 43 | avro-maven-plugin 44 | ${avro.version} 45 | 46 | 47 | generate-sources 48 | 49 | schema 50 | 51 | 52 | ${project.basedir}/src/main/resources/avro/ 53 | ${project.basedir}/src/main/java 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | avro-producer-service 63 | avro-consumer-service 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /build-docker-images.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source my-functions.sh 4 | 5 | check_script_input_parameter $1 6 | 7 | DOCKER_IMAGE_PREFIX="ivanfranchin" 8 | APP_VERSION="1.0.0" 9 | 10 | STRING_PRODUCER_SERVICE_APP_NAME="string-producer-service" 11 | STRING_CONSUMER_SERVICE_APP_NAME="string-consumer-service" 12 | JSON_PRODUCER_SERVICE_APP_NAME="json-producer-service" 13 | JSON_CONSUMER_SERVICE_APP_NAME="json-consumer-service" 14 | AVRO_PRODUCER_SERVICE_APP_NAME="avro-producer-service" 15 | AVRO_CONSUMER_SERVICE_APP_NAME="avro-consumer-service" 16 | AVRO_2_PRODUCER_SERVICE_APP_NAME="avro-2-producer-service" 17 | AVRO_2_CONSUMER_SERVICE_APP_NAME="avro-2-consumer-service" 18 | AVRO_3_PRODUCER_SERVICE_APP_NAME="avro-3-producer-service" 19 | AVRO_3_CONSUMER_SERVICE_APP_NAME="avro-3-consumer-service" 20 | 21 | STRING_PRODUCER_SERVICE_PROJECT_NAME="string-de-serialization/${STRING_PRODUCER_SERVICE_APP_NAME}" 22 | STRING_CONSUMER_SERVICE_PROJECT_NAME="string-de-serialization/${STRING_CONSUMER_SERVICE_APP_NAME}" 23 | JSON_PRODUCER_SERVICE_PROJECT_NAME="json-de-serialization/${JSON_PRODUCER_SERVICE_APP_NAME}" 24 | JSON_CONSUMER_SERVICE_PROJECT_NAME="json-de-serialization/${JSON_CONSUMER_SERVICE_APP_NAME}" 25 | AVRO_PRODUCER_SERVICE_PROJECT_NAME="avro-de-serialization/${AVRO_PRODUCER_SERVICE_APP_NAME}" 26 | AVRO_CONSUMER_SERVICE_PROJECT_NAME="avro-de-serialization/${AVRO_CONSUMER_SERVICE_APP_NAME}" 27 | AVRO_2_PRODUCER_SERVICE_PROJECT_NAME="avro-2-de-serialization/${AVRO_2_PRODUCER_SERVICE_APP_NAME}" 28 | AVRO_2_CONSUMER_SERVICE_PROJECT_NAME="avro-2-de-serialization/${AVRO_2_CONSUMER_SERVICE_APP_NAME}" 29 | AVRO_3_PRODUCER_SERVICE_PROJECT_NAME="avro-3-de-serialization/${AVRO_3_PRODUCER_SERVICE_APP_NAME}" 30 | AVRO_3_CONSUMER_SERVICE_PROJECT_NAME="avro-3-de-serialization/${AVRO_3_CONSUMER_SERVICE_APP_NAME}" 31 | 32 | STRING_PRODUCER_SERVICE_DOCKER_IMAGE_NAME="${DOCKER_IMAGE_PREFIX}/${STRING_PRODUCER_SERVICE_APP_NAME}:${APP_VERSION}" 33 | STRING_CONSUMER_SERVICE_DOCKER_IMAGE_NAME="${DOCKER_IMAGE_PREFIX}/${STRING_CONSUMER_SERVICE_APP_NAME}:${APP_VERSION}" 34 | JSON_PRODUCER_SERVICE_DOCKER_IMAGE_NAME="${DOCKER_IMAGE_PREFIX}/${JSON_PRODUCER_SERVICE_APP_NAME}:${APP_VERSION}" 35 | JSON_CONSUMER_SERVICE_DOCKER_IMAGE_NAME="${DOCKER_IMAGE_PREFIX}/${JSON_CONSUMER_SERVICE_APP_NAME}:${APP_VERSION}" 36 | AVRO_PRODUCER_SERVICE_DOCKER_IMAGE_NAME="${DOCKER_IMAGE_PREFIX}/${AVRO_PRODUCER_SERVICE_APP_NAME}:${APP_VERSION}" 37 | AVRO_CONSUMER_SERVICE_DOCKER_IMAGE_NAME="${DOCKER_IMAGE_PREFIX}/${AVRO_CONSUMER_SERVICE_APP_NAME}:${APP_VERSION}" 38 | AVRO_2_PRODUCER_SERVICE_DOCKER_IMAGE_NAME="${DOCKER_IMAGE_PREFIX}/${AVRO_2_PRODUCER_SERVICE_APP_NAME}:${APP_VERSION}" 39 | AVRO_2_CONSUMER_SERVICE_DOCKER_IMAGE_NAME="${DOCKER_IMAGE_PREFIX}/${AVRO_2_CONSUMER_SERVICE_APP_NAME}:${APP_VERSION}" 40 | AVRO_3_PRODUCER_SERVICE_DOCKER_IMAGE_NAME="${DOCKER_IMAGE_PREFIX}/${AVRO_3_PRODUCER_SERVICE_APP_NAME}:${APP_VERSION}" 41 | AVRO_3_CONSUMER_SERVICE_DOCKER_IMAGE_NAME="${DOCKER_IMAGE_PREFIX}/${AVRO_3_CONSUMER_SERVICE_APP_NAME}:${APP_VERSION}" 42 | 43 | SKIP_TESTS="true" 44 | 45 | if [ "$1" = "string-producer-service" ] || 46 | [ "$1" = "string-de-serialization" ] || 47 | [ "$1" = "all" ]; 48 | then 49 | 50 | echo 51 | echo "-----------------------" 52 | echo "STRING-PRODUCER-SERVICE" 53 | echo "-----------------------" 54 | 55 | ./mvnw clean compile jib:dockerBuild \ 56 | --projects "$STRING_PRODUCER_SERVICE_PROJECT_NAME" \ 57 | -DskipTests="$SKIP_TESTS" \ 58 | -Dimage="$STRING_PRODUCER_SERVICE_DOCKER_IMAGE_NAME" 59 | 60 | fi 61 | 62 | if [ "$1" = "string-consumer-service" ] || 63 | [ "$1" = "string-de-serialization" ] || 64 | [ "$1" = "all" ]; 65 | then 66 | 67 | echo 68 | echo "-----------------------" 69 | echo "STRING-CONSUMER-SERVICE" 70 | echo "-----------------------" 71 | 72 | ./mvnw clean compile jib:dockerBuild \ 73 | --projects "$STRING_CONSUMER_SERVICE_PROJECT_NAME" \ 74 | -DskipTests="$SKIP_TESTS" \ 75 | -Dimage="$STRING_CONSUMER_SERVICE_DOCKER_IMAGE_NAME" 76 | 77 | fi 78 | 79 | if [ "$1" = "json-producer-service" ] || 80 | [ "$1" = "json-de-serialization" ] || 81 | [ "$1" = "all" ]; 82 | then 83 | 84 | echo 85 | echo "---------------------" 86 | echo "JSON-PRODUCER-SERVICE" 87 | echo "---------------------" 88 | 89 | ./mvnw clean compile jib:dockerBuild \ 90 | --projects "$JSON_PRODUCER_SERVICE_PROJECT_NAME" \ 91 | -DskipTests="$SKIP_TESTS" \ 92 | -Dimage="$JSON_PRODUCER_SERVICE_DOCKER_IMAGE_NAME" 93 | 94 | fi 95 | 96 | if [ "$1" = "json-consumer-service" ] || 97 | [ "$1" = "json-de-serialization" ] || 98 | [ "$1" = "all" ]; 99 | then 100 | 101 | echo 102 | echo "---------------------" 103 | echo "JSON-CONSUMER-SERVICE" 104 | echo "---------------------" 105 | 106 | ./mvnw clean compile jib:dockerBuild \ 107 | --projects "$JSON_CONSUMER_SERVICE_PROJECT_NAME" \ 108 | -DskipTests="$SKIP_TESTS" \ 109 | -Dimage="$JSON_CONSUMER_SERVICE_DOCKER_IMAGE_NAME" 110 | 111 | fi 112 | 113 | if [ "$1" = "avro-producer-service" ] || 114 | [ "$1" = "avro-de-serialization" ] || 115 | [ "$1" = "all" ]; 116 | then 117 | 118 | echo 119 | echo "---------------------" 120 | echo "AVRO-PRODUCER-SERVICE" 121 | echo "---------------------" 122 | 123 | ./mvnw clean compile jib:dockerBuild \ 124 | --projects "$AVRO_PRODUCER_SERVICE_PROJECT_NAME" \ 125 | -DskipTests="$SKIP_TESTS" \ 126 | -Dimage="$AVRO_PRODUCER_SERVICE_DOCKER_IMAGE_NAME" 127 | 128 | fi 129 | 130 | if [ "$1" = "avro-consumer-service" ] || 131 | [ "$1" = "avro-de-serialization" ] || 132 | [ "$1" = "all" ]; 133 | then 134 | 135 | echo 136 | echo "---------------------" 137 | echo "AVRO-CONSUMER-SERVICE" 138 | echo "---------------------" 139 | 140 | ./mvnw clean compile jib:dockerBuild \ 141 | --projects "$AVRO_CONSUMER_SERVICE_PROJECT_NAME" \ 142 | -DskipTests="$SKIP_TESTS" \ 143 | -Dimage="$AVRO_CONSUMER_SERVICE_DOCKER_IMAGE_NAME" 144 | 145 | fi 146 | 147 | if [ "$1" = "avro-2-producer-service" ] || 148 | [ "$1" = "avro-2-de-serialization" ] || 149 | [ "$1" = "all" ]; 150 | then 151 | 152 | echo 153 | echo "-----------------------" 154 | echo "AVRO-2-PRODUCER-SERVICE" 155 | echo "-----------------------" 156 | 157 | ./mvnw clean compile jib:dockerBuild \ 158 | --projects "$AVRO_2_PRODUCER_SERVICE_PROJECT_NAME" \ 159 | -DskipTests="$SKIP_TESTS" \ 160 | -Dimage="$AVRO_2_PRODUCER_SERVICE_DOCKER_IMAGE_NAME" 161 | 162 | fi 163 | 164 | if [ "$1" = "avro-2-consumer-service" ] || 165 | [ "$1" = "avro-2-de-serialization" ] || 166 | [ "$1" = "all" ]; 167 | then 168 | 169 | echo 170 | echo "-----------------------" 171 | echo "AVRO-2-CONSUMER-SERVICE" 172 | echo "-----------------------" 173 | 174 | ./mvnw clean compile jib:dockerBuild \ 175 | --projects "$AVRO_2_CONSUMER_SERVICE_PROJECT_NAME" \ 176 | -DskipTests="$SKIP_TESTS" \ 177 | -Dimage="$AVRO_2_CONSUMER_SERVICE_DOCKER_IMAGE_NAME" 178 | 179 | fi 180 | 181 | if [ "$1" = "avro-3-producer-service" ] || 182 | [ "$1" = "avro-3-de-serialization" ] || 183 | [ "$1" = "all" ]; 184 | then 185 | 186 | echo 187 | echo "-----------------------" 188 | echo "AVRO-3-PRODUCER-SERVICE" 189 | echo "-----------------------" 190 | 191 | ./mvnw clean compile jib:dockerBuild \ 192 | --projects "$AVRO_3_PRODUCER_SERVICE_PROJECT_NAME" \ 193 | -DskipTests="$SKIP_TESTS" \ 194 | -Dimage="$AVRO_3_PRODUCER_SERVICE_DOCKER_IMAGE_NAME" 195 | 196 | fi 197 | 198 | if [ "$1" = "avro-3-consumer-service" ] || 199 | [ "$1" = "avro-3-de-serialization" ] || 200 | [ "$1" = "all" ]; 201 | then 202 | 203 | echo 204 | echo "-----------------------" 205 | echo "AVRO-3-CONSUMER-SERVICE" 206 | echo "-----------------------" 207 | 208 | ./mvnw clean compile jib:dockerBuild \ 209 | --projects "$AVRO_3_CONSUMER_SERVICE_PROJECT_NAME" \ 210 | -DskipTests="$SKIP_TESTS" \ 211 | -Dimage="$AVRO_3_CONSUMER_SERVICE_DOCKER_IMAGE_NAME" 212 | 213 | fi -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | 3 | zookeeper: 4 | image: 'confluentinc/cp-zookeeper:7.8.0' 5 | container_name: 'zookeeper' 6 | restart: 'unless-stopped' 7 | ports: 8 | - '2181:2181' 9 | environment: 10 | - 'ZOOKEEPER_CLIENT_PORT=2181' 11 | healthcheck: 12 | test: 'echo stat | nc localhost $$ZOOKEEPER_CLIENT_PORT' 13 | 14 | kafka: 15 | image: 'confluentinc/cp-kafka:7.8.0' 16 | container_name: 'kafka' 17 | restart: 'unless-stopped' 18 | depends_on: 19 | - 'zookeeper' 20 | ports: 21 | - '9092:9092' 22 | - '29092:29092' 23 | environment: 24 | - 'KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181' 25 | - 'KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' 26 | - 'KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092' 27 | - 'KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1' 28 | healthcheck: 29 | test: [ "CMD", "nc", "-z", "localhost", "9092" ] 30 | 31 | schema-registry: 32 | image: 'confluentinc/cp-schema-registry:7.8.0' 33 | container_name: 'schema-registry' 34 | restart: 'unless-stopped' 35 | depends_on: 36 | - 'kafka' 37 | ports: 38 | - '8081:8081' 39 | environment: 40 | - 'SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=kafka:9092' 41 | - 'SCHEMA_REGISTRY_HOST_NAME=schema-registry' 42 | - 'SCHEMA_REGISTRY_LISTENERS=http://0.0.0.0:8081' 43 | healthcheck: 44 | test: 'curl -f http://localhost:8081 || exit 1' 45 | 46 | schema-registry-ui: 47 | image: 'landoop/schema-registry-ui:0.9.5' 48 | container_name: 'kafka-schema-registry-ui' 49 | restart: 'unless-stopped' 50 | depends_on: 51 | - 'schema-registry' 52 | ports: 53 | - '8001:8000' 54 | environment: 55 | - 'SCHEMAREGISTRY_URL=http://schema-registry:8081' 56 | - 'PROXY=true' 57 | healthcheck: 58 | test: 'wget --quiet --tries=1 --spider http://localhost:8000 || exit 1' 59 | 60 | kafka-rest-proxy: 61 | image: 'confluentinc/cp-kafka-rest:7.8.0' 62 | container_name: 'kafka-rest-proxy' 63 | restart: 'unless-stopped' 64 | depends_on: 65 | - 'zookeeper' 66 | - 'kafka' 67 | ports: 68 | - '8082:8082' 69 | environment: 70 | - 'KAFKA_REST_BOOTSTRAP_SERVERS=PLAINTEXT://kafka:9092' 71 | - 'KAFKA_REST_ZOOKEEPER_CONNECT=zookeeper:2181' 72 | - 'KAFKA_REST_HOST_NAME=kafka-rest-proxy' 73 | - 'KAFKA_REST_LISTENERS=http://0.0.0.0:8082' 74 | - 'KAFKA_REST_SCHEMA_REGISTRY_URL=http://schema-registry:8081' 75 | - 'KAFKA_REST_CONSUMER_REQUEST_TIMEOUT_MS=30000' 76 | healthcheck: 77 | test: 'curl -f http://localhost:8082 || exit 1' 78 | 79 | kafka-topics-ui: 80 | image: 'landoop/kafka-topics-ui:0.9.4' 81 | container_name: 'kafka-topics-ui' 82 | restart: 'unless-stopped' 83 | depends_on: 84 | - 'kafka-rest-proxy' 85 | ports: 86 | - '8085:8000' 87 | environment: 88 | - 'KAFKA_REST_PROXY_URL=http://kafka-rest-proxy:8082' 89 | - 'PROXY=true' 90 | healthcheck: 91 | test: 'wget --quiet --tries=1 --spider http://localhost:8000 || exit 1' 92 | 93 | kafka-manager: 94 | container_name: 'kafka-manager' 95 | image: 'hlebalbau/kafka-manager:3.0.0.5' 96 | restart: 'unless-stopped' 97 | depends_on: 98 | - 'zookeeper' 99 | ports: 100 | - '9000:9000' 101 | environment: 102 | - 'ZK_HOSTS=zookeeper:2181' 103 | - 'APPLICATION_SECRET=random-secret' 104 | command: '-Dpidfile.path=/dev/null' 105 | healthcheck: 106 | test: 'curl -f http://localhost:9000 || exit 1' 107 | -------------------------------------------------------------------------------- /documentation/avro-2-de-serialization.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ivangfr/spring-kafka-de-serialization-types/83148510f1052698785d2704eafbfb115f20bdf2/documentation/avro-2-de-serialization.jpeg -------------------------------------------------------------------------------- /documentation/avro-3-de-serialization.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ivangfr/spring-kafka-de-serialization-types/83148510f1052698785d2704eafbfb115f20bdf2/documentation/avro-3-de-serialization.jpeg -------------------------------------------------------------------------------- /documentation/avro-de-serialization.excalidraw: -------------------------------------------------------------------------------- 1 | { 2 | "type": "excalidraw", 3 | "version": 2, 4 | "source": "https://excalidraw.com", 5 | "elements": [ 6 | { 7 | "type": "rectangle", 8 | "version": 1992, 9 | "versionNonce": 1545948604, 10 | "isDeleted": false, 11 | "id": "NKmNZxYxWMCKh3prRiPwX", 12 | "fillStyle": "hachure", 13 | "strokeWidth": 1, 14 | "strokeStyle": "solid", 15 | "roughness": 1, 16 | "opacity": 100, 17 | "angle": 0, 18 | "x": 76.26580861775972, 19 | "y": -295.7844894716043, 20 | "strokeColor": "#000000", 21 | "backgroundColor": "#82c91e", 22 | "width": 246, 23 | "height": 100, 24 | "seed": 236588676, 25 | "groupIds": [], 26 | "roundness": { 27 | "type": 3 28 | }, 29 | "boundElements": [ 30 | { 31 | "type": "text", 32 | "id": "GrVT2PZuYu1cRv3sXwFoI" 33 | }, 34 | { 35 | "id": "cA_hWJzSnGig-8CoXOII4", 36 | "type": "arrow" 37 | } 38 | ], 39 | "updated": 1678879817961, 40 | "link": null, 41 | "locked": false 42 | }, 43 | { 44 | "type": "text", 45 | "version": 938, 46 | "versionNonce": 1575779332, 47 | "isDeleted": false, 48 | "id": "GrVT2PZuYu1cRv3sXwFoI", 49 | "fillStyle": "hachure", 50 | "strokeWidth": 1, 51 | "strokeStyle": "solid", 52 | "roughness": 0, 53 | "opacity": 100, 54 | "angle": 0, 55 | "x": 105.77384999959565, 56 | "y": -279.3844894716043, 57 | "strokeColor": "#000000", 58 | "backgroundColor": "transparent", 59 | "width": 186.98391723632812, 60 | "height": 67.2, 61 | "seed": 294979727, 62 | "groupIds": [], 63 | "roundness": null, 64 | "boundElements": [], 65 | "updated": 1678879817961, 66 | "link": null, 67 | "locked": false, 68 | "fontSize": 28, 69 | "fontFamily": 1, 70 | "text": "avro-producer\n-service", 71 | "textAlign": "center", 72 | "verticalAlign": "middle", 73 | "containerId": "NKmNZxYxWMCKh3prRiPwX", 74 | "originalText": "avro-producer\n-service" 75 | }, 76 | { 77 | "type": "rectangle", 78 | "version": 2404, 79 | "versionNonce": 668300548, 80 | "isDeleted": false, 81 | "id": "GdyILsCD4rTRfB2LlEajF", 82 | "fillStyle": "hachure", 83 | "strokeWidth": 1, 84 | "strokeStyle": "solid", 85 | "roughness": 0, 86 | "opacity": 100, 87 | "angle": 0, 88 | "x": 369.6614313826775, 89 | "y": -293.72393920576224, 90 | "strokeColor": "#000000", 91 | "backgroundColor": "#15aabf", 92 | "width": 246, 93 | "height": 100, 94 | "seed": 1838921471, 95 | "groupIds": [], 96 | "roundness": { 97 | "type": 3 98 | }, 99 | "boundElements": [ 100 | { 101 | "type": "text", 102 | "id": "Pjp8Tj_wsaL6aNumfA2-8" 103 | }, 104 | { 105 | "id": "kR71fpC9EJ2vtymkWmfO3", 106 | "type": "arrow" 107 | } 108 | ], 109 | "updated": 1678879806656, 110 | "link": null, 111 | "locked": false 112 | }, 113 | { 114 | "type": "text", 115 | "version": 1357, 116 | "versionNonce": 36392252, 117 | "isDeleted": false, 118 | "id": "Pjp8Tj_wsaL6aNumfA2-8", 119 | "fillStyle": "hachure", 120 | "strokeWidth": 1, 121 | "strokeStyle": "solid", 122 | "roughness": 0, 123 | "opacity": 100, 124 | "angle": 0, 125 | "x": 397.055466722033, 126 | "y": -277.32393920576226, 127 | "strokeColor": "#000000", 128 | "backgroundColor": "transparent", 129 | "width": 191.21192932128906, 130 | "height": 67.2, 131 | "seed": 341736561, 132 | "groupIds": [], 133 | "roundness": null, 134 | "boundElements": [], 135 | "updated": 1678879806656, 136 | "link": null, 137 | "locked": false, 138 | "fontSize": 28, 139 | "fontFamily": 1, 140 | "text": "avro-consumer\n-service", 141 | "textAlign": "center", 142 | "verticalAlign": "middle", 143 | "containerId": "GdyILsCD4rTRfB2LlEajF", 144 | "originalText": "avro-consumer\n-service" 145 | }, 146 | { 147 | "type": "text", 148 | "version": 133, 149 | "versionNonce": 642139580, 150 | "isDeleted": false, 151 | "id": "lv-AGSCax4dA7SdjKXS01", 152 | "fillStyle": "hachure", 153 | "strokeWidth": 1, 154 | "strokeStyle": "solid", 155 | "roughness": 1, 156 | "opacity": 100, 157 | "angle": 0, 158 | "x": -37.7484875083814, 159 | "y": -92.42587504769686, 160 | "strokeColor": "#000000", 161 | "backgroundColor": "transparent", 162 | "width": 81.75994873046875, 163 | "height": 33.6, 164 | "seed": 36037695, 165 | "groupIds": [], 166 | "roundness": null, 167 | "boundElements": [], 168 | "updated": 1678879806656, 169 | "link": null, 170 | "locked": false, 171 | "fontSize": 28, 172 | "fontFamily": 1, 173 | "text": "Kafka", 174 | "textAlign": "left", 175 | "verticalAlign": "top", 176 | "containerId": null, 177 | "originalText": "Kafka" 178 | }, 179 | { 180 | "type": "rectangle", 181 | "version": 590, 182 | "versionNonce": 243686276, 183 | "isDeleted": false, 184 | "id": "gwGOOW99YxclikCNuOiyC", 185 | "fillStyle": "hachure", 186 | "strokeWidth": 1, 187 | "strokeStyle": "solid", 188 | "roughness": 1, 189 | "opacity": 100, 190 | "angle": 0, 191 | "x": -59.35066205536498, 192 | "y": -106.43794780160306, 193 | "strokeColor": "#000000", 194 | "backgroundColor": "#ced4da", 195 | "width": 759.6413082546653, 196 | "height": 112.53458658854157, 197 | "seed": 945659396, 198 | "groupIds": [ 199 | "0ROEoy_tGSp_GOIaAzPC7" 200 | ], 201 | "roundness": { 202 | "type": 3 203 | }, 204 | "boundElements": [], 205 | "updated": 1678879817961, 206 | "link": null, 207 | "locked": false 208 | }, 209 | { 210 | "type": "rectangle", 211 | "version": 748, 212 | "versionNonce": 1212288700, 213 | "isDeleted": false, 214 | "id": "c6bXALbEDlXFHZ3-dESbS", 215 | "fillStyle": "cross-hatch", 216 | "strokeWidth": 1, 217 | "strokeStyle": "solid", 218 | "roughness": 1, 219 | "opacity": 100, 220 | "angle": 0, 221 | "x": -25.429175306775733, 222 | "y": -45.88191752816567, 223 | "strokeColor": "#000000", 224 | "backgroundColor": "#868e96", 225 | "width": 708, 226 | "height": 43.6, 227 | "seed": 42155396, 228 | "groupIds": [ 229 | "0ROEoy_tGSp_GOIaAzPC7" 230 | ], 231 | "roundness": { 232 | "type": 3 233 | }, 234 | "boundElements": [ 235 | { 236 | "type": "text", 237 | "id": "OYsQvzF4kDoVvy7m5xy62" 238 | }, 239 | { 240 | "id": "cA_hWJzSnGig-8CoXOII4", 241 | "type": "arrow" 242 | }, 243 | { 244 | "id": "kR71fpC9EJ2vtymkWmfO3", 245 | "type": "arrow" 246 | } 247 | ], 248 | "updated": 1678879817961, 249 | "link": null, 250 | "locked": false 251 | }, 252 | { 253 | "type": "text", 254 | "version": 646, 255 | "versionNonce": 1390579644, 256 | "isDeleted": false, 257 | "id": "OYsQvzF4kDoVvy7m5xy62", 258 | "fillStyle": "hachure", 259 | "strokeWidth": 1, 260 | "strokeStyle": "solid", 261 | "roughness": 1, 262 | "opacity": 100, 263 | "angle": 0, 264 | "x": 202.36095530845864, 265 | "y": -36.081917528165675, 266 | "strokeColor": "#000000", 267 | "backgroundColor": "transparent", 268 | "width": 252.41973876953125, 269 | "height": 24, 270 | "seed": 1170795780, 271 | "groupIds": [ 272 | "0ROEoy_tGSp_GOIaAzPC7" 273 | ], 274 | "roundness": null, 275 | "boundElements": [], 276 | "updated": 1678879817962, 277 | "link": null, 278 | "locked": false, 279 | "fontSize": 20, 280 | "fontFamily": 1, 281 | "text": "avro-de-serialization-news", 282 | "textAlign": "center", 283 | "verticalAlign": "middle", 284 | "containerId": "c6bXALbEDlXFHZ3-dESbS", 285 | "originalText": "avro-de-serialization-news" 286 | }, 287 | { 288 | "type": "arrow", 289 | "version": 1123, 290 | "versionNonce": 805873924, 291 | "isDeleted": false, 292 | "id": "cA_hWJzSnGig-8CoXOII4", 293 | "fillStyle": "hachure", 294 | "strokeWidth": 2, 295 | "strokeStyle": "solid", 296 | "roughness": 1, 297 | "opacity": 100, 298 | "angle": 0, 299 | "x": 194.6782043194337, 300 | "y": -194.78448947160427, 301 | "strokeColor": "#000000", 302 | "backgroundColor": "transparent", 303 | "width": 0.6880983468813611, 304 | "height": 142.12126024722284, 305 | "seed": 2115424444, 306 | "groupIds": [], 307 | "roundness": { 308 | "type": 2 309 | }, 310 | "boundElements": [], 311 | "updated": 1678879823698, 312 | "link": null, 313 | "locked": false, 314 | "startBinding": { 315 | "elementId": "NKmNZxYxWMCKh3prRiPwX", 316 | "focus": 0.04257119600005342, 317 | "gap": 1 318 | }, 319 | "endBinding": { 320 | "elementId": "c6bXALbEDlXFHZ3-dESbS", 321 | "focus": -0.37669600425805716, 322 | "gap": 6.7813116962157665 323 | }, 324 | "lastCommittedPoint": null, 325 | "startArrowhead": null, 326 | "endArrowhead": "arrow", 327 | "points": [ 328 | [ 329 | 0, 330 | 0 331 | ], 332 | [ 333 | 0.6880983468813611, 334 | 51.8433847960855 335 | ], 336 | [ 337 | 0.5690872609807798, 338 | 142.12126024722284 339 | ] 340 | ] 341 | }, 342 | { 343 | "type": "arrow", 344 | "version": 1876, 345 | "versionNonce": 110422148, 346 | "isDeleted": false, 347 | "id": "kR71fpC9EJ2vtymkWmfO3", 348 | "fillStyle": "hachure", 349 | "strokeWidth": 2, 350 | "strokeStyle": "solid", 351 | "roughness": 1, 352 | "opacity": 100, 353 | "angle": 0, 354 | "x": 504.07705372114157, 355 | "y": -52.87000448136375, 356 | "strokeColor": "#000000", 357 | "backgroundColor": "transparent", 358 | "width": 0.4551302540264146, 359 | "height": 131.507803933339, 360 | "seed": 821385532, 361 | "groupIds": [], 362 | "roundness": { 363 | "type": 2 364 | }, 365 | "boundElements": [], 366 | "updated": 1678879823698, 367 | "link": null, 368 | "locked": false, 369 | "startBinding": { 370 | "elementId": "c6bXALbEDlXFHZ3-dESbS", 371 | "focus": 0.4954179733584482, 372 | "gap": 6.988086953198078 373 | }, 374 | "endBinding": { 375 | "elementId": "GdyILsCD4rTRfB2LlEajF", 376 | "focus": -0.09804207976871974, 377 | "gap": 9.346130791059466 378 | }, 379 | "lastCommittedPoint": null, 380 | "startArrowhead": null, 381 | "endArrowhead": "arrow", 382 | "points": [ 383 | [ 384 | 0, 385 | 0 386 | ], 387 | [ 388 | 0.4551302540264146, 389 | -131.507803933339 390 | ] 391 | ] 392 | }, 393 | { 394 | "type": "text", 395 | "version": 155, 396 | "versionNonce": 176153476, 397 | "isDeleted": false, 398 | "id": "8Xgjm5oMPiooNU1SFahLJ", 399 | "fillStyle": "hachure", 400 | "strokeWidth": 2, 401 | "strokeStyle": "solid", 402 | "roughness": 0, 403 | "opacity": 100, 404 | "angle": 0, 405 | "x": -26.2818974747438, 406 | "y": -172.8341931222418, 407 | "strokeColor": "#000000", 408 | "backgroundColor": "transparent", 409 | "width": 199.49981689453125, 410 | "height": 48, 411 | "seed": 849388479, 412 | "groupIds": [], 413 | "roundness": null, 414 | "boundElements": [], 415 | "updated": 1678879806656, 416 | "link": null, 417 | "locked": false, 418 | "fontSize": 20, 419 | "fontFamily": 1, 420 | "text": "key: StringSerializer\nvalue: AvroSerializer", 421 | "textAlign": "left", 422 | "verticalAlign": "top", 423 | "containerId": null, 424 | "originalText": "key: StringSerializer\nvalue: AvroSerializer" 425 | }, 426 | { 427 | "type": "text", 428 | "version": 410, 429 | "versionNonce": 1216677564, 430 | "isDeleted": false, 431 | "id": "cw4AWbHk93zAAgzH5CcVi", 432 | "fillStyle": "hachure", 433 | "strokeWidth": 2, 434 | "strokeStyle": "solid", 435 | "roughness": 0, 436 | "opacity": 100, 437 | "angle": 0, 438 | "x": 526.5770061982159, 439 | "y": -172.6714327055751, 440 | "strokeColor": "#000000", 441 | "backgroundColor": "transparent", 442 | "width": 224.7397918701172, 443 | "height": 48, 444 | "seed": 898367295, 445 | "groupIds": [], 446 | "roundness": null, 447 | "boundElements": [], 448 | "updated": 1678879806656, 449 | "link": null, 450 | "locked": false, 451 | "fontSize": 20, 452 | "fontFamily": 1, 453 | "text": "key: StringDeserializer\nvalue: AvroDeserializer", 454 | "textAlign": "left", 455 | "verticalAlign": "top", 456 | "containerId": null, 457 | "originalText": "key: StringDeserializer\nvalue: AvroDeserializer" 458 | } 459 | ], 460 | "appState": { 461 | "gridSize": null, 462 | "viewBackgroundColor": "#ffffff" 463 | }, 464 | "files": {} 465 | } -------------------------------------------------------------------------------- /documentation/avro-de-serialization.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ivangfr/spring-kafka-de-serialization-types/83148510f1052698785d2704eafbfb115f20bdf2/documentation/avro-de-serialization.jpeg -------------------------------------------------------------------------------- /documentation/json-de-serialization.excalidraw: -------------------------------------------------------------------------------- 1 | { 2 | "type": "excalidraw", 3 | "version": 2, 4 | "source": "https://excalidraw.com", 5 | "elements": [ 6 | { 7 | "type": "rectangle", 8 | "version": 1994, 9 | "versionNonce": 1725481604, 10 | "isDeleted": false, 11 | "id": "NKmNZxYxWMCKh3prRiPwX", 12 | "fillStyle": "hachure", 13 | "strokeWidth": 1, 14 | "strokeStyle": "solid", 15 | "roughness": 1, 16 | "opacity": 100, 17 | "angle": 0, 18 | "x": 76.02142385213472, 19 | "y": -295.9039963075418, 20 | "strokeColor": "#000000", 21 | "backgroundColor": "#82c91e", 22 | "width": 246, 23 | "height": 100, 24 | "seed": 617930556, 25 | "groupIds": [], 26 | "roundness": { 27 | "type": 3 28 | }, 29 | "boundElements": [ 30 | { 31 | "type": "text", 32 | "id": "GrVT2PZuYu1cRv3sXwFoI" 33 | }, 34 | { 35 | "id": "cA_hWJzSnGig-8CoXOII4", 36 | "type": "arrow" 37 | } 38 | ], 39 | "updated": 1678879863578, 40 | "link": null, 41 | "locked": false 42 | }, 43 | { 44 | "type": "text", 45 | "version": 939, 46 | "versionNonce": 572865084, 47 | "isDeleted": false, 48 | "id": "GrVT2PZuYu1cRv3sXwFoI", 49 | "fillStyle": "hachure", 50 | "strokeWidth": 1, 51 | "strokeStyle": "solid", 52 | "roughness": 0, 53 | "opacity": 100, 54 | "angle": 0, 55 | "x": 109.47747091024019, 56 | "y": -279.5039963075418, 57 | "strokeColor": "#000000", 58 | "backgroundColor": "transparent", 59 | "width": 179.08790588378906, 60 | "height": 67.2, 61 | "seed": 294979727, 62 | "groupIds": [], 63 | "roundness": null, 64 | "boundElements": [], 65 | "updated": 1678879855174, 66 | "link": null, 67 | "locked": false, 68 | "fontSize": 28, 69 | "fontFamily": 1, 70 | "text": "json-producer\n-service", 71 | "textAlign": "center", 72 | "verticalAlign": "middle", 73 | "containerId": "NKmNZxYxWMCKh3prRiPwX", 74 | "originalText": "json-producer\n-service" 75 | }, 76 | { 77 | "type": "rectangle", 78 | "version": 2406, 79 | "versionNonce": 1864749572, 80 | "isDeleted": false, 81 | "id": "GdyILsCD4rTRfB2LlEajF", 82 | "fillStyle": "hachure", 83 | "strokeWidth": 1, 84 | "strokeStyle": "solid", 85 | "roughness": 1, 86 | "opacity": 100, 87 | "angle": 0, 88 | "x": 369.6614313826775, 89 | "y": -293.72393920576224, 90 | "strokeColor": "#000000", 91 | "backgroundColor": "#15aabf", 92 | "width": 246, 93 | "height": 100, 94 | "seed": 569219004, 95 | "groupIds": [], 96 | "roundness": { 97 | "type": 3 98 | }, 99 | "boundElements": [ 100 | { 101 | "type": "text", 102 | "id": "Pjp8Tj_wsaL6aNumfA2-8" 103 | }, 104 | { 105 | "id": "kR71fpC9EJ2vtymkWmfO3", 106 | "type": "arrow" 107 | } 108 | ], 109 | "updated": 1678879863578, 110 | "link": null, 111 | "locked": false 112 | }, 113 | { 114 | "type": "text", 115 | "version": 1358, 116 | "versionNonce": 1786417852, 117 | "isDeleted": false, 118 | "id": "Pjp8Tj_wsaL6aNumfA2-8", 119 | "fillStyle": "hachure", 120 | "strokeWidth": 1, 121 | "strokeStyle": "solid", 122 | "roughness": 0, 123 | "opacity": 100, 124 | "angle": 0, 125 | "x": 401.0034723983025, 126 | "y": -277.32393920576226, 127 | "strokeColor": "#000000", 128 | "backgroundColor": "transparent", 129 | "width": 183.31591796875, 130 | "height": 67.2, 131 | "seed": 341736561, 132 | "groupIds": [], 133 | "roundness": null, 134 | "boundElements": [], 135 | "updated": 1678879855175, 136 | "link": null, 137 | "locked": false, 138 | "fontSize": 28, 139 | "fontFamily": 1, 140 | "text": "json-consumer\n-service", 141 | "textAlign": "center", 142 | "verticalAlign": "middle", 143 | "containerId": "GdyILsCD4rTRfB2LlEajF", 144 | "originalText": "json-consumer\n-service" 145 | }, 146 | { 147 | "type": "text", 148 | "version": 134, 149 | "versionNonce": 529587004, 150 | "isDeleted": false, 151 | "id": "lv-AGSCax4dA7SdjKXS01", 152 | "fillStyle": "hachure", 153 | "strokeWidth": 1, 154 | "strokeStyle": "solid", 155 | "roughness": 1, 156 | "opacity": 100, 157 | "angle": 0, 158 | "x": -37.7484875083814, 159 | "y": -92.42587504769686, 160 | "strokeColor": "#000000", 161 | "backgroundColor": "transparent", 162 | "width": 81.75994873046875, 163 | "height": 33.6, 164 | "seed": 36037695, 165 | "groupIds": [], 166 | "roundness": null, 167 | "boundElements": [], 168 | "updated": 1678879855175, 169 | "link": null, 170 | "locked": false, 171 | "fontSize": 28, 172 | "fontFamily": 1, 173 | "text": "Kafka", 174 | "textAlign": "left", 175 | "verticalAlign": "top", 176 | "containerId": null, 177 | "originalText": "Kafka" 178 | }, 179 | { 180 | "type": "rectangle", 181 | "version": 593, 182 | "versionNonce": 438628740, 183 | "isDeleted": false, 184 | "id": "gwGOOW99YxclikCNuOiyC", 185 | "fillStyle": "hachure", 186 | "strokeWidth": 1, 187 | "strokeStyle": "solid", 188 | "roughness": 1, 189 | "opacity": 100, 190 | "angle": 0, 191 | "x": -59.59504682098998, 192 | "y": -106.55745463754056, 193 | "strokeColor": "#000000", 194 | "backgroundColor": "#ced4da", 195 | "width": 759.6413082546653, 196 | "height": 112.53458658854157, 197 | "seed": 1549484092, 198 | "groupIds": [ 199 | "3i6r-drNto_GB_QZbnbKt" 200 | ], 201 | "roundness": { 202 | "type": 3 203 | }, 204 | "boundElements": [], 205 | "updated": 1678879863578, 206 | "link": null, 207 | "locked": false 208 | }, 209 | { 210 | "type": "rectangle", 211 | "version": 751, 212 | "versionNonce": 1729372420, 213 | "isDeleted": false, 214 | "id": "c6bXALbEDlXFHZ3-dESbS", 215 | "fillStyle": "cross-hatch", 216 | "strokeWidth": 1, 217 | "strokeStyle": "solid", 218 | "roughness": 1, 219 | "opacity": 100, 220 | "angle": 0, 221 | "x": -25.673560072400733, 222 | "y": -46.00142436410317, 223 | "strokeColor": "#000000", 224 | "backgroundColor": "#868e96", 225 | "width": 708, 226 | "height": 43.6, 227 | "seed": 1006622908, 228 | "groupIds": [ 229 | "3i6r-drNto_GB_QZbnbKt" 230 | ], 231 | "roundness": { 232 | "type": 3 233 | }, 234 | "boundElements": [ 235 | { 236 | "type": "text", 237 | "id": "OYsQvzF4kDoVvy7m5xy62" 238 | }, 239 | { 240 | "id": "cA_hWJzSnGig-8CoXOII4", 241 | "type": "arrow" 242 | }, 243 | { 244 | "id": "kR71fpC9EJ2vtymkWmfO3", 245 | "type": "arrow" 246 | } 247 | ], 248 | "updated": 1678879863578, 249 | "link": null, 250 | "locked": false 251 | }, 252 | { 253 | "type": "text", 254 | "version": 649, 255 | "versionNonce": 1030840452, 256 | "isDeleted": false, 257 | "id": "OYsQvzF4kDoVvy7m5xy62", 258 | "fillStyle": "hachure", 259 | "strokeWidth": 1, 260 | "strokeStyle": "solid", 261 | "roughness": 1, 262 | "opacity": 100, 263 | "angle": 0, 264 | "x": 204.93657023765786, 265 | "y": -36.201424364103175, 266 | "strokeColor": "#000000", 267 | "backgroundColor": "transparent", 268 | "width": 246.7797393798828, 269 | "height": 24, 270 | "seed": 1551720764, 271 | "groupIds": [ 272 | "3i6r-drNto_GB_QZbnbKt" 273 | ], 274 | "roundness": null, 275 | "boundElements": [], 276 | "updated": 1678879863578, 277 | "link": null, 278 | "locked": false, 279 | "fontSize": 20, 280 | "fontFamily": 1, 281 | "text": "json-de-serialization-news", 282 | "textAlign": "center", 283 | "verticalAlign": "middle", 284 | "containerId": "c6bXALbEDlXFHZ3-dESbS", 285 | "originalText": "json-de-serialization-news" 286 | }, 287 | { 288 | "type": "arrow", 289 | "version": 1125, 290 | "versionNonce": 127900348, 291 | "isDeleted": false, 292 | "id": "cA_hWJzSnGig-8CoXOII4", 293 | "fillStyle": "hachure", 294 | "strokeWidth": 2, 295 | "strokeStyle": "solid", 296 | "roughness": 1, 297 | "opacity": 100, 298 | "angle": 0, 299 | "x": 194.5377482906166, 300 | "y": -195.78079143976237, 301 | "strokeColor": "#000000", 302 | "backgroundColor": "transparent", 303 | "width": 0.8285543756984453, 304 | "height": 142.99805537944343, 305 | "seed": 2108559236, 306 | "groupIds": [], 307 | "roundness": { 308 | "type": 2 309 | }, 310 | "boundElements": [], 311 | "updated": 1678879869710, 312 | "link": null, 313 | "locked": false, 314 | "startBinding": { 315 | "elementId": "NKmNZxYxWMCKh3prRiPwX", 316 | "focus": 0.04257119600005342, 317 | "gap": 1 318 | }, 319 | "endBinding": { 320 | "elementId": "c6bXALbEDlXFHZ3-dESbS", 321 | "focus": -0.37669600425805716, 322 | "gap": 6.7813116962157665 323 | }, 324 | "lastCommittedPoint": null, 325 | "startArrowhead": null, 326 | "endArrowhead": "arrow", 327 | "points": [ 328 | [ 329 | 0, 330 | 0 331 | ], 332 | [ 333 | 0.8285543756984453, 334 | 52.839686764243595 335 | ], 336 | [ 337 | 0.5078746904305831, 338 | 142.99805537944343 339 | ] 340 | ] 341 | }, 342 | { 343 | "type": "arrow", 344 | "version": 1878, 345 | "versionNonce": 1658118972, 346 | "isDeleted": false, 347 | "id": "kR71fpC9EJ2vtymkWmfO3", 348 | "fillStyle": "hachure", 349 | "strokeWidth": 2, 350 | "strokeStyle": "solid", 351 | "roughness": 1, 352 | "opacity": 100, 353 | "angle": 0, 354 | "x": 503.8755800973082, 355 | "y": -52.98951131730125, 356 | "strokeColor": "#000000", 357 | "backgroundColor": "transparent", 358 | "width": 0.597456184885516, 359 | "height": 131.3882970974015, 360 | "seed": 2009693956, 361 | "groupIds": [], 362 | "roundness": { 363 | "type": 2 364 | }, 365 | "boundElements": [], 366 | "updated": 1678879869710, 367 | "link": null, 368 | "locked": false, 369 | "startBinding": { 370 | "elementId": "c6bXALbEDlXFHZ3-dESbS", 371 | "focus": 0.4954179733584482, 372 | "gap": 6.988086953198078 373 | }, 374 | "endBinding": { 375 | "elementId": "GdyILsCD4rTRfB2LlEajF", 376 | "focus": -0.09804207976871974, 377 | "gap": 9.346130791059466 378 | }, 379 | "lastCommittedPoint": null, 380 | "startArrowhead": null, 381 | "endArrowhead": "arrow", 382 | "points": [ 383 | [ 384 | 0, 385 | 0 386 | ], 387 | [ 388 | 0.597456184885516, 389 | -131.3882970974015 390 | ] 391 | ] 392 | }, 393 | { 394 | "type": "text", 395 | "version": 156, 396 | "versionNonce": 1737864580, 397 | "isDeleted": false, 398 | "id": "8Xgjm5oMPiooNU1SFahLJ", 399 | "fillStyle": "hachure", 400 | "strokeWidth": 2, 401 | "strokeStyle": "solid", 402 | "roughness": 0, 403 | "opacity": 100, 404 | "angle": 0, 405 | "x": -26.2818974747438, 406 | "y": -172.8341931222418, 407 | "strokeColor": "#000000", 408 | "backgroundColor": "transparent", 409 | "width": 200.19981384277344, 410 | "height": 48, 411 | "seed": 849388479, 412 | "groupIds": [], 413 | "roundness": null, 414 | "boundElements": [], 415 | "updated": 1678879855175, 416 | "link": null, 417 | "locked": false, 418 | "fontSize": 20, 419 | "fontFamily": 1, 420 | "text": "key: StringSerializer\nvalue: JsonSerializer", 421 | "textAlign": "left", 422 | "verticalAlign": "top", 423 | "containerId": null, 424 | "originalText": "key: StringSerializer\nvalue: JsonSerializer" 425 | }, 426 | { 427 | "type": "text", 428 | "version": 411, 429 | "versionNonce": 1307399356, 430 | "isDeleted": false, 431 | "id": "cw4AWbHk93zAAgzH5CcVi", 432 | "fillStyle": "hachure", 433 | "strokeWidth": 2, 434 | "strokeStyle": "solid", 435 | "roughness": 0, 436 | "opacity": 100, 437 | "angle": 0, 438 | "x": 526.5770061982159, 439 | "y": -172.6714327055751, 440 | "strokeColor": "#000000", 441 | "backgroundColor": "transparent", 442 | "width": 225.43978881835938, 443 | "height": 48, 444 | "seed": 898367295, 445 | "groupIds": [], 446 | "roundness": null, 447 | "boundElements": [], 448 | "updated": 1678879855175, 449 | "link": null, 450 | "locked": false, 451 | "fontSize": 20, 452 | "fontFamily": 1, 453 | "text": "key: StringDeserializer\nvalue: JsonDeserializer", 454 | "textAlign": "left", 455 | "verticalAlign": "top", 456 | "containerId": null, 457 | "originalText": "key: StringDeserializer\nvalue: JsonDeserializer" 458 | } 459 | ], 460 | "appState": { 461 | "gridSize": null, 462 | "viewBackgroundColor": "#ffffff" 463 | }, 464 | "files": {} 465 | } -------------------------------------------------------------------------------- /documentation/json-de-serialization.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ivangfr/spring-kafka-de-serialization-types/83148510f1052698785d2704eafbfb115f20bdf2/documentation/json-de-serialization.jpeg -------------------------------------------------------------------------------- /documentation/string-de-serialization.excalidraw: -------------------------------------------------------------------------------- 1 | { 2 | "type": "excalidraw", 3 | "version": 2, 4 | "source": "https://excalidraw.com", 5 | "elements": [ 6 | { 7 | "type": "rectangle", 8 | "version": 1996, 9 | "versionNonce": 901030588, 10 | "isDeleted": false, 11 | "id": "NKmNZxYxWMCKh3prRiPwX", 12 | "fillStyle": "hachure", 13 | "strokeWidth": 1, 14 | "strokeStyle": "solid", 15 | "roughness": 1, 16 | "opacity": 100, 17 | "angle": 0, 18 | "x": 76.02142385213472, 19 | "y": -295.7381841330626, 20 | "strokeColor": "#000000", 21 | "backgroundColor": "#82c91e", 22 | "width": 246, 23 | "height": 100, 24 | "seed": 1239189380, 25 | "groupIds": [], 26 | "roundness": { 27 | "type": 3 28 | }, 29 | "boundElements": [ 30 | { 31 | "type": "text", 32 | "id": "GrVT2PZuYu1cRv3sXwFoI" 33 | }, 34 | { 35 | "id": "cA_hWJzSnGig-8CoXOII4", 36 | "type": "arrow" 37 | } 38 | ], 39 | "updated": 1678879909126, 40 | "link": null, 41 | "locked": false 42 | }, 43 | { 44 | "type": "text", 45 | "version": 940, 46 | "versionNonce": 846237188, 47 | "isDeleted": false, 48 | "id": "GrVT2PZuYu1cRv3sXwFoI", 49 | "fillStyle": "hachure", 50 | "strokeWidth": 1, 51 | "strokeStyle": "solid", 52 | "roughness": 0, 53 | "opacity": 100, 54 | "angle": 0, 55 | "x": 97.81547597615815, 56 | "y": -279.3381841330626, 57 | "strokeColor": "#000000", 58 | "backgroundColor": "transparent", 59 | "width": 202.41189575195312, 60 | "height": 67.2, 61 | "seed": 294979727, 62 | "groupIds": [], 63 | "roundness": null, 64 | "boundElements": [], 65 | "updated": 1678879900453, 66 | "link": null, 67 | "locked": false, 68 | "fontSize": 28, 69 | "fontFamily": 1, 70 | "text": "string-producer\n-service", 71 | "textAlign": "center", 72 | "verticalAlign": "middle", 73 | "containerId": "NKmNZxYxWMCKh3prRiPwX", 74 | "originalText": "string-producer\n-service" 75 | }, 76 | { 77 | "type": "rectangle", 78 | "version": 2408, 79 | "versionNonce": 420014908, 80 | "isDeleted": false, 81 | "id": "GdyILsCD4rTRfB2LlEajF", 82 | "fillStyle": "hachure", 83 | "strokeWidth": 1, 84 | "strokeStyle": "solid", 85 | "roughness": 1, 86 | "opacity": 100, 87 | "angle": 0, 88 | "x": 369.6614313826775, 89 | "y": -293.72393920576224, 90 | "strokeColor": "#000000", 91 | "backgroundColor": "#15aabf", 92 | "width": 246, 93 | "height": 100, 94 | "seed": 661190404, 95 | "groupIds": [], 96 | "roundness": { 97 | "type": 3 98 | }, 99 | "boundElements": [ 100 | { 101 | "type": "text", 102 | "id": "Pjp8Tj_wsaL6aNumfA2-8" 103 | }, 104 | { 105 | "id": "kR71fpC9EJ2vtymkWmfO3", 106 | "type": "arrow" 107 | } 108 | ], 109 | "updated": 1678879909126, 110 | "link": null, 111 | "locked": false 112 | }, 113 | { 114 | "type": "text", 115 | "version": 1359, 116 | "versionNonce": 481832324, 117 | "isDeleted": false, 118 | "id": "Pjp8Tj_wsaL6aNumfA2-8", 119 | "fillStyle": "hachure", 120 | "strokeWidth": 1, 121 | "strokeStyle": "solid", 122 | "roughness": 0, 123 | "opacity": 100, 124 | "angle": 0, 125 | "x": 389.3414774642205, 126 | "y": -277.32393920576226, 127 | "strokeColor": "#000000", 128 | "backgroundColor": "transparent", 129 | "width": 206.63990783691406, 130 | "height": 67.2, 131 | "seed": 341736561, 132 | "groupIds": [], 133 | "roundness": null, 134 | "boundElements": [], 135 | "updated": 1678879900454, 136 | "link": null, 137 | "locked": false, 138 | "fontSize": 28, 139 | "fontFamily": 1, 140 | "text": "string-consumer\n-service", 141 | "textAlign": "center", 142 | "verticalAlign": "middle", 143 | "containerId": "GdyILsCD4rTRfB2LlEajF", 144 | "originalText": "string-consumer\n-service" 145 | }, 146 | { 147 | "type": "text", 148 | "version": 135, 149 | "versionNonce": 736091396, 150 | "isDeleted": false, 151 | "id": "lv-AGSCax4dA7SdjKXS01", 152 | "fillStyle": "hachure", 153 | "strokeWidth": 1, 154 | "strokeStyle": "solid", 155 | "roughness": 1, 156 | "opacity": 100, 157 | "angle": 0, 158 | "x": -37.7484875083814, 159 | "y": -92.42587504769686, 160 | "strokeColor": "#000000", 161 | "backgroundColor": "transparent", 162 | "width": 81.75994873046875, 163 | "height": 33.6, 164 | "seed": 36037695, 165 | "groupIds": [], 166 | "roundness": null, 167 | "boundElements": [], 168 | "updated": 1678879900454, 169 | "link": null, 170 | "locked": false, 171 | "fontSize": 28, 172 | "fontFamily": 1, 173 | "text": "Kafka", 174 | "textAlign": "left", 175 | "verticalAlign": "top", 176 | "containerId": null, 177 | "originalText": "Kafka" 178 | }, 179 | { 180 | "type": "rectangle", 181 | "version": 597, 182 | "versionNonce": 197141436, 183 | "isDeleted": false, 184 | "id": "gwGOOW99YxclikCNuOiyC", 185 | "fillStyle": "hachure", 186 | "strokeWidth": 1, 187 | "strokeStyle": "solid", 188 | "roughness": 1, 189 | "opacity": 100, 190 | "angle": 0, 191 | "x": -59.59504682098998, 192 | "y": -106.44594340707181, 193 | "strokeColor": "#000000", 194 | "backgroundColor": "#ced4da", 195 | "width": 759.6413082546653, 196 | "height": 112.53458658854157, 197 | "seed": 146503300, 198 | "groupIds": [ 199 | "sa3Ax-WgNGnA-_zmnjbX3" 200 | ], 201 | "roundness": { 202 | "type": 3 203 | }, 204 | "boundElements": [], 205 | "updated": 1678879909126, 206 | "link": null, 207 | "locked": false 208 | }, 209 | { 210 | "type": "rectangle", 211 | "version": 755, 212 | "versionNonce": 66866236, 213 | "isDeleted": false, 214 | "id": "c6bXALbEDlXFHZ3-dESbS", 215 | "fillStyle": "cross-hatch", 216 | "strokeWidth": 1, 217 | "strokeStyle": "solid", 218 | "roughness": 1, 219 | "opacity": 100, 220 | "angle": 0, 221 | "x": -25.673560072400733, 222 | "y": -45.88991313363442, 223 | "strokeColor": "#000000", 224 | "backgroundColor": "#868e96", 225 | "width": 708, 226 | "height": 41, 227 | "seed": 714212868, 228 | "groupIds": [ 229 | "sa3Ax-WgNGnA-_zmnjbX3" 230 | ], 231 | "roundness": { 232 | "type": 3 233 | }, 234 | "boundElements": [ 235 | { 236 | "type": "text", 237 | "id": "OYsQvzF4kDoVvy7m5xy62" 238 | }, 239 | { 240 | "id": "cA_hWJzSnGig-8CoXOII4", 241 | "type": "arrow" 242 | }, 243 | { 244 | "id": "kR71fpC9EJ2vtymkWmfO3", 245 | "type": "arrow" 246 | } 247 | ], 248 | "updated": 1678879909126, 249 | "link": null, 250 | "locked": false 251 | }, 252 | { 253 | "type": "text", 254 | "version": 653, 255 | "versionNonce": 1671390396, 256 | "isDeleted": false, 257 | "id": "OYsQvzF4kDoVvy7m5xy62", 258 | "fillStyle": "hachure", 259 | "strokeWidth": 1, 260 | "strokeStyle": "solid", 261 | "roughness": 1, 262 | "opacity": 100, 263 | "angle": 0, 264 | "x": 196.60659129478677, 265 | "y": -37.38991313363442, 266 | "strokeColor": "#000000", 267 | "backgroundColor": "transparent", 268 | "width": 263.439697265625, 269 | "height": 24, 270 | "seed": 32155012, 271 | "groupIds": [ 272 | "sa3Ax-WgNGnA-_zmnjbX3" 273 | ], 274 | "roundness": null, 275 | "boundElements": [], 276 | "updated": 1678879909126, 277 | "link": null, 278 | "locked": false, 279 | "fontSize": 20, 280 | "fontFamily": 1, 281 | "text": "string-de-serialization-news", 282 | "textAlign": "center", 283 | "verticalAlign": "middle", 284 | "containerId": "c6bXALbEDlXFHZ3-dESbS", 285 | "originalText": "string-de-serialization-news" 286 | }, 287 | { 288 | "type": "arrow", 289 | "version": 1128, 290 | "versionNonce": 2125259780, 291 | "isDeleted": false, 292 | "id": "cA_hWJzSnGig-8CoXOII4", 293 | "fillStyle": "hachure", 294 | "strokeWidth": 2, 295 | "strokeStyle": "solid", 296 | "roughness": 1, 297 | "opacity": 100, 298 | "angle": 0, 299 | "x": 194.5377482906166, 300 | "y": -195.78079143976237, 301 | "strokeColor": "#000000", 302 | "backgroundColor": "transparent", 303 | "width": 0.8285543756984453, 304 | "height": 143.10956660991218, 305 | "seed": 727868860, 306 | "groupIds": [], 307 | "roundness": { 308 | "type": 2 309 | }, 310 | "boundElements": [], 311 | "updated": 1678879914106, 312 | "link": null, 313 | "locked": false, 314 | "startBinding": null, 315 | "endBinding": { 316 | "elementId": "c6bXALbEDlXFHZ3-dESbS", 317 | "focus": -0.37669600425805716, 318 | "gap": 6.7813116962157665 319 | }, 320 | "lastCommittedPoint": null, 321 | "startArrowhead": null, 322 | "endArrowhead": "arrow", 323 | "points": [ 324 | [ 325 | 0, 326 | 0 327 | ], 328 | [ 329 | 0.8285543756984453, 330 | 52.839686764243595 331 | ], 332 | [ 333 | 0.5078040562517856, 334 | 143.10956660991218 335 | ] 336 | ] 337 | }, 338 | { 339 | "type": "arrow", 340 | "version": 1882, 341 | "versionNonce": 184140676, 342 | "isDeleted": false, 343 | "id": "kR71fpC9EJ2vtymkWmfO3", 344 | "fillStyle": "hachure", 345 | "strokeWidth": 2, 346 | "strokeStyle": "solid", 347 | "roughness": 1, 348 | "opacity": 100, 349 | "angle": 0, 350 | "x": 503.87546724536554, 351 | "y": -52.8780000868325, 352 | "strokeColor": "#000000", 353 | "backgroundColor": "transparent", 354 | "width": 0.5976844632031657, 355 | "height": 131.49980832787026, 356 | "seed": 973491772, 357 | "groupIds": [], 358 | "roundness": { 359 | "type": 2 360 | }, 361 | "boundElements": [], 362 | "updated": 1678879914106, 363 | "link": null, 364 | "locked": false, 365 | "startBinding": { 366 | "elementId": "c6bXALbEDlXFHZ3-dESbS", 367 | "focus": 0.4954179733584482, 368 | "gap": 6.988086953198078 369 | }, 370 | "endBinding": { 371 | "elementId": "GdyILsCD4rTRfB2LlEajF", 372 | "focus": -0.09804207976871974, 373 | "gap": 9.346130791059466 374 | }, 375 | "lastCommittedPoint": null, 376 | "startArrowhead": null, 377 | "endArrowhead": "arrow", 378 | "points": [ 379 | [ 380 | 0, 381 | 0 382 | ], 383 | [ 384 | 0.5976844632031657, 385 | -131.49980832787026 386 | ] 387 | ] 388 | }, 389 | { 390 | "type": "text", 391 | "version": 157, 392 | "versionNonce": 1752712764, 393 | "isDeleted": false, 394 | "id": "8Xgjm5oMPiooNU1SFahLJ", 395 | "fillStyle": "hachure", 396 | "strokeWidth": 2, 397 | "strokeStyle": "solid", 398 | "roughness": 0, 399 | "opacity": 100, 400 | "angle": 0, 401 | "x": -26.2818974747438, 402 | "y": -172.8341931222418, 403 | "strokeColor": "#000000", 404 | "backgroundColor": "transparent", 405 | "width": 212.039794921875, 406 | "height": 48, 407 | "seed": 849388479, 408 | "groupIds": [], 409 | "roundness": null, 410 | "boundElements": [], 411 | "updated": 1678879900454, 412 | "link": null, 413 | "locked": false, 414 | "fontSize": 20, 415 | "fontFamily": 1, 416 | "text": "key: StringSerializer\nvalue: StringSerializer", 417 | "textAlign": "left", 418 | "verticalAlign": "top", 419 | "containerId": null, 420 | "originalText": "key: StringSerializer\nvalue: StringSerializer" 421 | }, 422 | { 423 | "type": "text", 424 | "version": 412, 425 | "versionNonce": 784550788, 426 | "isDeleted": false, 427 | "id": "cw4AWbHk93zAAgzH5CcVi", 428 | "fillStyle": "hachure", 429 | "strokeWidth": 2, 430 | "strokeStyle": "solid", 431 | "roughness": 0, 432 | "opacity": 100, 433 | "angle": 0, 434 | "x": 526.5770061982159, 435 | "y": -172.8341931222418, 436 | "strokeColor": "#000000", 437 | "backgroundColor": "transparent", 438 | "width": 237.27976989746094, 439 | "height": 48, 440 | "seed": 898367295, 441 | "groupIds": [], 442 | "roundness": null, 443 | "boundElements": [], 444 | "updated": 1678879900454, 445 | "link": null, 446 | "locked": false, 447 | "fontSize": 20, 448 | "fontFamily": 1, 449 | "text": "key: StringDeserializer\nvalue: StringDeserializer", 450 | "textAlign": "left", 451 | "verticalAlign": "top", 452 | "containerId": null, 453 | "originalText": "key: StringDeserializer\nvalue: StringDeserializer" 454 | } 455 | ], 456 | "appState": { 457 | "gridSize": null, 458 | "viewBackgroundColor": "#ffffff" 459 | }, 460 | "files": {} 461 | } -------------------------------------------------------------------------------- /documentation/string-de-serialization.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ivangfr/spring-kafka-de-serialization-types/83148510f1052698785d2704eafbfb115f20bdf2/documentation/string-de-serialization.jpeg -------------------------------------------------------------------------------- /json-de-serialization/README.md: -------------------------------------------------------------------------------- 1 | # spring-kafka-de-serialization-types 2 | ## `> json-de-serialization` 3 | 4 | ![json-de-serialization](../documentation/json-de-serialization.jpeg) 5 | 6 | This sample demonstrates a **producer** that pushes `News` messages to a topic in `Kafka` and a **consumer** that listens those messages from `Kafka`: 7 | - **Producer** serializes the message `key` using `StringSerializer` and the message `value` using `JsonSerializer`; 8 | - **Consumer** deserializes the message `key` using `StringDeserializer` and the message `value` using `JsonDeserializer`; 9 | - **Producer** creates the Kafka topics and **Consumer** doesn't. 10 | 11 | ## Start Environment 12 | 13 | Before starting producer and consumer, the services present in `docker-compose.yml` file must be up and running as explained in [Start Environment](https://github.com/ivangfr/spring-kafka-de-serialization-types#start-environment) section of the main README. 14 | 15 | ## Running applications using Maven 16 | 17 | > **Note**: run `json-producer-service` first so that it can create the `Kafka` topics. 18 | 19 | - **json-producer-service** 20 | 21 | - Open a terminal and navigate to the `spring-kafka-de-serialization-types` root folder; 22 | - Run application: 23 | ``` 24 | ./mvnw clean spring-boot:run --projects json-de-serialization/json-producer-service 25 | ``` 26 | - As soon as the producer is up and running, it will start pushing automatically and randomly `News` messages to `Kafka` topic `json-de-serialization-news`. The default `interval` between messages is `3 seconds`. 27 | 28 | - **json-consumer-service** 29 | 30 | - Open another terminal and make sure you are in `spring-kafka-de-serialization-types` root folder; 31 | - Run application: 32 | ``` 33 | ./mvnw clean spring-boot:run --projects json-de-serialization/json-consumer-service 34 | ``` 35 | - Once the consumer is up and running, it will start listening `News` messages from the `Kafka` topic `json-de-serialization-news`. 36 | 37 | ## Running applications as Docker containers 38 | 39 | - ### Build Docker images 40 | 41 | In a terminal and, inside the `spring-kafka-de-serialization-types` root folder, run: 42 | ``` 43 | ./build-docker-images.sh json-de-serialization 44 | ``` 45 | 46 | - ### Environment variables 47 | 48 | **json-producer-service** and **json-consumer-service** 49 | 50 | | Environment Variable | Description | 51 | |----------------------|-------------------------------------------------------------------------| 52 | | `KAFKA_HOST` | Specify host of the `Kafka` message broker to use (default `localhost`) | 53 | | `KAFKA_PORT` | Specify port of the `Kafka` message broker to use (default `29092`) | 54 | 55 | - ### Run Docker containers 56 | 57 | > **Note**: run `json-producer-service` first so that it can create the `Kafka` topics. 58 | 59 | - **json-producer-service** 60 | 61 | In a terminal, run the following Docker command: 62 | ``` 63 | docker run --rm --name json-producer-service -p 9082:9082 \ 64 | -e KAFKA_HOST=kafka -e KAFKA_PORT=9092 \ 65 | --network=spring-kafka-de-serialization-types_default \ 66 | ivanfranchin/json-producer-service:1.0.0 67 | ``` 68 | 69 | - **json-consumer-service** 70 | 71 | In another terminal, run the Docker command below: 72 | ``` 73 | docker run --rm --name json-consumer-service -p 9083:9083 \ 74 | -e KAFKA_HOST=kafka -e KAFKA_PORT=9092 \ 75 | --network=spring-kafka-de-serialization-types_default \ 76 | ivanfranchin/json-consumer-service:1.0.0 77 | ``` 78 | 79 | ## Shutdown 80 | 81 | - Go to the terminals where the applications are running and press `Ctrl+C`; 82 | - Stop the services present in `docker-compose.yml` as explained in [Shutdown](https://github.com/ivangfr/spring-kafka-de-serialization-types#shutdown) section of the main README. 83 | 84 | ## Cleanup 85 | 86 | To remove the Docker images created by this example, go to a terminal and, inside the `spring-kafka-de-serialization-types` root folder, run the following script: 87 | ``` 88 | ./remove-docker-images.sh json-de-serialization 89 | ``` 90 | -------------------------------------------------------------------------------- /json-de-serialization/json-consumer-service/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.ivanfranchin 7 | json-de-serialization 8 | 1.0.0 9 | ../pom.xml 10 | 11 | json-consumer-service 12 | json-consumer-service 13 | Demo project for Spring Boot 14 | 15 | 16 | org.springframework.boot 17 | spring-boot-starter-web 18 | 19 | 20 | org.springframework.kafka 21 | spring-kafka 22 | 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-starter-test 27 | test 28 | 29 | 30 | org.springframework.kafka 31 | spring-kafka-test 32 | test 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /json-de-serialization/json-consumer-service/src/main/java/com/ivanfranchin/jsonconsumerservice/JsonConsumerServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.jsonconsumerservice; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class JsonConsumerServiceApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(JsonConsumerServiceApplication.class, args); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /json-de-serialization/json-consumer-service/src/main/java/com/ivanfranchin/jsonconsumerservice/news/News.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.jsonconsumerservice.news; 2 | 3 | public record News(String id, Integer fromId, String fromName, String title) { 4 | } 5 | -------------------------------------------------------------------------------- /json-de-serialization/json-consumer-service/src/main/java/com/ivanfranchin/jsonconsumerservice/news/NewsListener.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.jsonconsumerservice.news; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.kafka.annotation.KafkaListener; 6 | import org.springframework.kafka.listener.adapter.ConsumerRecordMetadata; 7 | import org.springframework.messaging.handler.annotation.Payload; 8 | import org.springframework.stereotype.Service; 9 | 10 | @Service 11 | public class NewsListener { 12 | 13 | private static final Logger log = LoggerFactory.getLogger(NewsListener.class); 14 | 15 | @KafkaListener(topics = "${spring.kafka.consumer.topic}", groupId = "${spring.kafka.consumer.group-id}") 16 | public void listen(@Payload News news, ConsumerRecordMetadata metadata) { 17 | log.info("Received message\n---\nTOPIC: {}; PARTITION: {}; OFFSET: {};\nPAYLOAD: {}\n---", 18 | metadata.topic(), metadata.partition(), metadata.offset(), news); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /json-de-serialization/json-consumer-service/src/main/java/com/ivanfranchin/jsonconsumerservice/news/NewsListenerConfig.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.jsonconsumerservice.news; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerConfig; 4 | import org.apache.kafka.common.serialization.StringDeserializer; 5 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 6 | import org.springframework.context.annotation.Bean; 7 | import org.springframework.context.annotation.Configuration; 8 | import org.springframework.kafka.annotation.EnableKafka; 9 | import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; 10 | import org.springframework.kafka.core.ConsumerFactory; 11 | import org.springframework.kafka.core.DefaultKafkaConsumerFactory; 12 | import org.springframework.kafka.support.serializer.JsonDeserializer; 13 | 14 | import java.util.Map; 15 | 16 | @EnableKafka 17 | @Configuration 18 | public class NewsListenerConfig { 19 | 20 | private final KafkaProperties kafkaProperties; 21 | 22 | public NewsListenerConfig(KafkaProperties kafkaProperties) { 23 | this.kafkaProperties = kafkaProperties; 24 | } 25 | 26 | @Bean 27 | ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory() { 28 | ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); 29 | factory.setConsumerFactory(consumerFactory()); 30 | factory.setConcurrency(kafkaProperties.getListener().getConcurrency()); 31 | return factory; 32 | } 33 | 34 | @Bean 35 | ConsumerFactory consumerFactory() { 36 | Map props = kafkaProperties.buildConsumerProperties(null); 37 | props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); 38 | props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, JsonDeserializer.class); 39 | props.put(ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, false); 40 | return new DefaultKafkaConsumerFactory<>(props, new StringDeserializer(), new JsonDeserializer<>(News.class)); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /json-de-serialization/json-consumer-service/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=9083 2 | 3 | spring.application.name=json-consumer-service 4 | 5 | spring.kafka.bootstrap-servers=${KAFKA_HOST:localhost}:${KAFKA_PORT:29092} 6 | spring.kafka.consumer.group-id=json-consumer-service-group 7 | spring.kafka.consumer.topic=json-de-serialization-news 8 | spring.kafka.consumer.auto-offset-reset=earliest 9 | spring.kafka.listener.concurrency=2 -------------------------------------------------------------------------------- /json-de-serialization/json-consumer-service/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _ _ 2 | (_)___ ___ _ __ ___ ___ _ __ ___ _ _ _ __ ___ ___ _ __ ___ ___ _ ____ _(_) ___ ___ 3 | | / __|/ _ \| '_ \ _____ / __/ _ \| '_ \/ __| | | | '_ ` _ \ / _ \ '__|____/ __|/ _ \ '__\ \ / / |/ __/ _ \ 4 | | \__ \ (_) | | | |_____| (_| (_) | | | \__ \ |_| | | | | | | __/ | |_____\__ \ __/ | \ V /| | (_| __/ 5 | _/ |___/\___/|_| |_| \___\___/|_| |_|___/\__,_|_| |_| |_|\___|_| |___/\___|_| \_/ |_|\___\___| 6 | |__/ 7 | :: Spring Boot :: ${spring-boot.formatted-version} 8 | -------------------------------------------------------------------------------- /json-de-serialization/json-consumer-service/src/test/java/com/ivanfranchin/jsonconsumerservice/JsonConsumerServiceApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.jsonconsumerservice; 2 | 3 | import org.junit.jupiter.api.Disabled; 4 | import org.junit.jupiter.api.Test; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | 7 | @Disabled 8 | @SpringBootTest 9 | class JsonConsumerServiceApplicationTests { 10 | 11 | @Test 12 | void contextLoads() { 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /json-de-serialization/json-producer-service/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.ivanfranchin 7 | json-de-serialization 8 | 1.0.0 9 | ../pom.xml 10 | 11 | json-producer-service 12 | json-producer-service 13 | Demo project for Spring Boot 14 | 15 | 16 | org.springframework.boot 17 | spring-boot-starter-web 18 | 19 | 20 | org.springframework.kafka 21 | spring-kafka 22 | 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-starter-test 27 | test 28 | 29 | 30 | org.springframework.kafka 31 | spring-kafka-test 32 | test 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /json-de-serialization/json-producer-service/src/main/java/com/ivanfranchin/jsonproducerservice/JsonProducerServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.jsonproducerservice; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class JsonProducerServiceApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(JsonProducerServiceApplication.class, args); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /json-de-serialization/json-producer-service/src/main/java/com/ivanfranchin/jsonproducerservice/news/News.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.jsonproducerservice.news; 2 | 3 | public record News(String id, Integer fromId, String fromName, String title) { 4 | } 5 | -------------------------------------------------------------------------------- /json-de-serialization/json-producer-service/src/main/java/com/ivanfranchin/jsonproducerservice/news/NewsEmitter.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.jsonproducerservice.news; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 6 | import org.springframework.kafka.core.KafkaTemplate; 7 | import org.springframework.stereotype.Component; 8 | 9 | @Component 10 | public class NewsEmitter { 11 | 12 | private static final Logger log = LoggerFactory.getLogger(NewsEmitter.class); 13 | 14 | private final KafkaTemplate kafkaTemplate; 15 | private final KafkaProperties kafkaProperties; 16 | 17 | public NewsEmitter(KafkaTemplate kafkaTemplate, KafkaProperties kafkaProperties) { 18 | this.kafkaTemplate = kafkaTemplate; 19 | this.kafkaProperties = kafkaProperties; 20 | } 21 | 22 | public void send(News news) { 23 | String kafkaTopic = kafkaProperties.getProducer().getProperties().get("topic"); 24 | log.info("Sending News '{}' to topic '{}'", news, kafkaTopic); 25 | kafkaTemplate.send(kafkaTopic, news.id(), news); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /json-de-serialization/json-producer-service/src/main/java/com/ivanfranchin/jsonproducerservice/news/NewsEmitterConfig.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.jsonproducerservice.news; 2 | 3 | import org.apache.kafka.clients.admin.AdminClientConfig; 4 | import org.apache.kafka.clients.admin.NewTopic; 5 | import org.apache.kafka.clients.producer.ProducerConfig; 6 | import org.apache.kafka.common.serialization.StringSerializer; 7 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 8 | import org.springframework.context.annotation.Bean; 9 | import org.springframework.context.annotation.Configuration; 10 | import org.springframework.kafka.core.DefaultKafkaProducerFactory; 11 | import org.springframework.kafka.core.KafkaAdmin; 12 | import org.springframework.kafka.core.KafkaTemplate; 13 | import org.springframework.kafka.core.ProducerFactory; 14 | import org.springframework.kafka.support.serializer.JsonSerializer; 15 | 16 | import java.util.HashMap; 17 | import java.util.Map; 18 | 19 | @Configuration 20 | public class NewsEmitterConfig { 21 | 22 | private final KafkaProperties kafkaProperties; 23 | 24 | public NewsEmitterConfig(KafkaProperties kafkaProperties) { 25 | this.kafkaProperties = kafkaProperties; 26 | } 27 | 28 | @Bean 29 | KafkaTemplate kafkaTemplate() { 30 | return new KafkaTemplate<>(producerFactory()); 31 | } 32 | 33 | @Bean 34 | ProducerFactory producerFactory() { 35 | Map props = kafkaProperties.buildProducerProperties(null); 36 | props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); 37 | props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class); 38 | props.put(JsonSerializer.ADD_TYPE_INFO_HEADERS, false); 39 | return new DefaultKafkaProducerFactory<>(props); 40 | } 41 | 42 | // As the application will create a topic in Kafka, it is advisable to update the BOOTSTRAP_SERVERS_CONFIG 43 | // property of KafkaAdmin with the configured bootstrap server URLs. Otherwise, it will default to 'localhost:9082'. 44 | @Bean 45 | KafkaAdmin kafkaAdmin() { 46 | Map configs = new HashMap<>(); 47 | configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBootstrapServers()); 48 | return new KafkaAdmin(configs); 49 | } 50 | 51 | @Bean 52 | NewTopic newTopic() { 53 | Map producerProperties = kafkaProperties.getProducer().getProperties(); 54 | return new NewTopic(producerProperties.get("topic"), Integer.parseInt(producerProperties.get("num-partitions")), (short) 1); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /json-de-serialization/json-producer-service/src/main/java/com/ivanfranchin/jsonproducerservice/simulation/SimulationRunner.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.jsonproducerservice.simulation; 2 | 3 | import com.ivanfranchin.jsonproducerservice.news.News; 4 | import com.ivanfranchin.jsonproducerservice.news.NewsEmitter; 5 | import org.springframework.boot.CommandLineRunner; 6 | import org.springframework.stereotype.Component; 7 | 8 | import java.util.UUID; 9 | import java.util.concurrent.Executors; 10 | import java.util.concurrent.TimeUnit; 11 | 12 | @Component 13 | public class SimulationRunner implements CommandLineRunner { 14 | 15 | private final NewsEmitter newsEmitter; 16 | 17 | public SimulationRunner(NewsEmitter newsEmitter) { 18 | this.newsEmitter = newsEmitter; 19 | } 20 | 21 | @Override 22 | public void run(String... args) { 23 | Executors.newSingleThreadScheduledExecutor() 24 | .scheduleAtFixedRate(this::generateAndSendNews, 0, 3, TimeUnit.SECONDS); 25 | } 26 | 27 | private void generateAndSendNews() { 28 | newsEmitter.send(generate()); 29 | } 30 | 31 | private News generate() { 32 | return new News(UUID.randomUUID().toString(), 1, "Channel", "Title"); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /json-de-serialization/json-producer-service/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=9082 2 | 3 | spring.application.name=json-producer-service 4 | 5 | spring.kafka.bootstrap-servers=${KAFKA_HOST:localhost}:${KAFKA_PORT:29092} 6 | spring.kafka.producer.properties.topic=json-de-serialization-news 7 | spring.kafka.producer.properties.num-partitions=4 8 | -------------------------------------------------------------------------------- /json-de-serialization/json-producer-service/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _ _ _ 2 | (_)___ ___ _ __ _ __ _ __ ___ __| |_ _ ___ ___ _ __ ___ ___ _ ____ _(_) ___ ___ 3 | | / __|/ _ \| '_ \ _____| '_ \| '__/ _ \ / _` | | | |/ __/ _ \ '__|____/ __|/ _ \ '__\ \ / / |/ __/ _ \ 4 | | \__ \ (_) | | | |_____| |_) | | | (_) | (_| | |_| | (_| __/ | |_____\__ \ __/ | \ V /| | (_| __/ 5 | _/ |___/\___/|_| |_| | .__/|_| \___/ \__,_|\__,_|\___\___|_| |___/\___|_| \_/ |_|\___\___| 6 | |__/ |_| 7 | :: Spring Boot :: ${spring-boot.formatted-version} 8 | -------------------------------------------------------------------------------- /json-de-serialization/json-producer-service/src/test/java/com/ivanfranchin/jsonproducerservice/JsonProducerServiceApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.jsonproducerservice; 2 | 3 | import org.junit.jupiter.api.Disabled; 4 | import org.junit.jupiter.api.Test; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | 7 | @Disabled 8 | @SpringBootTest 9 | class JsonProducerServiceApplicationTests { 10 | 11 | @Test 12 | void contextLoads() { 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /json-de-serialization/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.ivanfranchin 7 | spring-kafka-de-serialization-types 8 | 1.0.0 9 | ../pom.xml 10 | 11 | json-de-serialization 12 | pom 13 | json-de-serialization 14 | Demo project for Spring Boot 15 | 16 | 17 | 18 | 19 | org.springframework.boot 20 | spring-boot-maven-plugin 21 | 22 | 23 | 24 | 25 | 26 | json-producer-service 27 | json-consumer-service 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Apache Maven Wrapper startup batch script, version 3.3.2 23 | # 24 | # Optional ENV vars 25 | # ----------------- 26 | # JAVA_HOME - location of a JDK home dir, required when download maven via java source 27 | # MVNW_REPOURL - repo url base for downloading maven distribution 28 | # MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven 29 | # MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output 30 | # ---------------------------------------------------------------------------- 31 | 32 | set -euf 33 | [ "${MVNW_VERBOSE-}" != debug ] || set -x 34 | 35 | # OS specific support. 36 | native_path() { printf %s\\n "$1"; } 37 | case "$(uname)" in 38 | CYGWIN* | MINGW*) 39 | [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" 40 | native_path() { cygpath --path --windows "$1"; } 41 | ;; 42 | esac 43 | 44 | # set JAVACMD and JAVACCMD 45 | set_java_home() { 46 | # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched 47 | if [ -n "${JAVA_HOME-}" ]; then 48 | if [ -x "$JAVA_HOME/jre/sh/java" ]; then 49 | # IBM's JDK on AIX uses strange locations for the executables 50 | JAVACMD="$JAVA_HOME/jre/sh/java" 51 | JAVACCMD="$JAVA_HOME/jre/sh/javac" 52 | else 53 | JAVACMD="$JAVA_HOME/bin/java" 54 | JAVACCMD="$JAVA_HOME/bin/javac" 55 | 56 | if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then 57 | echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 58 | echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 59 | return 1 60 | fi 61 | fi 62 | else 63 | JAVACMD="$( 64 | 'set' +e 65 | 'unset' -f command 2>/dev/null 66 | 'command' -v java 67 | )" || : 68 | JAVACCMD="$( 69 | 'set' +e 70 | 'unset' -f command 2>/dev/null 71 | 'command' -v javac 72 | )" || : 73 | 74 | if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then 75 | echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 76 | return 1 77 | fi 78 | fi 79 | } 80 | 81 | # hash string like Java String::hashCode 82 | hash_string() { 83 | str="${1:-}" h=0 84 | while [ -n "$str" ]; do 85 | char="${str%"${str#?}"}" 86 | h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) 87 | str="${str#?}" 88 | done 89 | printf %x\\n $h 90 | } 91 | 92 | verbose() { :; } 93 | [ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } 94 | 95 | die() { 96 | printf %s\\n "$1" >&2 97 | exit 1 98 | } 99 | 100 | trim() { 101 | # MWRAPPER-139: 102 | # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. 103 | # Needed for removing poorly interpreted newline sequences when running in more 104 | # exotic environments such as mingw bash on Windows. 105 | printf "%s" "${1}" | tr -d '[:space:]' 106 | } 107 | 108 | # parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties 109 | while IFS="=" read -r key value; do 110 | case "${key-}" in 111 | distributionUrl) distributionUrl=$(trim "${value-}") ;; 112 | distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; 113 | esac 114 | done <"${0%/*}/.mvn/wrapper/maven-wrapper.properties" 115 | [ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in ${0%/*}/.mvn/wrapper/maven-wrapper.properties" 116 | 117 | case "${distributionUrl##*/}" in 118 | maven-mvnd-*bin.*) 119 | MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ 120 | case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in 121 | *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; 122 | :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; 123 | :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; 124 | :Linux*x86_64*) distributionPlatform=linux-amd64 ;; 125 | *) 126 | echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 127 | distributionPlatform=linux-amd64 128 | ;; 129 | esac 130 | distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" 131 | ;; 132 | maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; 133 | *) MVN_CMD="mvn${0##*/mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; 134 | esac 135 | 136 | # apply MVNW_REPOURL and calculate MAVEN_HOME 137 | # maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ 138 | [ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" 139 | distributionUrlName="${distributionUrl##*/}" 140 | distributionUrlNameMain="${distributionUrlName%.*}" 141 | distributionUrlNameMain="${distributionUrlNameMain%-bin}" 142 | MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" 143 | MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" 144 | 145 | exec_maven() { 146 | unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : 147 | exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" 148 | } 149 | 150 | if [ -d "$MAVEN_HOME" ]; then 151 | verbose "found existing MAVEN_HOME at $MAVEN_HOME" 152 | exec_maven "$@" 153 | fi 154 | 155 | case "${distributionUrl-}" in 156 | *?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; 157 | *) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; 158 | esac 159 | 160 | # prepare tmp dir 161 | if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then 162 | clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } 163 | trap clean HUP INT TERM EXIT 164 | else 165 | die "cannot create temp dir" 166 | fi 167 | 168 | mkdir -p -- "${MAVEN_HOME%/*}" 169 | 170 | # Download and Install Apache Maven 171 | verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." 172 | verbose "Downloading from: $distributionUrl" 173 | verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" 174 | 175 | # select .zip or .tar.gz 176 | if ! command -v unzip >/dev/null; then 177 | distributionUrl="${distributionUrl%.zip}.tar.gz" 178 | distributionUrlName="${distributionUrl##*/}" 179 | fi 180 | 181 | # verbose opt 182 | __MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' 183 | [ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v 184 | 185 | # normalize http auth 186 | case "${MVNW_PASSWORD:+has-password}" in 187 | '') MVNW_USERNAME='' MVNW_PASSWORD='' ;; 188 | has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; 189 | esac 190 | 191 | if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then 192 | verbose "Found wget ... using wget" 193 | wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" 194 | elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then 195 | verbose "Found curl ... using curl" 196 | curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" 197 | elif set_java_home; then 198 | verbose "Falling back to use Java to download" 199 | javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" 200 | targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" 201 | cat >"$javaSource" <<-END 202 | public class Downloader extends java.net.Authenticator 203 | { 204 | protected java.net.PasswordAuthentication getPasswordAuthentication() 205 | { 206 | return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); 207 | } 208 | public static void main( String[] args ) throws Exception 209 | { 210 | setDefault( new Downloader() ); 211 | java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); 212 | } 213 | } 214 | END 215 | # For Cygwin/MinGW, switch paths to Windows format before running javac and java 216 | verbose " - Compiling Downloader.java ..." 217 | "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" 218 | verbose " - Running Downloader.java ..." 219 | "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" 220 | fi 221 | 222 | # If specified, validate the SHA-256 sum of the Maven distribution zip file 223 | if [ -n "${distributionSha256Sum-}" ]; then 224 | distributionSha256Result=false 225 | if [ "$MVN_CMD" = mvnd.sh ]; then 226 | echo "Checksum validation is not supported for maven-mvnd." >&2 227 | echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 228 | exit 1 229 | elif command -v sha256sum >/dev/null; then 230 | if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c >/dev/null 2>&1; then 231 | distributionSha256Result=true 232 | fi 233 | elif command -v shasum >/dev/null; then 234 | if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then 235 | distributionSha256Result=true 236 | fi 237 | else 238 | echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 239 | echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 240 | exit 1 241 | fi 242 | if [ $distributionSha256Result = false ]; then 243 | echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 244 | echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 245 | exit 1 246 | fi 247 | fi 248 | 249 | # unzip and move 250 | if command -v unzip >/dev/null; then 251 | unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" 252 | else 253 | tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" 254 | fi 255 | printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/mvnw.url" 256 | mv -- "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" 257 | 258 | clean || : 259 | exec_maven "$@" 260 | -------------------------------------------------------------------------------- /mvnw.cmd: -------------------------------------------------------------------------------- 1 | <# : batch portion 2 | @REM ---------------------------------------------------------------------------- 3 | @REM Licensed to the Apache Software Foundation (ASF) under one 4 | @REM or more contributor license agreements. See the NOTICE file 5 | @REM distributed with this work for additional information 6 | @REM regarding copyright ownership. The ASF licenses this file 7 | @REM to you under the Apache License, Version 2.0 (the 8 | @REM "License"); you may not use this file except in compliance 9 | @REM with the License. You may obtain a copy of the License at 10 | @REM 11 | @REM http://www.apache.org/licenses/LICENSE-2.0 12 | @REM 13 | @REM Unless required by applicable law or agreed to in writing, 14 | @REM software distributed under the License is distributed on an 15 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | @REM KIND, either express or implied. See the License for the 17 | @REM specific language governing permissions and limitations 18 | @REM under the License. 19 | @REM ---------------------------------------------------------------------------- 20 | 21 | @REM ---------------------------------------------------------------------------- 22 | @REM Apache Maven Wrapper startup batch script, version 3.3.2 23 | @REM 24 | @REM Optional ENV vars 25 | @REM MVNW_REPOURL - repo url base for downloading maven distribution 26 | @REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven 27 | @REM MVNW_VERBOSE - true: enable verbose log; others: silence the output 28 | @REM ---------------------------------------------------------------------------- 29 | 30 | @IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) 31 | @SET __MVNW_CMD__= 32 | @SET __MVNW_ERROR__= 33 | @SET __MVNW_PSMODULEP_SAVE=%PSModulePath% 34 | @SET PSModulePath= 35 | @FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( 36 | IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) 37 | ) 38 | @SET PSModulePath=%__MVNW_PSMODULEP_SAVE% 39 | @SET __MVNW_PSMODULEP_SAVE= 40 | @SET __MVNW_ARG0_NAME__= 41 | @SET MVNW_USERNAME= 42 | @SET MVNW_PASSWORD= 43 | @IF NOT "%__MVNW_CMD__%"=="" (%__MVNW_CMD__% %*) 44 | @echo Cannot start maven from wrapper >&2 && exit /b 1 45 | @GOTO :EOF 46 | : end batch / begin powershell #> 47 | 48 | $ErrorActionPreference = "Stop" 49 | if ($env:MVNW_VERBOSE -eq "true") { 50 | $VerbosePreference = "Continue" 51 | } 52 | 53 | # calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties 54 | $distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl 55 | if (!$distributionUrl) { 56 | Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" 57 | } 58 | 59 | switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { 60 | "maven-mvnd-*" { 61 | $USE_MVND = $true 62 | $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" 63 | $MVN_CMD = "mvnd.cmd" 64 | break 65 | } 66 | default { 67 | $USE_MVND = $false 68 | $MVN_CMD = $script -replace '^mvnw','mvn' 69 | break 70 | } 71 | } 72 | 73 | # apply MVNW_REPOURL and calculate MAVEN_HOME 74 | # maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ 75 | if ($env:MVNW_REPOURL) { 76 | $MVNW_REPO_PATTERN = if ($USE_MVND) { "/org/apache/maven/" } else { "/maven/mvnd/" } 77 | $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace '^.*'+$MVNW_REPO_PATTERN,'')" 78 | } 79 | $distributionUrlName = $distributionUrl -replace '^.*/','' 80 | $distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' 81 | $MAVEN_HOME_PARENT = "$HOME/.m2/wrapper/dists/$distributionUrlNameMain" 82 | if ($env:MAVEN_USER_HOME) { 83 | $MAVEN_HOME_PARENT = "$env:MAVEN_USER_HOME/wrapper/dists/$distributionUrlNameMain" 84 | } 85 | $MAVEN_HOME_NAME = ([System.Security.Cryptography.MD5]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' 86 | $MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" 87 | 88 | if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { 89 | Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" 90 | Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" 91 | exit $? 92 | } 93 | 94 | if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { 95 | Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" 96 | } 97 | 98 | # prepare tmp dir 99 | $TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile 100 | $TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" 101 | $TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null 102 | trap { 103 | if ($TMP_DOWNLOAD_DIR.Exists) { 104 | try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } 105 | catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } 106 | } 107 | } 108 | 109 | New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null 110 | 111 | # Download and Install Apache Maven 112 | Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." 113 | Write-Verbose "Downloading from: $distributionUrl" 114 | Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" 115 | 116 | $webclient = New-Object System.Net.WebClient 117 | if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { 118 | $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) 119 | } 120 | [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 121 | $webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null 122 | 123 | # If specified, validate the SHA-256 sum of the Maven distribution zip file 124 | $distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum 125 | if ($distributionSha256Sum) { 126 | if ($USE_MVND) { 127 | Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." 128 | } 129 | Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash 130 | if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { 131 | Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." 132 | } 133 | } 134 | 135 | # unzip and move 136 | Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null 137 | Rename-Item -Path "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" -NewName $MAVEN_HOME_NAME | Out-Null 138 | try { 139 | Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null 140 | } catch { 141 | if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { 142 | Write-Error "fail to move MAVEN_HOME" 143 | } 144 | } finally { 145 | try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } 146 | catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } 147 | } 148 | 149 | Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" 150 | -------------------------------------------------------------------------------- /my-functions.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # -- check_script_input_parameter -- 4 | # $1: input parameter 5 | function check_script_input_parameter() { 6 | if [ "$1" != "all" ] && 7 | [ "$1" != "string-de-serialization" ] && 8 | [ "$1" != "string-producer-service" ] && 9 | [ "$1" != "string-consumer-service" ] && 10 | [ "$1" != "json-de-serialization" ] && 11 | [ "$1" != "json-producer-service" ] && 12 | [ "$1" != "json-consumer-service" ] && 13 | [ "$1" != "avro-de-serialization" ] && 14 | [ "$1" != "avro-producer-service" ] && 15 | [ "$1" != "avro-consumer-service" ] && 16 | [ "$1" != "avro-2-de-serialization" ] && 17 | [ "$1" != "avro-2-producer-service" ] && 18 | [ "$1" != "avro-2-consumer-service" ] && 19 | [ "$1" != "avro-3-de-serialization" ] && 20 | [ "$1" != "avro-3-producer-service" ] && 21 | [ "$1" != "avro-3-consumer-service" ]; 22 | then 23 | printf "Invalid application name or sample provided!" 24 | printf "\nValid Parameters:" 25 | 26 | printf "\n\tall" 27 | 28 | printf "\n\tstring-de-serialization" 29 | printf "\n\t\tstring-producer-service" 30 | printf "\n\t\tstring-consumer-service" 31 | 32 | printf "\n\tjson-de-serialization" 33 | printf "\n\t\tjson-producer-service" 34 | printf "\n\t\tjson-consumer-service" 35 | 36 | printf "\n\tavro-de-serialization" 37 | printf "\n\t\tavro-producer-service" 38 | printf "\n\t\tavro-consumer-service" 39 | 40 | printf "\n\tavro-2-de-serialization" 41 | printf "\n\t\tavro-2-producer-service" 42 | printf "\n\t\tavro-2-consumer-service" 43 | 44 | printf "\n\tavro-3-de-serialization" 45 | printf "\n\t\tavro-3-producer-service" 46 | printf "\n\t\tavro-3-consumer-service" 47 | 48 | printf "\n" 49 | exit 1 50 | fi 51 | } -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework.boot 7 | spring-boot-starter-parent 8 | 3.4.0 9 | 10 | 11 | com.ivanfranchin 12 | spring-kafka-de-serialization-types 13 | 1.0.0 14 | pom 15 | spring-kafka-de-serialization-types 16 | Demo project for Spring Boot 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 21 32 | 3.4.4 33 | 34 | 35 | 36 | 37 | 38 | com.google.cloud.tools 39 | jib-maven-plugin 40 | ${jib-maven-plugin.version} 41 | 42 | 43 | 44 | 45 | 46 | string-de-serialization 47 | json-de-serialization 48 | avro-de-serialization 49 | avro-2-de-serialization 50 | avro-3-de-serialization 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /remove-docker-images.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source my-functions.sh 4 | 5 | check_script_input_parameter $1 6 | 7 | if [ "$1" = "string-producer-service" ] || 8 | [ "$1" = "string-de-serialization" ] || 9 | [ "$1" = "all" ]; 10 | then 11 | 12 | docker rmi ivanfranchin/string-producer-service:1.0.0 13 | 14 | fi 15 | 16 | if [ "$1" = "string-consumer-service" ] || 17 | [ "$1" = "string-de-serialization" ] || 18 | [ "$1" = "all" ]; 19 | then 20 | 21 | docker rmi ivanfranchin/string-consumer-service:1.0.0 22 | 23 | fi 24 | 25 | if [ "$1" = "json-producer-service" ] || 26 | [ "$1" = "json-de-serialization" ] || 27 | [ "$1" = "all" ]; 28 | then 29 | 30 | docker rmi ivanfranchin/json-producer-service:1.0.0 31 | 32 | fi 33 | 34 | if [ "$1" = "json-consumer-service" ] || 35 | [ "$1" = "json-de-serialization" ] || 36 | [ "$1" = "all" ]; 37 | then 38 | 39 | docker rmi ivanfranchin/json-consumer-service:1.0.0 40 | 41 | fi 42 | 43 | if [ "$1" = "avro-producer-service" ] || 44 | [ "$1" = "avro-de-serialization" ] || 45 | [ "$1" = "all" ]; 46 | then 47 | 48 | docker rmi ivanfranchin/avro-producer-service:1.0.0 49 | 50 | fi 51 | 52 | if [ "$1" = "avro-consumer-service" ] || 53 | [ "$1" = "avro-de-serialization" ] || 54 | [ "$1" = "all" ]; 55 | then 56 | 57 | docker rmi ivanfranchin/avro-consumer-service:1.0.0 58 | 59 | fi 60 | 61 | if [ "$1" = "avro-2-producer-service" ] || 62 | [ "$1" = "avro-2-de-serialization" ] || 63 | [ "$1" = "all" ]; 64 | then 65 | 66 | docker rmi ivanfranchin/avro-2-producer-service:1.0.0 67 | 68 | fi 69 | 70 | if [ "$1" = "avro-2-consumer-service" ] || 71 | [ "$1" = "avro-2-de-serialization" ] || 72 | [ "$1" = "all" ]; 73 | then 74 | 75 | docker rmi ivanfranchin/avro-2-consumer-service:1.0.0 76 | 77 | fi 78 | 79 | if [ "$1" = "avro-3-producer-service" ] || 80 | [ "$1" = "avro-3-de-serialization" ] || 81 | [ "$1" = "all" ]; 82 | then 83 | 84 | docker rmi ivanfranchin/avro-3-producer-service:1.0.0 85 | 86 | fi 87 | 88 | if [ "$1" = "avro-3-consumer-service" ] || 89 | [ "$1" = "avro-3-de-serialization" ] || 90 | [ "$1" = "all" ]; 91 | then 92 | 93 | docker rmi ivanfranchin/avro-3-consumer-service:1.0.0 94 | 95 | fi 96 | -------------------------------------------------------------------------------- /string-de-serialization/README.md: -------------------------------------------------------------------------------- 1 | # spring-kafka-de-serialization-types 2 | ## `> string-de-serialization` 3 | 4 | ![string-de-serialization](../documentation/string-de-serialization.jpeg) 5 | 6 | This sample demonstrates a **producer** that pushes `News` messages to a topic in `Kafka` and a **consumer** that listens those messages from `Kafka`: 7 | - **Producer** serializes the message `key` and `value` using `StringSerializer`; 8 | - **Consumer** deserializes the message `key` and `value` using `StringDeserializer`; 9 | - **Producer** creates the Kafka topics and **Consumer** doesn't. 10 | 11 | ## Start Environment 12 | 13 | Before starting producer and consumer, the services present in `docker-compose.yml` file must be up and running as explained in [Start Environment](https://github.com/ivangfr/spring-kafka-de-serialization-types#start-environment) section of the main README. 14 | 15 | ## Running applications using Maven 16 | 17 | > **Note**: run `string-producer-service` first so that it can create the `Kafka` topics. 18 | 19 | - **string-producer-service** 20 | 21 | - Open a terminal and navigate to the `spring-kafka-de-serialization-types` root folder; 22 | - Run application: 23 | ``` 24 | ./mvnw clean spring-boot:run --projects string-de-serialization/string-producer-service 25 | ``` 26 | - As soon as the producer is up and running, it will start pushing automatically and randomly `News` messages to `Kafka` topic `string-de-serialization-news`. The default `interval` between messages is `3 seconds`. 27 | 28 | - **string-consumer-service** 29 | 30 | - Open another terminal and make sure you are in `spring-kafka-de-serialization-types` root folder; 31 | - Run application: 32 | ``` 33 | ./mvnw clean spring-boot:run --projects string-de-serialization/string-consumer-service 34 | ``` 35 | - Once the consumer is up and running, it will start listening `News` messages from the `Kafka` topic `string-de-serialization-news`. 36 | 37 | ## Running applications as Docker containers 38 | 39 | - ### Build Docker images 40 | 41 | In a terminal and, inside the `spring-kafka-de-serialization-types` root folder, run: 42 | ``` 43 | ./build-docker-images.sh string-de-serialization 44 | ``` 45 | 46 | - ### Environment variables 47 | 48 | **string-producer-service** and **string-consumer-service** 49 | 50 | | Environment Variable | Description | 51 | |----------------------|-------------------------------------------------------------------------| 52 | | `KAFKA_HOST` | Specify host of the `Kafka` message broker to use (default `localhost`) | 53 | | `KAFKA_PORT` | Specify port of the `Kafka` message broker to use (default `29092`) | 54 | 55 | - ### Run Docker containers 56 | 57 | > **Note**: run `string-producer-service` first so that it can create the `Kafka` topics. 58 | 59 | - **string-producer-service** 60 | 61 | In a terminal, run the following Docker command: 62 | ``` 63 | docker run --rm --name string-producer-service -p 9080:9080 \ 64 | -e KAFKA_HOST=kafka -e KAFKA_PORT=9092 \ 65 | --network=spring-kafka-de-serialization-types_default \ 66 | ivanfranchin/string-producer-service:1.0.0 67 | ``` 68 | 69 | - **string-consumer-service** 70 | 71 | In another terminal, run the Docker command below: 72 | ``` 73 | docker run --rm --name string-consumer-service -p 9081:9081 \ 74 | -e KAFKA_HOST=kafka -e KAFKA_PORT=9092 \ 75 | --network=spring-kafka-de-serialization-types_default \ 76 | ivanfranchin/string-consumer-service:1.0.0 77 | ``` 78 | 79 | ## Shutdown 80 | 81 | - Go to the terminals where the applications are running and press `Ctrl+C`; 82 | - Stop the services present in `docker-compose.yml` as explained in [Shutdown](https://github.com/ivangfr/spring-kafka-de-serialization-types#shutdown) section of the main README. 83 | 84 | ## Cleanup 85 | 86 | To remove the Docker images created by this example, go to a terminal and, inside the `spring-kafka-de-serialization-types` root folder, run the following script: 87 | ``` 88 | ./remove-docker-images.sh string-de-serialization 89 | ``` 90 | -------------------------------------------------------------------------------- /string-de-serialization/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.ivanfranchin 7 | spring-kafka-de-serialization-types 8 | 1.0.0 9 | ../pom.xml 10 | 11 | string-de-serialization 12 | pom 13 | string-de-serialization 14 | Demo project for Spring Boot 15 | 16 | 17 | 18 | 19 | org.springframework.boot 20 | spring-boot-maven-plugin 21 | 22 | 23 | 24 | 25 | 26 | string-producer-service 27 | string-consumer-service 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /string-de-serialization/string-consumer-service/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.ivanfranchin 7 | string-de-serialization 8 | 1.0.0 9 | ../pom.xml 10 | 11 | string-consumer-service 12 | string-consumer-service 13 | Demo project for Spring Boot 14 | 15 | 16 | org.springframework.boot 17 | spring-boot-starter-web 18 | 19 | 20 | org.springframework.kafka 21 | spring-kafka 22 | 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-starter-test 27 | test 28 | 29 | 30 | org.springframework.kafka 31 | spring-kafka-test 32 | test 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /string-de-serialization/string-consumer-service/src/main/java/com/ivanfranchin/stringconsumerservice/StringConsumerServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.stringconsumerservice; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class StringConsumerServiceApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(StringConsumerServiceApplication.class, args); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /string-de-serialization/string-consumer-service/src/main/java/com/ivanfranchin/stringconsumerservice/news/NewsListener.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.stringconsumerservice.news; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.kafka.annotation.KafkaListener; 6 | import org.springframework.kafka.listener.adapter.ConsumerRecordMetadata; 7 | import org.springframework.messaging.handler.annotation.Payload; 8 | import org.springframework.stereotype.Service; 9 | 10 | @Service 11 | public class NewsListener { 12 | 13 | private static final Logger log = LoggerFactory.getLogger(NewsListener.class); 14 | 15 | @KafkaListener(topics = "${spring.kafka.consumer.topic}", groupId = "${spring.kafka.consumer.group-id}") 16 | public void listen(@Payload String news, ConsumerRecordMetadata metadata) { 17 | log.info("Received message\n---\nTOPIC: {}; PARTITION: {}; OFFSET: {};\nPAYLOAD: {}\n---", 18 | metadata.topic(), metadata.partition(), metadata.offset(), news); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /string-de-serialization/string-consumer-service/src/main/java/com/ivanfranchin/stringconsumerservice/news/NewsListenerConfig.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.stringconsumerservice.news; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerConfig; 4 | import org.apache.kafka.common.serialization.StringDeserializer; 5 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 6 | import org.springframework.context.annotation.Bean; 7 | import org.springframework.context.annotation.Configuration; 8 | import org.springframework.kafka.annotation.EnableKafka; 9 | import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; 10 | import org.springframework.kafka.core.ConsumerFactory; 11 | import org.springframework.kafka.core.DefaultKafkaConsumerFactory; 12 | 13 | import java.util.Map; 14 | 15 | @EnableKafka 16 | @Configuration 17 | public class NewsListenerConfig { 18 | 19 | private final KafkaProperties kafkaProperties; 20 | 21 | public NewsListenerConfig(KafkaProperties kafkaProperties) { 22 | this.kafkaProperties = kafkaProperties; 23 | } 24 | 25 | @Bean 26 | ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory() { 27 | ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); 28 | factory.setConsumerFactory(consumerFactory()); 29 | factory.setConcurrency(kafkaProperties.getListener().getConcurrency()); 30 | return factory; 31 | } 32 | 33 | @Bean 34 | ConsumerFactory consumerFactory() { 35 | Map props = kafkaProperties.buildConsumerProperties(null); 36 | props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); 37 | props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); 38 | props.put(ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, false); 39 | return new DefaultKafkaConsumerFactory<>(props); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /string-de-serialization/string-consumer-service/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=9081 2 | 3 | spring.application.name=string-consumer-service 4 | 5 | spring.kafka.bootstrap-servers=${KAFKA_HOST:localhost}:${KAFKA_PORT:29092} 6 | spring.kafka.consumer.group-id=string-consumer-service-group 7 | spring.kafka.consumer.topic=string-de-serialization-news 8 | spring.kafka.consumer.auto-offset-reset=earliest 9 | spring.kafka.listener.concurrency=2 -------------------------------------------------------------------------------- /string-de-serialization/string-consumer-service/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _ _ _ 2 | ___| |_ _ __(_)_ __ __ _ ___ ___ _ __ ___ _ _ _ __ ___ ___ _ __ ___ ___ _ ____ _(_) ___ ___ 3 | / __| __| '__| | '_ \ / _` |_____ / __/ _ \| '_ \/ __| | | | '_ ` _ \ / _ \ '__|____/ __|/ _ \ '__\ \ / / |/ __/ _ \ 4 | \__ \ |_| | | | | | | (_| |_____| (_| (_) | | | \__ \ |_| | | | | | | __/ | |_____\__ \ __/ | \ V /| | (_| __/ 5 | |___/\__|_| |_|_| |_|\__, | \___\___/|_| |_|___/\__,_|_| |_| |_|\___|_| |___/\___|_| \_/ |_|\___\___| 6 | |___/ 7 | :: Spring Boot :: ${spring-boot.formatted-version} 8 | -------------------------------------------------------------------------------- /string-de-serialization/string-consumer-service/src/test/java/com/ivanfranchin/jsonconsumerservice/StringConsumerServiceApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.jsonconsumerservice; 2 | 3 | import org.junit.jupiter.api.Disabled; 4 | import org.junit.jupiter.api.Test; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | 7 | @Disabled 8 | @SpringBootTest 9 | class StringConsumerServiceApplicationTests { 10 | 11 | @Test 12 | void contextLoads() { 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /string-de-serialization/string-producer-service/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.ivanfranchin 7 | string-de-serialization 8 | 1.0.0 9 | ../pom.xml 10 | 11 | string-producer-service 12 | string-producer-service 13 | Demo project for Spring Boot 14 | 15 | 16 | org.springframework.boot 17 | spring-boot-starter-web 18 | 19 | 20 | org.springframework.kafka 21 | spring-kafka 22 | 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-starter-test 27 | test 28 | 29 | 30 | org.springframework.kafka 31 | spring-kafka-test 32 | test 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /string-de-serialization/string-producer-service/src/main/java/com/ivanfranchin/stringproducerservice/StringProducerServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.stringproducerservice; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class StringProducerServiceApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(StringProducerServiceApplication.class, args); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /string-de-serialization/string-producer-service/src/main/java/com/ivanfranchin/stringproducerservice/news/News.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.stringproducerservice.news; 2 | 3 | public record News(String id, Integer fromId, String fromName, String title) { 4 | 5 | @Override 6 | public String toString() { 7 | return String.format("id:%s;fromId:%s;fromName:%s;title:%s", id, fromId, fromName, title); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /string-de-serialization/string-producer-service/src/main/java/com/ivanfranchin/stringproducerservice/news/NewsEmitter.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.stringproducerservice.news; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 6 | import org.springframework.kafka.core.KafkaTemplate; 7 | import org.springframework.stereotype.Component; 8 | 9 | @Component 10 | public class NewsEmitter { 11 | 12 | private static final Logger log = LoggerFactory.getLogger(NewsEmitter.class); 13 | 14 | private final KafkaTemplate kafkaTemplate; 15 | private final KafkaProperties kafkaProperties; 16 | 17 | public NewsEmitter(KafkaTemplate kafkaTemplate, KafkaProperties kafkaProperties) { 18 | this.kafkaTemplate = kafkaTemplate; 19 | this.kafkaProperties = kafkaProperties; 20 | } 21 | 22 | public void send(News news) { 23 | String kafkaTopic = kafkaProperties.getProducer().getProperties().get("topic"); 24 | log.info("Sending News '{}' to topic '{}'", news, kafkaTopic); 25 | kafkaTemplate.send(kafkaTopic, news.id(), news.toString()); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /string-de-serialization/string-producer-service/src/main/java/com/ivanfranchin/stringproducerservice/news/NewsEmitterConfig.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.stringproducerservice.news; 2 | 3 | import org.apache.kafka.clients.admin.AdminClientConfig; 4 | import org.apache.kafka.clients.admin.NewTopic; 5 | import org.apache.kafka.clients.producer.ProducerConfig; 6 | import org.apache.kafka.common.serialization.StringSerializer; 7 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 8 | import org.springframework.context.annotation.Bean; 9 | import org.springframework.context.annotation.Configuration; 10 | import org.springframework.kafka.core.DefaultKafkaProducerFactory; 11 | import org.springframework.kafka.core.KafkaAdmin; 12 | import org.springframework.kafka.core.KafkaTemplate; 13 | import org.springframework.kafka.core.ProducerFactory; 14 | 15 | import java.util.HashMap; 16 | import java.util.Map; 17 | 18 | @Configuration 19 | public class NewsEmitterConfig { 20 | 21 | private final KafkaProperties kafkaProperties; 22 | 23 | public NewsEmitterConfig(KafkaProperties kafkaProperties) { 24 | this.kafkaProperties = kafkaProperties; 25 | } 26 | 27 | @Bean 28 | KafkaTemplate kafkaTemplate() { 29 | return new KafkaTemplate<>(producerFactory()); 30 | } 31 | 32 | @Bean 33 | ProducerFactory producerFactory() { 34 | Map props = new HashMap<>(kafkaProperties.buildProducerProperties(null)); 35 | props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); 36 | props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); 37 | return new DefaultKafkaProducerFactory<>(props); 38 | } 39 | 40 | // As the application will create a topic in Kafka, it is advisable to update the BOOTSTRAP_SERVERS_CONFIG 41 | // property of KafkaAdmin with the configured bootstrap server URLs. Otherwise, it will default to 'localhost:9082'. 42 | @Bean 43 | KafkaAdmin kafkaAdmin() { 44 | Map configs = new HashMap<>(); 45 | configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBootstrapServers()); 46 | return new KafkaAdmin(configs); 47 | } 48 | 49 | @Bean 50 | NewTopic newTopic() { 51 | Map producerProperties = kafkaProperties.getProducer().getProperties(); 52 | return new NewTopic(producerProperties.get("topic"), Integer.parseInt(producerProperties.get("num-partitions")), (short) 1); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /string-de-serialization/string-producer-service/src/main/java/com/ivanfranchin/stringproducerservice/simulation/SimulationRunner.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.stringproducerservice.simulation; 2 | 3 | import com.ivanfranchin.stringproducerservice.news.News; 4 | import com.ivanfranchin.stringproducerservice.news.NewsEmitter; 5 | import org.springframework.boot.CommandLineRunner; 6 | import org.springframework.stereotype.Component; 7 | 8 | import java.util.UUID; 9 | import java.util.concurrent.Executors; 10 | import java.util.concurrent.TimeUnit; 11 | 12 | @Component 13 | public class SimulationRunner implements CommandLineRunner { 14 | 15 | private final NewsEmitter newsEmitter; 16 | 17 | public SimulationRunner(NewsEmitter newsEmitter) { 18 | this.newsEmitter = newsEmitter; 19 | } 20 | 21 | @Override 22 | public void run(String... args) { 23 | Executors.newSingleThreadScheduledExecutor() 24 | .scheduleAtFixedRate(this::generateAndSendNews, 0, 3, TimeUnit.SECONDS); 25 | } 26 | 27 | private void generateAndSendNews() { 28 | newsEmitter.send(generate()); 29 | } 30 | 31 | private News generate() { 32 | return new News(UUID.randomUUID().toString(), 1, "Channel", "Title"); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /string-de-serialization/string-producer-service/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=9080 2 | 3 | spring.application.name=string-producer-service 4 | 5 | spring.kafka.bootstrap-servers=${KAFKA_HOST:localhost}:${KAFKA_PORT:29092} 6 | spring.kafka.producer.properties.topic=string-de-serialization-news 7 | spring.kafka.producer.properties.num-partitions=4 8 | -------------------------------------------------------------------------------- /string-de-serialization/string-producer-service/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | _ _ _ _ 2 | ___| |_ _ __(_)_ __ __ _ _ __ _ __ ___ __| |_ _ ___ ___ _ __ ___ ___ _ ____ _(_) ___ ___ 3 | / __| __| '__| | '_ \ / _` |_____| '_ \| '__/ _ \ / _` | | | |/ __/ _ \ '__|____/ __|/ _ \ '__\ \ / / |/ __/ _ \ 4 | \__ \ |_| | | | | | | (_| |_____| |_) | | | (_) | (_| | |_| | (_| __/ | |_____\__ \ __/ | \ V /| | (_| __/ 5 | |___/\__|_| |_|_| |_|\__, | | .__/|_| \___/ \__,_|\__,_|\___\___|_| |___/\___|_| \_/ |_|\___\___| 6 | |___/ |_| 7 | :: Spring Boot :: ${spring-boot.formatted-version} 8 | -------------------------------------------------------------------------------- /string-de-serialization/string-producer-service/src/test/java/com/ivanfranchin/jsonproducerservice/StringProducerServiceApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.ivanfranchin.jsonproducerservice; 2 | 3 | import org.junit.jupiter.api.Disabled; 4 | import org.junit.jupiter.api.Test; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | 7 | @Disabled 8 | @SpringBootTest 9 | class StringProducerServiceApplicationTests { 10 | 11 | @Test 12 | void contextLoads() { 13 | } 14 | } 15 | --------------------------------------------------------------------------------