├── .github
    └── dependabot.yml
├── .gitignore
├── README.md
├── pom.xml
└── src
    └── main
        ├── java
            └── com
            │   └── codingharbour
            │       └── kafka
            │           └── protobuf
            │               ├── consumer
            │                   ├── GenericProtobufConsumer.java
            │                   └── ProtobufConsumer.java
            │               └── producer
            │                   └── ProtobufProducer.java
        └── protobuf
            └── SimpleMessage.proto


/.github/dependabot.yml:
--------------------------------------------------------------------------------
 1 | # To get started with Dependabot version updates, you'll need to specify which
 2 | # package ecosystems to update and where the package manifests are located.
 3 | # Please see the documentation for all configuration options:
 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
 5 | 
 6 | version: 2
 7 | updates:
 8 |   - package-ecosystem: "maven" # See documentation for possible values
 9 |     directory: "/" # Location of package manifests
10 |     schedule:
11 |       interval: "weekly"
12 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /.idea/*
2 | /target/*
3 | *.iml
4 | 
5 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # kafka-protobuf
2 | 
3 | > Read the entire tutorial: https://codingharbour.com/apache-kafka/how-to-use-protobuf-with-apache-kafka-and-schema-registry/
4 | 
5 | Examples of working with Protobuf (de)serialization
6 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <project xmlns="http://maven.apache.org/POM/4.0.0"
 3 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 4 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 5 |     <modelVersion>4.0.0</modelVersion>
 6 | 
 7 |     <groupId>com.codingharbour</groupId>
 8 |     <artifactId>kafka-protobuf</artifactId>
 9 |     <version>1.0-SNAPSHOT</version>
10 | 
11 |     <dependencies>
12 |         <dependency>
13 |             <groupId>com.google.protobuf</groupId>
14 |             <artifactId>protobuf-java</artifactId>
15 |             <version>3.25.2</version>
16 |         </dependency>
17 |         <dependency>
18 |             <groupId>org.apache.kafka</groupId>
19 |             <artifactId>kafka-clients</artifactId>
20 |             <version>3.6.0</version>
21 |         </dependency>
22 |         <dependency>
23 |             <groupId>io.confluent</groupId>
24 |             <artifactId>kafka-protobuf-serializer</artifactId>
25 |             <version>7.5.2</version>
26 |         </dependency>
27 |     </dependencies>
28 | 
29 |     <repositories>
30 |         <repository>
31 |             <id>confluent</id>
32 |             <name>confluent</name>
33 |             <url>https://packages.confluent.io/maven/</url>
34 |         </repository>
35 |     </repositories>
36 | 
37 |     <build>
38 |         <plugins>
39 |             <plugin>
40 |                 <groupId>org.apache.maven.plugins</groupId>
41 |                 <artifactId>maven-compiler-plugin</artifactId>
42 |                 <configuration>
43 |                     <source>1.8</source>
44 |                     <target>1.8</target>
45 |                 </configuration>
46 |             </plugin>
47 |             <plugin>
48 |                 <groupId>com.github.os72</groupId>
49 |                 <artifactId>protoc-jar-maven-plugin</artifactId>
50 |                 <version>3.11.4</version>
51 |                 <executions>
52 |                     <execution>
53 |                         <phase>generate-sources</phase>
54 |                         <goals>
55 |                             <goal>run</goal>
56 |                         </goals>
57 |                         <configuration>
58 |                             <inputDirectories>
59 |                                 <include>${project.basedir}/src/main/protobuf</include>
60 |                             </inputDirectories>
61 |                             <outputTargets>
62 |                                 <outputTarget>
63 |                                     <type>java</type>
64 |                                     <addSources>main</addSources>
65 |                                     <outputDirectory>${project.basedir}/target/generated-sources/protobuf</outputDirectory>
66 |                                 </outputTarget>
67 |                             </outputTargets>
68 |                         </configuration>
69 |                     </execution>
70 |                 </executions>
71 |             </plugin>
72 |         </plugins>
73 |     </build>
74 | 
75 | </project>


--------------------------------------------------------------------------------
/src/main/java/com/codingharbour/kafka/protobuf/consumer/GenericProtobufConsumer.java:
--------------------------------------------------------------------------------
 1 | package com.codingharbour.kafka.protobuf.consumer;
 2 | 
 3 | import com.google.protobuf.Descriptors.FieldDescriptor;
 4 | import com.google.protobuf.DynamicMessage;
 5 | import io.confluent.kafka.serializers.protobuf.KafkaProtobufDeserializer;
 6 | import io.confluent.kafka.serializers.protobuf.KafkaProtobufDeserializerConfig;
 7 | import org.apache.kafka.clients.consumer.ConsumerConfig;
 8 | import org.apache.kafka.clients.consumer.ConsumerRecord;
 9 | import org.apache.kafka.clients.consumer.ConsumerRecords;
10 | import org.apache.kafka.clients.consumer.KafkaConsumer;
11 | import org.apache.kafka.clients.producer.ProducerConfig;
12 | import org.apache.kafka.common.serialization.StringDeserializer;
13 | 
14 | import java.time.Duration;
15 | import java.util.Collections;
16 | import java.util.Properties;
17 | 
18 | public class GenericProtobufConsumer {
19 |     public static void main(String[] args) {
20 |         GenericProtobufConsumer protobufConsumer = new GenericProtobufConsumer();
21 |         protobufConsumer.readMessages();
22 |     }
23 | 
24 |     public void readMessages() {
25 |         Properties properties = new Properties();
26 |         properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
27 |         properties.put(ConsumerConfig.GROUP_ID_CONFIG, "generic-protobuf-consumer-group");
28 |         properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
29 |         properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
30 | 
31 |         properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
32 |         properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, KafkaProtobufDeserializer.class);
33 |         properties.put(KafkaProtobufDeserializerConfig.SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081");
34 | 
35 |         KafkaConsumer<String, DynamicMessage> consumer = new KafkaConsumer<>(properties);
36 | 
37 |         consumer.subscribe(Collections.singleton("protobuf-topic"));
38 | 
39 |         //poll the record from the topic
40 |         while (true) {
41 |             ConsumerRecords<String, DynamicMessage> records = consumer.poll(Duration.ofMillis(100));
42 | 
43 |             for (ConsumerRecord<String, DynamicMessage> record : records) {
44 |                 for (FieldDescriptor field : record.value().getAllFields().keySet()) {
45 |                     System.out.println(field.getName() + ": " + record.value().getField(field));
46 |                 }
47 |             }
48 |             consumer.commitAsync();
49 |         }
50 |     }
51 | }
52 | 


--------------------------------------------------------------------------------
/src/main/java/com/codingharbour/kafka/protobuf/consumer/ProtobufConsumer.java:
--------------------------------------------------------------------------------
 1 | package com.codingharbour.kafka.protobuf.consumer;
 2 | 
 3 | import io.confluent.kafka.serializers.protobuf.KafkaProtobufDeserializer;
 4 | import io.confluent.kafka.serializers.protobuf.KafkaProtobufDeserializerConfig;
 5 | import org.apache.kafka.clients.consumer.ConsumerConfig;
 6 | import org.apache.kafka.clients.consumer.ConsumerRecord;
 7 | import org.apache.kafka.clients.consumer.ConsumerRecords;
 8 | import org.apache.kafka.clients.consumer.KafkaConsumer;
 9 | import org.apache.kafka.clients.producer.ProducerConfig;
10 | import org.apache.kafka.common.serialization.StringDeserializer;
11 | 
12 | import java.time.Duration;
13 | import java.util.Collections;
14 | import java.util.Properties;
15 | 
16 | import static com.codingharbour.protobuf.SimpleMessageProtos.SimpleMessage;
17 | 
18 | public class ProtobufConsumer {
19 | 
20 |     public static void main(String[] args) {
21 |         ProtobufConsumer protobufConsumer = new ProtobufConsumer();
22 |         protobufConsumer.readMessages();
23 |     }
24 | 
25 |     public void readMessages() {
26 |         Properties properties = new Properties();
27 |         properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
28 |         properties.put(ConsumerConfig.GROUP_ID_CONFIG, "protobuf-consumer-group");
29 |         properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
30 |         properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
31 | 
32 |         properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
33 |         properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, KafkaProtobufDeserializer.class);
34 | 
35 |         properties.put(KafkaProtobufDeserializerConfig.SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081");
36 |         properties.put(KafkaProtobufDeserializerConfig.SPECIFIC_PROTOBUF_VALUE_TYPE, SimpleMessage.class.getName());
37 | 
38 |         KafkaConsumer<String, SimpleMessage> consumer = new KafkaConsumer<>(properties);
39 | 
40 |         consumer.subscribe(Collections.singleton("protobuf-topic"));
41 | 
42 |         //poll the record from the topic
43 |         while (true) {
44 |             ConsumerRecords<String, SimpleMessage> records = consumer.poll(Duration.ofMillis(100));
45 | 
46 |             for (ConsumerRecord<String, SimpleMessage> record : records) {
47 |                 System.out.println("Message content: " + record.value().getContent());
48 |                 System.out.println("Message time: " + record.value().getDateTime());
49 |             }
50 |             consumer.commitAsync();
51 |         }
52 |     }
53 | }
54 | 


--------------------------------------------------------------------------------
/src/main/java/com/codingharbour/kafka/protobuf/producer/ProtobufProducer.java:
--------------------------------------------------------------------------------
 1 | package com.codingharbour.kafka.protobuf.producer;
 2 | 
 3 | import com.codingharbour.protobuf.SimpleMessageProtos.SimpleMessage;
 4 | import io.confluent.kafka.serializers.protobuf.KafkaProtobufSerializer;
 5 | import io.confluent.kafka.serializers.protobuf.KafkaProtobufSerializerConfig;
 6 | import org.apache.kafka.clients.producer.KafkaProducer;
 7 | import org.apache.kafka.clients.producer.Producer;
 8 | import org.apache.kafka.clients.producer.ProducerConfig;
 9 | import org.apache.kafka.clients.producer.ProducerRecord;
10 | import org.apache.kafka.common.serialization.StringSerializer;
11 | 
12 | import java.time.Instant;
13 | import java.util.Properties;
14 | 
15 | public class ProtobufProducer {
16 | 
17 |     public static void main(String[] args) {
18 |         ProtobufProducer protobufProducer = new ProtobufProducer();
19 |         protobufProducer.writeMessage();
20 |     }
21 | 
22 |     public void writeMessage() {
23 |         //create kafka producer
24 |         Properties properties = new Properties();
25 |         properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
26 |         properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
27 |         properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaProtobufSerializer.class);
28 |         properties.put(KafkaProtobufSerializerConfig.SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081");
29 | 
30 |         Producer<String, SimpleMessage> producer = new KafkaProducer<>(properties);
31 | 
32 |         //prepare the message
33 |         SimpleMessage simpleMessage =
34 |                 SimpleMessage.newBuilder()
35 |                         .setContent("Hello world")
36 |                         .setDateTime(Instant.now().toString())
37 |                         .build();
38 | 
39 |         System.out.println(simpleMessage);
40 | 
41 |         //prepare the kafka record
42 |         ProducerRecord<String, SimpleMessage> record
43 |                 = new ProducerRecord<>("protobuf-topic", null, simpleMessage);
44 | 
45 |         producer.send(record);
46 |         //ensures record is sent before closing the producer
47 |         producer.flush();
48 | 
49 |         producer.close();
50 |     }
51 | 
52 | }
53 | 


--------------------------------------------------------------------------------
/src/main/protobuf/SimpleMessage.proto:
--------------------------------------------------------------------------------
1 | syntax = "proto3";
2 | 
3 | package com.codingharbour.protobuf;
4 | option java_outer_classname = "SimpleMessageProtos";
5 | 
6 | message SimpleMessage {
7 | 	string content = 1;
8 | 	string date_time = 2;
9 | }


--------------------------------------------------------------------------------