├── .gitignore
├── LICENSE
├── README.md
├── pom.xml
└── src
    └── main
        ├── java
            └── io
            │   └── redpanda
            │       └── examples
            │           └── WordCount.java
        └── resources
            └── log4j2.properties


/.gitignore:
--------------------------------------------------------------------------------
 1 | # Compiled class file
 2 | *.class
 3 | 
 4 | # Log file
 5 | *.log
 6 | 
 7 | # BlueJ files
 8 | *.ctxt
 9 | 
10 | # Mobile Tools for Java (J2ME)
11 | .mtj.tmp/
12 | 
13 | # Package Files #
14 | *.jar
15 | *.war
16 | *.nar
17 | *.ear
18 | *.zip
19 | *.tar.gz
20 | *.rar
21 | 
22 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
23 | hs_err_pid*
24 | 
25 | target/
26 | dependency-reduced-pom.xml
27 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2021 Nyalia Lui
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # flink-kafka-examples
2 | A repo of Java examples using Apache Flink with flink-connector-kafka
3 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <!--
  2 | Licensed to the Apache Software Foundation (ASF) under one
  3 | or more contributor license agreements.  See the NOTICE file
  4 | distributed with this work for additional information
  5 | regarding copyright ownership.  The ASF licenses this file
  6 | to you under the Apache License, Version 2.0 (the
  7 | "License"); you may not use this file except in compliance
  8 | with the License.  You may obtain a copy of the License at
  9 | 
 10 |   http://www.apache.org/licenses/LICENSE-2.0
 11 | 
 12 | Unless required by applicable law or agreed to in writing,
 13 | software distributed under the License is distributed on an
 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 15 | KIND, either express or implied.  See the License for the
 16 | specific language governing permissions and limitations
 17 | under the License.
 18 | -->
 19 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 20 | 	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 21 | 	<modelVersion>4.0.0</modelVersion>
 22 | 
 23 | 	<groupId>io.redpanda.examples</groupId>
 24 | 	<artifactId>flink-kafka-examples</artifactId>
 25 | 	<version>1.0</version>
 26 | 	<packaging>jar</packaging>
 27 | 
 28 | 	<name>Flink Quickstart Job</name>
 29 | 
 30 | 	<properties>
 31 | 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 32 | 		<flink.version>1.14.0</flink.version>
 33 | 		<target.java.version>1.8</target.java.version>
 34 | 		<scala.binary.version>2.11</scala.binary.version>
 35 | 		<maven.compiler.source>${target.java.version}</maven.compiler.source>
 36 | 		<maven.compiler.target>${target.java.version}</maven.compiler.target>
 37 | 		<log4j.version>2.14.1</log4j.version>
 38 | 	</properties>
 39 | 
 40 | 	<repositories>
 41 | 		<repository>
 42 | 			<id>apache.snapshots</id>
 43 | 			<name>Apache Development Snapshot Repository</name>
 44 | 			<url>https://repository.apache.org/content/repositories/snapshots/</url>
 45 | 			<releases>
 46 | 				<enabled>false</enabled>
 47 | 			</releases>
 48 | 			<snapshots>
 49 | 				<enabled>true</enabled>
 50 | 			</snapshots>
 51 | 		</repository>
 52 | 	</repositories>
 53 | 
 54 | 	<dependencies>
 55 | 		<!-- Apache Flink dependencies -->
 56 | 		<!-- These dependencies are provided, because they should not be packaged into the JAR file. -->
 57 | 		<dependency>
 58 | 			<groupId>org.apache.flink</groupId>
 59 | 			<artifactId>flink-java</artifactId>
 60 | 			<version>${flink.version}</version>
 61 | 			<scope>provided</scope>
 62 | 		</dependency>
 63 | 		<dependency>
 64 | 			<groupId>org.apache.flink</groupId>
 65 | 			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 66 | 			<version>${flink.version}</version>
 67 | 			<scope>provided</scope>
 68 | 		</dependency>
 69 | 		<dependency>
 70 | 			<groupId>org.apache.flink</groupId>
 71 | 			<artifactId>flink-clients_${scala.binary.version}</artifactId>
 72 | 			<version>${flink.version}</version>
 73 | 			<scope>provided</scope>
 74 | 		</dependency>
 75 | 
 76 | 		<!-- Add connector dependencies here. They must be in the default scope (compile). -->
 77 | 
 78 | 		<!-- Example:
 79 | 
 80 | 		<dependency>
 81 | 			<groupId>org.apache.flink</groupId>
 82 | 			<artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
 83 | 			<version>${flink.version}</version>
 84 | 		</dependency>
 85 | 		-->
 86 | 		<dependency>
 87 | 		  <groupId>org.apache.flink</groupId>
 88 | 		  <artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
 89 | 		  <version>${flink.version}</version>
 90 | 		</dependency>
 91 | 
 92 | 		<!-- Add logging framework, to produce console output when running in the IDE. -->
 93 | 		<!-- These dependencies are excluded from the application JAR by default. -->
 94 | 		<dependency>
 95 | 			<groupId>org.apache.logging.log4j</groupId>
 96 | 			<artifactId>log4j-slf4j-impl</artifactId>
 97 | 			<version>${log4j.version}</version>
 98 | 			<scope>runtime</scope>
 99 | 		</dependency>
100 | 		<dependency>
101 | 			<groupId>org.apache.logging.log4j</groupId>
102 | 			<artifactId>log4j-api</artifactId>
103 | 			<version>${log4j.version}</version>
104 | 			<scope>runtime</scope>
105 | 		</dependency>
106 | 		<dependency>
107 | 			<groupId>org.apache.logging.log4j</groupId>
108 | 			<artifactId>log4j-core</artifactId>
109 | 			<version>${log4j.version}</version>
110 | 			<scope>runtime</scope>
111 | 		</dependency>
112 | 	</dependencies>
113 | 
114 | 	<build>
115 | 		<plugins>
116 | 
117 | 			<!-- Java Compiler -->
118 | 			<plugin>
119 | 				<groupId>org.apache.maven.plugins</groupId>
120 | 				<artifactId>maven-compiler-plugin</artifactId>
121 | 				<version>3.1</version>
122 | 				<configuration>
123 | 					<source>${target.java.version}</source>
124 | 					<target>${target.java.version}</target>
125 | 				</configuration>
126 | 			</plugin>
127 | 
128 | 			<!-- We use the maven-shade plugin to create a fat jar that contains all necessary dependencies. -->
129 | 			<!-- Change the value of <mainClass>...</mainClass> if your program entry point changes. -->
130 | 			<plugin>
131 | 				<groupId>org.apache.maven.plugins</groupId>
132 | 				<artifactId>maven-shade-plugin</artifactId>
133 | 				<version>3.1.1</version>
134 | 				<executions>
135 | 					<!-- Run shade goal on package phase -->
136 | 					<execution>
137 | 						<phase>package</phase>
138 | 						<goals>
139 | 							<goal>shade</goal>
140 | 						</goals>
141 | 						<configuration>
142 | 							<artifactSet>
143 | 								<excludes>
144 | 									<exclude>org.apache.flink:flink-shaded-force-shading</exclude>
145 | 									<exclude>com.google.code.findbugs:jsr305</exclude>
146 | 									<exclude>org.slf4j:*</exclude>
147 | 									<exclude>org.apache.logging.log4j:*</exclude>
148 | 								</excludes>
149 | 							</artifactSet>
150 | 							<filters>
151 | 								<filter>
152 | 									<!-- Do not copy the signatures in the META-INF folder.
153 | 									Otherwise, this might cause SecurityExceptions when using the JAR. -->
154 | 									<artifact>*:*</artifact>
155 | 									<excludes>
156 | 										<exclude>META-INF/*.SF</exclude>
157 | 										<exclude>META-INF/*.DSA</exclude>
158 | 										<exclude>META-INF/*.RSA</exclude>
159 | 									</excludes>
160 | 								</filter>
161 | 							</filters>
162 | 							<transformers>
163 | 								<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
164 | 									<mainClass>io.redpanda.examples.WordCount</mainClass>
165 | 								</transformer>
166 | 							</transformers>
167 | 						</configuration>
168 | 					</execution>
169 | 				</executions>
170 | 			</plugin>
171 | 		</plugins>
172 | 
173 | 		<pluginManagement>
174 | 			<plugins>
175 | 
176 | 				<!-- This improves the out-of-the-box experience in Eclipse by resolving some warnings. -->
177 | 				<plugin>
178 | 					<groupId>org.eclipse.m2e</groupId>
179 | 					<artifactId>lifecycle-mapping</artifactId>
180 | 					<version>1.0.0</version>
181 | 					<configuration>
182 | 						<lifecycleMappingMetadata>
183 | 							<pluginExecutions>
184 | 								<pluginExecution>
185 | 									<pluginExecutionFilter>
186 | 										<groupId>org.apache.maven.plugins</groupId>
187 | 										<artifactId>maven-shade-plugin</artifactId>
188 | 										<versionRange>[3.1.1,)</versionRange>
189 | 										<goals>
190 | 											<goal>shade</goal>
191 | 										</goals>
192 | 									</pluginExecutionFilter>
193 | 									<action>
194 | 										<ignore/>
195 | 									</action>
196 | 								</pluginExecution>
197 | 								<pluginExecution>
198 | 									<pluginExecutionFilter>
199 | 										<groupId>org.apache.maven.plugins</groupId>
200 | 										<artifactId>maven-compiler-plugin</artifactId>
201 | 										<versionRange>[3.1,)</versionRange>
202 | 										<goals>
203 | 											<goal>testCompile</goal>
204 | 											<goal>compile</goal>
205 | 										</goals>
206 | 									</pluginExecutionFilter>
207 | 									<action>
208 | 										<ignore/>
209 | 									</action>
210 | 								</pluginExecution>
211 | 							</pluginExecutions>
212 | 						</lifecycleMappingMetadata>
213 | 					</configuration>
214 | 				</plugin>
215 | 			</plugins>
216 | 		</pluginManagement>
217 | 	</build>
218 | </project>
219 | 


--------------------------------------------------------------------------------
/src/main/java/io/redpanda/examples/WordCount.java:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Licensed to the Apache Software Foundation (ASF) under one
  3 |  * or more contributor license agreements.  See the NOTICE file
  4 |  * distributed with this work for additional information
  5 |  * regarding copyright ownership.  The ASF licenses this file
  6 |  * to you under the Apache License, Version 2.0 (the
  7 |  * "License"); you may not use this file except in compliance
  8 |  * with the License.  You may obtain a copy of the License at
  9 |  *
 10 |  *     http://www.apache.org/licenses/LICENSE-2.0
 11 |  *
 12 |  * Unless required by applicable law or agreed to in writing, software
 13 |  * distributed under the License is distributed on an "AS IS" BASIS,
 14 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 |  * See the License for the specific language governing permissions and
 16 |  * limitations under the License.
 17 |  */
 18 | 
 19 | package io.redpanda.examples;
 20 | 
 21 | 
 22 | import org.apache.flink.api.common.functions.FlatMapFunction;
 23 | import org.apache.flink.streaming.api.datastream.DataStream;
 24 | import org.apache.flink.api.java.tuple.Tuple2;
 25 | import org.apache.flink.util.Collector;
 26 | import org.apache.flink.connector.kafka.source.KafkaSource;
 27 | import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer;
 28 | import org.apache.flink.connector.kafka.sink.KafkaSink;
 29 | import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema;
 30 | import org.apache.flink.api.common.serialization.SimpleStringSchema;
 31 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 32 | import org.apache.flink.api.common.eventtime.WatermarkStrategy;
 33 | 
 34 | /**
 35 |  * This is a re-write of the Apache Flink WordCount example using Kafka connectors.
 36 |  * Find the original example at 
 37 |  * https://github.com/apache/flink/blob/master/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/wordcount/WordCount.java
 38 |  */
 39 | public class WordCount {
 40 | 
 41 | 	final static String inputTopic = "input-topic";
 42 | 	final static String outputTopic = "output-topic";
 43 | 	final static String jobTitle = "WordCount";
 44 | 
 45 | 	public static void main(String[] args) throws Exception {
 46 | 	    final String bootstrapServers = args.length > 0 ? args[0] : "localhost:9092";
 47 | 
 48 | 		// Set up the streaming execution environment
 49 | 		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 50 | 
 51 | 		KafkaSource<String> source = KafkaSource.<String>builder()
 52 | 		    .setBootstrapServers(bootstrapServers)
 53 | 		    .setTopics(inputTopic)
 54 | 		    .setGroupId("my-group")
 55 | 		    .setStartingOffsets(OffsetsInitializer.earliest())
 56 | 		    .setValueOnlyDeserializer(new SimpleStringSchema())
 57 | 		    .build();
 58 | 
 59 | 		KafkaRecordSerializationSchema<String> serializer = KafkaRecordSerializationSchema.builder()
 60 | 			.setValueSerializationSchema(new SimpleStringSchema())
 61 | 			.setTopic(outputTopic)
 62 | 			.build();
 63 | 
 64 | 		KafkaSink<String> sink = KafkaSink.<String>builder()
 65 | 			.setBootstrapServers(bootstrapServers)
 66 | 			.setRecordSerializer(serializer)
 67 | 			.build();
 68 | 
 69 | 		DataStream<String> text = env.fromSource(source, WatermarkStrategy.noWatermarks(), "Kafka Source");
 70 | 
 71 | 
 72 | 		// Split up the lines in pairs (2-tuples) containing: (word,1)
 73 |         DataStream<String> counts = text.flatMap(new Tokenizer())
 74 | 		// Group by the tuple field "0" and sum up tuple field "1"
 75 | 		.keyBy(value -> value.f0)
 76 | 		.sum(1)
 77 | 		.flatMap(new Reducer());
 78 | 
 79 | 		// Add the sink to so results
 80 | 		// are written to the outputTopic
 81 |         counts.sinkTo(sink);
 82 | 
 83 | 		// Execute program
 84 | 		env.execute(jobTitle);
 85 | 	}
 86 | 
 87 |     /**
 88 |      * Implements the string tokenizer that splits sentences into words as a user-defined
 89 |      * FlatMapFunction. The function takes a line (String) and splits it into multiple pairs in the
 90 |      * form of "(word,1)" ({@code Tuple2<String, Integer>}).
 91 |      */
 92 |     public static final class Tokenizer
 93 |             implements FlatMapFunction<String, Tuple2<String, Integer>> {
 94 | 
 95 |         @Override
 96 |         public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
 97 |             // Normalize and split the line
 98 |             String[] tokens = value.toLowerCase().split("\\W+");
 99 | 
100 |             // Emit the pairs
101 |             for (String token : tokens) {
102 |                 if (token.length() > 0) {
103 |                     out.collect(new Tuple2<>(token, 1));
104 |                 }
105 |             }
106 |         }
107 |     }
108 | 
109 |     // Implements a simple reducer using FlatMap to
110 |     // reduce the Tuple2 into a single string for 
111 |     // writing to kafka topics
112 |     public static final class Reducer
113 |             implements FlatMapFunction<Tuple2<String, Integer>, String> {
114 | 
115 |         @Override
116 |         public void flatMap(Tuple2<String, Integer> value, Collector<String> out) {
117 |         	// Convert the pairs to a string
118 |         	// for easy writing to Kafka Topic
119 |         	String count = value.f0 + " " + value.f1;
120 |         	out.collect(count);
121 |         }
122 |     }
123 | }
124 | 


--------------------------------------------------------------------------------
/src/main/resources/log4j2.properties:
--------------------------------------------------------------------------------
 1 | ################################################################################
 2 | #  Licensed to the Apache Software Foundation (ASF) under one
 3 | #  or more contributor license agreements.  See the NOTICE file
 4 | #  distributed with this work for additional information
 5 | #  regarding copyright ownership.  The ASF licenses this file
 6 | #  to you under the Apache License, Version 2.0 (the
 7 | #  "License"); you may not use this file except in compliance
 8 | #  with the License.  You may obtain a copy of the License at
 9 | #
10 | #      http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | #  Unless required by applicable law or agreed to in writing, software
13 | #  distributed under the License is distributed on an "AS IS" BASIS,
14 | #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | #  See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | ################################################################################
18 | 
19 | rootLogger.level = INFO
20 | rootLogger.appenderRef.console.ref = ConsoleAppender
21 | 
22 | appender.console.name = ConsoleAppender
23 | appender.console.type = CONSOLE
24 | appender.console.layout.type = PatternLayout
25 | appender.console.layout.pattern = %d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
26 | 


--------------------------------------------------------------------------------