├── .travis.yml
├── resources
├── flink_job.png
└── streamers.png
├── .gitignore
├── dependency-reduced-pom.xml
├── src
└── main
│ ├── java
│ └── com
│ │ └── samaitra
│ │ └── SocketWindowWordCount.java
│ └── resources
│ └── example-ignite.xml
├── pom.xml
├── README.md
└── LICENSE
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: java
2 | mvn install
--------------------------------------------------------------------------------
/resources/flink_job.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/samaitra/streamers/HEAD/resources/flink_job.png
--------------------------------------------------------------------------------
/resources/streamers.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/samaitra/streamers/HEAD/resources/streamers.png
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Compiled class file
2 | *.class
3 |
4 | # Log file
5 | *.log
6 |
7 | # BlueJ files
8 | *.ctxt
9 |
10 | # Mobile Tools for Java (J2ME)
11 | .mtj.tmp/
12 |
13 | # Package Files #
14 | *.jar
15 | *.war
16 | *.nar
17 | *.ear
18 | *.zip
19 | *.tar.gz
20 | *.rar
21 |
22 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
23 | hs_err_pid*
24 |
25 | # idea
26 | .idea/*
--------------------------------------------------------------------------------
/dependency-reduced-pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 | 4.0.0
4 | com.samaitra
5 | streamers
6 | streamers
7 | 1.0-SNAPSHOT
8 | http://maven.apache.org
9 |
10 |
11 |
12 | maven-compiler-plugin
13 |
14 | 1.8
15 | 1.8
16 |
17 |
18 |
19 | maven-shade-plugin
20 | 3.1.1
21 |
22 |
23 | package
24 |
25 | shade
26 |
27 |
28 |
29 |
30 | com.samaitra.SocketWindowWordCount
31 |
32 |
33 | META-INF/spring.handlers
34 |
35 |
36 | META-INF/spring.schemas
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 | junit
48 | junit
49 | 3.8.1
50 | test
51 |
52 |
53 |
54 |
--------------------------------------------------------------------------------
/src/main/java/com/samaitra/SocketWindowWordCount.java:
--------------------------------------------------------------------------------
1 | package com.samaitra;
2 |
3 | import java.util.HashMap;
4 | import java.util.Map;
5 | import java.util.Properties;
6 | import org.apache.flink.api.common.functions.FlatMapFunction;
7 | import org.apache.flink.api.common.functions.MapFunction;
8 | import org.apache.flink.api.common.serialization.SimpleStringSchema;
9 | import org.apache.flink.api.java.tuple.Tuple2;
10 | import org.apache.flink.configuration.Configuration;
11 | import org.apache.flink.streaming.api.datastream.DataStream;
12 | import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
13 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
14 | import org.apache.flink.streaming.api.windowing.time.Time;
15 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010;
16 | import org.apache.flink.util.Collector;
17 | import org.apache.ignite.sink.flink.IgniteSink;
18 |
19 | public class SocketWindowWordCount {
20 | public static void main(String[] args) throws Exception {
21 | /** Ignite test configuration file. */
22 | final String GRID_CONF_FILE = "PATH_TO_PROJECT/streamers/src/main/resources/example-ignite.xml";
23 |
24 | IgniteSink igniteSink = new IgniteSink("testCache", GRID_CONF_FILE);
25 |
26 | igniteSink.setAllowOverwrite(true);
27 |
28 | igniteSink.setAutoFlushFrequency(5L);
29 |
30 | Configuration p = new Configuration();
31 | igniteSink.open(p);
32 |
33 | Properties properties = new Properties();
34 | properties.setProperty("bootstrap.servers", "localhost:9092");
35 | properties.setProperty("group.id", "test");
36 |
37 | // get the execution environment
38 | final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
39 |
40 | // get input data by connecting to kafka
41 | DataStream text = env
42 | .addSource(new FlinkKafkaConsumer010<>("mytopic", new SimpleStringSchema(), properties));
43 |
44 | // parse the data, group it, window it, and aggregate the counts
45 | SingleOutputStreamOperator