├── .gitignore
├── README.md
├── src
    └── main
    │   ├── java
    │       └── io
    │       │   └── github
    │       │       └── streamingwithflink
    │       │           ├── chapter5
    │       │               ├── util
    │       │               │   ├── SmokeLevel.java
    │       │               │   ├── Alert.java
    │       │               │   └── SmokeLevelSource.java
    │       │               ├── RollingSum.java
    │       │               ├── KeyedTransformations.java
    │       │               ├── MultiStreamTransformations.java
    │       │               └── BasicTransformations.java
    │       │           ├── util
    │       │               ├── SensorReading.java
    │       │               ├── SensorTimeAssigner.java
    │       │               └── SensorSource.java
    │       │           ├── chapter1
    │       │               └── AverageSensorReadings.java
    │       │           └── chapter6
    │       │               ├── CoProcessFunctionTimers.java
    │       │               └── CustomWindow.java
    │   └── resources
    │       └── log4j.properties
├── pom.xml
└── LICENSE


/.gitignore:
--------------------------------------------------------------------------------
1 | target
2 | dependency-reduced-pom.xml
3 | *iml
4 | .idea
5 | derby.log
6 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | ## Java Examples for Stream Processing with Apache Flink
 2 | 
 3 | This repository hosts Java code examples for ["Stream Processing with Apache Flink"](http://shop.oreilly.com/product/0636920057321.do) by [Fabian Hueske](https://twitter.com/fhueske) and [Vasia Kalavri](https://twitter.com/vkalavri).
 4 | 
 5 | **Note:** The Java examples are not comlete yet. <br>
 6 | The [Scala examples](https://github.com/streaming-with-flink/examples-scala) are complete and we are working on translating them to Java.
 7 | 
 8 | <a href="http://shop.oreilly.com/product/0636920057321.do">
 9 |   <img width="240" src="https://covers.oreillystatic.com/images/0636920057321/cat.gif">
10 | </a>
11 | 


--------------------------------------------------------------------------------
/src/main/java/io/github/streamingwithflink/chapter5/util/SmokeLevel.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2015 Fabian Hueske / Vasia Kalavri
 3 |  *
 4 |  * Licensed under the Apache License, Version 2.0 (the "License");
 5 |  * you may not use this file except in compliance with the License.
 6 |  * You may obtain a copy of the License at
 7 |  *
 8 |  *  http://www.apache.org/licenses/LICENSE-2.0
 9 |  *
10 |  * Unless required by applicable law or agreed to in writing, software
11 |  * distributed under the License is distributed on an "AS IS" BASIS,
12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |  * See the License for the specific language governing permissions and
14 |  * limitations under the License.
15 |  */
16 | package io.github.streamingwithflink.chapter5.util;
17 | 
18 | public enum SmokeLevel {
19 |     LOW,
20 |     HIGH
21 | }
22 | 


--------------------------------------------------------------------------------
/src/main/resources/log4j.properties:
--------------------------------------------------------------------------------
 1 | ################################################################################
 2 | #  Licensed to the Apache Software Foundation (ASF) under one
 3 | #  or more contributor license agreements.  See the NOTICE file
 4 | #  distributed with this work for additional information
 5 | #  regarding copyright ownership.  The ASF licenses this file
 6 | #  to you under the Apache License, Version 2.0 (the
 7 | #  "License"); you may not use this file except in compliance
 8 | #  with the License.  You may obtain a copy of the License at
 9 | #
10 | #      http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | #  Unless required by applicable law or agreed to in writing, software
13 | #  distributed under the License is distributed on an "AS IS" BASIS,
14 | #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | #  See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | ################################################################################
18 | 
19 | log4j.rootLogger=INFO, console
20 | 
21 | log4j.appender.console=org.apache.log4j.ConsoleAppender
22 | log4j.appender.console.layout=org.apache.log4j.PatternLayout
23 | log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
24 | 


--------------------------------------------------------------------------------
/src/main/java/io/github/streamingwithflink/chapter5/util/Alert.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2015 Fabian Hueske / Vasia Kalavri
 3 |  *
 4 |  * Licensed under the Apache License, Version 2.0 (the "License");
 5 |  * you may not use this file except in compliance with the License.
 6 |  * You may obtain a copy of the License at
 7 |  *
 8 |  *  http://www.apache.org/licenses/LICENSE-2.0
 9 |  *
10 |  * Unless required by applicable law or agreed to in writing, software
11 |  * distributed under the License is distributed on an "AS IS" BASIS,
12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |  * See the License for the specific language governing permissions and
14 |  * limitations under the License.
15 |  */
16 | package io.github.streamingwithflink.chapter5.util;
17 | 
18 | /**
19 |  * POJO representing an alert.
20 |  */
21 | public class Alert {
22 | 
23 |     public String message;
24 |     public long timestamp;
25 | 
26 |     /**
27 |      * Empty default constructor to comply with Flink's POJO requirements.
28 |      */
29 |     public Alert() { }
30 | 
31 |     public Alert(String message, long timestamp) {
32 |         this.message = message;
33 |         this.timestamp = timestamp;
34 |     }
35 | 
36 |     public String toString() {
37 |         return "(" + message + ", " + timestamp + ")";
38 |     }
39 | }
40 | 


--------------------------------------------------------------------------------
/src/main/java/io/github/streamingwithflink/util/SensorReading.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2015 Fabian Hueske / Vasia Kalavri
 3 |  *
 4 |  * Licensed under the Apache License, Version 2.0 (the "License");
 5 |  * you may not use this file except in compliance with the License.
 6 |  * You may obtain a copy of the License at
 7 |  *
 8 |  *  http://www.apache.org/licenses/LICENSE-2.0
 9 |  *
10 |  * Unless required by applicable law or agreed to in writing, software
11 |  * distributed under the License is distributed on an "AS IS" BASIS,
12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |  * See the License for the specific language governing permissions and
14 |  * limitations under the License.
15 |  */
16 | package io.github.streamingwithflink.util;
17 | 
18 | /**
19 |  * POJO to hold sensor reading data.
20 |  */
21 | public class SensorReading {
22 | 
23 |     // id of the sensor
24 |     public String id;
25 |     // timestamp of the reading
26 |     public long timestamp;
27 |     // temperature value of the reading
28 |     public double temperature;
29 | 
30 |     /**
31 |      * Empty default constructor to satify Flink's POJO requirements.
32 |      */
33 |     public SensorReading() { }
34 | 
35 |     public SensorReading(String id, long timestamp, double temperature) {
36 |         this.id = id;
37 |         this.timestamp = timestamp;
38 |         this.temperature = temperature;
39 |     }
40 | 
41 |     public String toString() {
42 |         return "(" + this.id + ", " + this.timestamp + ", " + this.temperature + ")";
43 |     }
44 | }
45 | 


--------------------------------------------------------------------------------
/src/main/java/io/github/streamingwithflink/util/SensorTimeAssigner.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2015 Fabian Hueske / Vasia Kalavri
 3 |  *
 4 |  * Licensed under the Apache License, Version 2.0 (the "License");
 5 |  * you may not use this file except in compliance with the License.
 6 |  * You may obtain a copy of the License at
 7 |  *
 8 |  *  http://www.apache.org/licenses/LICENSE-2.0
 9 |  *
10 |  * Unless required by applicable law or agreed to in writing, software
11 |  * distributed under the License is distributed on an "AS IS" BASIS,
12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |  * See the License for the specific language governing permissions and
14 |  * limitations under the License.
15 |  */
16 | package io.github.streamingwithflink.util;
17 | 
18 | import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor;
19 | import org.apache.flink.streaming.api.windowing.time.Time;
20 | 
21 | /**
22 |  * Assigns timestamps to SensorReadings based on their internal timestamp and
23 |  * emits watermarks with five seconds slack.
24 |  */
25 | public class SensorTimeAssigner extends BoundedOutOfOrdernessTimestampExtractor<SensorReading> {
26 | 
27 |     /**
28 |      * Configures the extractor with 5 seconds out-of-order interval.
29 |      */
30 |     public SensorTimeAssigner() {
31 |         super(Time.seconds(5));
32 |     }
33 | 
34 |     /**
35 |      * Extracts timestamp from SensorReading.
36 |      *
37 |      * @param r sensor reading
38 |      * @return the timestamp of the sensor reading.
39 |      */
40 |     @Override
41 |     public long extractTimestamp(SensorReading r) {
42 |         return r.timestamp;
43 |     }
44 | }
45 | 


--------------------------------------------------------------------------------
/src/main/java/io/github/streamingwithflink/chapter5/RollingSum.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2015 Fabian Hueske / Vasia Kalavri
 3 |  *
 4 |  * Licensed under the Apache License, Version 2.0 (the "License");
 5 |  * you may not use this file except in compliance with the License.
 6 |  * You may obtain a copy of the License at
 7 |  *
 8 |  *  http://www.apache.org/licenses/LICENSE-2.0
 9 |  *
10 |  * Unless required by applicable law or agreed to in writing, software
11 |  * distributed under the License is distributed on an "AS IS" BASIS,
12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |  * See the License for the specific language governing permissions and
14 |  * limitations under the License.
15 |  */
16 | package io.github.streamingwithflink.chapter5;
17 | 
18 | import org.apache.flink.api.java.tuple.Tuple3;
19 | import org.apache.flink.streaming.api.datastream.DataStream;
20 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
21 | 
22 | /**
23 |  * Program demonstrating a rolling sum.
24 |  */
25 | public class RollingSum {
26 | 
27 |     public static void main(String[] args) throws Exception {
28 | 
29 |         // set up the streaming execution environment
30 |         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
31 | 
32 |         DataStream<Tuple3<Integer, Integer, Integer>> inputStream = env.fromElements(
33 |             Tuple3.of(1, 2, 2), Tuple3.of(2, 3, 1), Tuple3.of(2, 2, 4), Tuple3.of(1, 5, 3));
34 | 
35 |         DataStream<Tuple3<Integer, Integer, Integer>> resultStream = inputStream
36 |             .keyBy(0) // key on first field of tuples
37 |             .sum(1); // sum the second field of the tuple
38 | 
39 |         resultStream.print();
40 | 
41 |         // execute the application
42 |         env.execute();
43 |     }
44 | 
45 | }
46 | 


--------------------------------------------------------------------------------
/src/main/java/io/github/streamingwithflink/chapter5/util/SmokeLevelSource.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2015 Fabian Hueske / Vasia Kalavri
 3 |  *
 4 |  * Licensed under the Apache License, Version 2.0 (the "License");
 5 |  * you may not use this file except in compliance with the License.
 6 |  * You may obtain a copy of the License at
 7 |  *
 8 |  *  http://www.apache.org/licenses/LICENSE-2.0
 9 |  *
10 |  * Unless required by applicable law or agreed to in writing, software
11 |  * distributed under the License is distributed on an "AS IS" BASIS,
12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |  * See the License for the specific language governing permissions and
14 |  * limitations under the License.
15 |  */
16 | package io.github.streamingwithflink.chapter5.util;
17 | 
18 | import org.apache.flink.streaming.api.functions.source.SourceFunction;
19 | 
20 | import java.util.Random;
21 | 
22 | /**
23 |  * Flink SourceFunction to generate random SmokeLevel events.
24 |  */
25 | public class SmokeLevelSource implements SourceFunction<SmokeLevel> {
26 | 
27 |     // flag indicating whether source is still running
28 |     private boolean running = true;
29 | 
30 |     /**
31 |      * Continuously emit one smoke level event per second.
32 |      */
33 |     @Override
34 |     public void run(SourceContext<SmokeLevel> srcCtx) throws Exception {
35 | 
36 |         // initialize random number generator
37 |         Random rand = new Random();
38 | 
39 |         // emit data until being canceled
40 |         while (running) {
41 | 
42 |             if (rand.nextGaussian() > 0.8) {
43 |                 // emit a high SmokeLevel
44 |                 srcCtx.collect(SmokeLevel.HIGH);
45 |             } else {
46 |                 srcCtx.collect(SmokeLevel.LOW);
47 |             }
48 | 
49 |             // wait for 1 second
50 |             Thread.sleep(1000);
51 |         }
52 |     }
53 | 
54 |     /**
55 |      * Cancel the emission of smoke level events.
56 |      */
57 |     @Override
58 |     public void cancel() {
59 |         this.running = false;
60 | 
61 |     }
62 | }
63 | 


--------------------------------------------------------------------------------
/src/main/java/io/github/streamingwithflink/chapter5/KeyedTransformations.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2015 Fabian Hueske / Vasia Kalavri
 3 |  *
 4 |  * Licensed under the Apache License, Version 2.0 (the "License");
 5 |  * you may not use this file except in compliance with the License.
 6 |  * You may obtain a copy of the License at
 7 |  *
 8 |  *  http://www.apache.org/licenses/LICENSE-2.0
 9 |  *
10 |  * Unless required by applicable law or agreed to in writing, software
11 |  * distributed under the License is distributed on an "AS IS" BASIS,
12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |  * See the License for the specific language governing permissions and
14 |  * limitations under the License.
15 |  */
16 | package io.github.streamingwithflink.chapter5;
17 | 
18 | import io.github.streamingwithflink.util.SensorReading;
19 | import io.github.streamingwithflink.util.SensorSource;
20 | import io.github.streamingwithflink.util.SensorTimeAssigner;
21 | import org.apache.flink.streaming.api.TimeCharacteristic;
22 | import org.apache.flink.streaming.api.datastream.DataStream;
23 | import org.apache.flink.streaming.api.datastream.KeyedStream;
24 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
25 | 
26 | /**
27 |  * Example program to demonstrate keyed transformation functions: keyBy, reduce.
28 |  */
29 | public class KeyedTransformations {
30 | 
31 |     public static void main(String[] args) throws Exception {
32 | 
33 |         // set up the streaming execution environment
34 |         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
35 | 
36 |         // use event time for the application
37 |         env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
38 |         // configure watermark interval
39 |         env.getConfig().setAutoWatermarkInterval(1000L);
40 | 
41 |         // ingest sensor stream
42 |         DataStream<SensorReading> readings = env
43 |             // SensorSource generates random temperature readings
44 |             .addSource(new SensorSource())
45 |             // assign timestamps and watermarks which are required for event time
46 |             .assignTimestampsAndWatermarks(new SensorTimeAssigner());
47 | 
48 |         // group sensor readings by sensor id
49 |         KeyedStream<SensorReading, String> keyed = readings
50 |             .keyBy(r -> r.id);
51 | 
52 |         // a rolling reduce that computes the highest temperature of each sensor and
53 |         // the corresponding timestamp
54 |         DataStream<SensorReading> maxTempPerSensor = keyed
55 |             .reduce((r1, r2) -> {
56 |                 if (r1.temperature > r2.temperature) {
57 |                     return r1;
58 |                 } else {
59 |                     return r2;
60 |                 }
61 |             });
62 | 
63 |         maxTempPerSensor.print();
64 | 
65 |         // execute application
66 |         env.execute("Keyed Transformations Example");
67 |     }
68 | }
69 | 


--------------------------------------------------------------------------------
/src/main/java/io/github/streamingwithflink/util/SensorSource.java:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2015 Fabian Hueske / Vasia Kalavri
 3 |  *
 4 |  * Licensed under the Apache License, Version 2.0 (the "License");
 5 |  * you may not use this file except in compliance with the License.
 6 |  * You may obtain a copy of the License at
 7 |  *
 8 |  *  http://www.apache.org/licenses/LICENSE-2.0
 9 |  *
10 |  * Unless required by applicable law or agreed to in writing, software
11 |  * distributed under the License is distributed on an "AS IS" BASIS,
12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |  * See the License for the specific language governing permissions and
14 |  * limitations under the License.
15 |  */
16 | package io.github.streamingwithflink.util;
17 | 
18 | import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
19 | 
20 | import java.util.Calendar;
21 | import java.util.Random;
22 | 
23 | /**
24 |  * Flink SourceFunction to generate SensorReadings with random temperature values.
25 |  *
26 |  * Each parallel instance of the source simulates 10 sensors which emit one sensor reading every 100 ms.
27 |  *
28 |  * Note: This is a simple data-generating source function that does not checkpoint its state.
29 |  * In case of a failure, the source does not replay any data.
30 |  */
31 | public class SensorSource extends RichParallelSourceFunction<SensorReading> {
32 | 
33 |     // flag indicating whether source is still running
34 |     private boolean running = true;
35 | 
36 |     /** run() continuously emits SensorReadings by emitting them through the SourceContext. */
37 |     @Override
38 |     public void run(SourceContext<SensorReading> srcCtx) throws Exception {
39 | 
40 |         // initialize random number generator
41 |         Random rand = new Random();
42 |         // look up index of this parallel task
43 |         int taskIdx = this.getRuntimeContext().getIndexOfThisSubtask();
44 | 
45 |         // initialize sensor ids and temperatures
46 |         String[] sensorIds = new String[10];
47 |         double[] curFTemp = new double[10];
48 |         for (int i = 0; i < 10; i++) {
49 |             sensorIds[i] = "sensor_" + (taskIdx * 10 + i);
50 |             curFTemp[i] = 65 + (rand.nextGaussian() * 20);
51 |         }
52 | 
53 |         while (running) {
54 | 
55 |             // get current time
56 |             long curTime = Calendar.getInstance().getTimeInMillis();
57 | 
58 |             // emit SensorReadings
59 |             for (int i = 0; i < 10; i++) {
60 |                 // update current temperature
61 |                 curFTemp[i] += rand.nextGaussian() * 0.5;
62 |                 // emit reading
63 |                 srcCtx.collect(new SensorReading(sensorIds[i], curTime, curFTemp[i]));
64 |             }
65 | 
66 |             // wait for 100 ms
67 |             Thread.sleep(100);
68 |         }
69 |     }
70 | 
71 |     /** Cancels this SourceFunction. */
72 |     @Override
73 |     public void cancel() {
74 |         this.running = false;
75 |     }
76 | }
77 | 


--------------------------------------------------------------------------------
/src/main/java/io/github/streamingwithflink/chapter1/AverageSensorReadings.java:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright 2015 Fabian Hueske / Vasia Kalavri
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *  http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | package io.github.streamingwithflink.chapter1;
 17 | 
 18 | import io.github.streamingwithflink.util.SensorReading;
 19 | import io.github.streamingwithflink.util.SensorSource;
 20 | import io.github.streamingwithflink.util.SensorTimeAssigner;
 21 | import org.apache.flink.streaming.api.TimeCharacteristic;
 22 | import org.apache.flink.streaming.api.datastream.DataStream;
 23 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 24 | import org.apache.flink.streaming.api.functions.windowing.WindowFunction;
 25 | import org.apache.flink.streaming.api.windowing.time.Time;
 26 | import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 27 | import org.apache.flink.util.Collector;
 28 | 
 29 | public class AverageSensorReadings {
 30 | 
 31 |     /**
 32 |      * main() defines and executes the DataStream program.
 33 |      *
 34 |      * @param args program arguments
 35 |      * @throws Exception
 36 |      */
 37 |     public static void main(String[] args) throws Exception {
 38 | 
 39 |         // set up the streaming execution environment
 40 |         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 41 | 
 42 |         // use event time for the application
 43 |         env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 44 |         // configure watermark interval
 45 |         env.getConfig().setAutoWatermarkInterval(1000L);
 46 | 
 47 |         // ingest sensor stream
 48 |         DataStream<SensorReading> sensorData = env
 49 |             // SensorSource generates random temperature readings
 50 |             .addSource(new SensorSource())
 51 |             // assign timestamps and watermarks which are required for event time
 52 |             .assignTimestampsAndWatermarks(new SensorTimeAssigner());
 53 | 
 54 |         DataStream<SensorReading> avgTemp = sensorData
 55 |             // convert Fahrenheit to Celsius using and inlined map function
 56 |             .map( r -> new SensorReading(r.id, r.timestamp, (r.temperature - 32) * (5.0 / 9.0)))
 57 |             // organize stream by sensor
 58 |             .keyBy(r -> r.id)
 59 |             // group readings in 1 second windows
 60 |             .timeWindow(Time.seconds(1))
 61 |             // compute average temperature using a user-defined function
 62 |             .apply(new TemperatureAverager());
 63 | 
 64 |         // print result stream to standard out
 65 |         avgTemp.print();
 66 | 
 67 |         // execute application
 68 |         env.execute("Compute average sensor temperature");
 69 |     }
 70 | 
 71 |     /**
 72 |      *  User-defined WindowFunction to compute the average temperature of SensorReadings
 73 |      */
 74 |     public static class TemperatureAverager implements WindowFunction<SensorReading, SensorReading, String, TimeWindow> {
 75 | 
 76 |         /**
 77 |          * apply() is invoked once for each window.
 78 |          *
 79 |          * @param sensorId the key (sensorId) of the window
 80 |          * @param window meta data for the window
 81 |          * @param input an iterable over the collected sensor readings that were assigned to the window
 82 |          * @param out a collector to emit results from the function
 83 |          */
 84 |         @Override
 85 |         public void apply(String sensorId, TimeWindow window, Iterable<SensorReading> input, Collector<SensorReading> out) {
 86 | 
 87 |             // compute the average temperature
 88 |             int cnt = 0;
 89 |             double sum = 0.0;
 90 |             for (SensorReading r : input) {
 91 |                 cnt++;
 92 |                 sum += r.temperature;
 93 |             }
 94 |             double avgTemp = sum / cnt;
 95 | 
 96 |             // emit a SensorReading with the average temperature
 97 |             out.collect(new SensorReading(sensorId, window.getEnd(), avgTemp));
 98 |         }
 99 |     }
100 | }
101 | 


--------------------------------------------------------------------------------
/src/main/java/io/github/streamingwithflink/chapter5/MultiStreamTransformations.java:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright 2015 Fabian Hueske / Vasia Kalavri
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *  http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | package io.github.streamingwithflink.chapter5;
 17 | 
 18 | import io.github.streamingwithflink.chapter5.util.Alert;
 19 | import io.github.streamingwithflink.chapter5.util.SmokeLevel;
 20 | import io.github.streamingwithflink.chapter5.util.SmokeLevelSource;
 21 | import io.github.streamingwithflink.util.SensorReading;
 22 | import io.github.streamingwithflink.util.SensorSource;
 23 | import io.github.streamingwithflink.util.SensorTimeAssigner;
 24 | import org.apache.flink.streaming.api.TimeCharacteristic;
 25 | import org.apache.flink.streaming.api.datastream.DataStream;
 26 | import org.apache.flink.streaming.api.datastream.KeyedStream;
 27 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 28 | import org.apache.flink.streaming.api.functions.co.CoFlatMapFunction;
 29 | import org.apache.flink.util.Collector;
 30 | 
 31 | /**
 32 |  * A simple application that outputs an alert whenever there is a high risk of fire.
 33 |  * The application receives the stream of temperature sensor readings and a stream of smoke level measurements.
 34 |  * When the temperature is over a given threshold and the smoke level is high, we emit a fire alert.
 35 |  */
 36 | public class MultiStreamTransformations {
 37 | 
 38 |     public static void main(String[] args) throws Exception {
 39 | 
 40 |         // set up the streaming execution environment
 41 |         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 42 | 
 43 |         // use event time for the application
 44 |         env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 45 |         // configure watermark interval
 46 |         env.getConfig().setAutoWatermarkInterval(1000L);
 47 | 
 48 |         // ingest sensor stream
 49 |         DataStream<SensorReading> tempReadings = env
 50 |             // SensorSource generates random temperature readings
 51 |             .addSource(new SensorSource())
 52 |             // assign timestamps and watermarks which are required for event time
 53 |             .assignTimestampsAndWatermarks(new SensorTimeAssigner());
 54 | 
 55 |         // ingest smoke level stream
 56 |         DataStream<SmokeLevel> smokeReadings = env
 57 |             .addSource(new SmokeLevelSource())
 58 |             .setParallelism(1);
 59 | 
 60 |         // group sensor readings by sensor id
 61 |         KeyedStream<SensorReading, String> keyedTempReadings = tempReadings
 62 |             .keyBy(r -> r.id);
 63 | 
 64 |         // connect the two streams and raise an alert if the temperature and
 65 |         // smoke levels are high
 66 |         DataStream<Alert> alerts = keyedTempReadings
 67 |             .connect(smokeReadings.broadcast())
 68 |             .flatMap(new RaiseAlertFlatMap());
 69 | 
 70 |         alerts.print();
 71 | 
 72 |         // execute the application
 73 |         env.execute("Multi-Stream Transformations Example");
 74 |     }
 75 | 
 76 |     /**
 77 |      * A CoFlatMapFunction that processes a stream of temperature readings ans a control stream
 78 |      * of smoke level events. The control stream updates a shared variable with the current smoke level.
 79 |      * For every event in the sensor stream, if the temperature reading is above 100 degrees
 80 |      * and the smoke level is high, a "Risk of fire" alert is generated.
 81 |      */
 82 |     public static class RaiseAlertFlatMap implements CoFlatMapFunction<SensorReading, SmokeLevel, Alert> {
 83 | 
 84 |         private SmokeLevel smokeLevel = SmokeLevel.LOW;
 85 | 
 86 |         @Override
 87 |         public void flatMap1(SensorReading tempReading, Collector<Alert> out) throws Exception {
 88 |             // high chance of fire => true
 89 |             if (this.smokeLevel == SmokeLevel.HIGH && tempReading.temperature > 100) {
 90 |                 out.collect(new Alert("Risk of fire! " + tempReading, tempReading.timestamp));
 91 |             }
 92 |         }
 93 | 
 94 |         @Override
 95 |         public void flatMap2(SmokeLevel smokeLevel, Collector<Alert> out) {
 96 |             // update smoke level
 97 |             this.smokeLevel = smokeLevel;
 98 |         }
 99 |     }
100 | }
101 | 


--------------------------------------------------------------------------------
/src/main/java/io/github/streamingwithflink/chapter5/BasicTransformations.java:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright 2015 Fabian Hueske / Vasia Kalavri
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *  http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | package io.github.streamingwithflink.chapter5;
 17 | 
 18 | import io.github.streamingwithflink.util.SensorReading;
 19 | import io.github.streamingwithflink.util.SensorSource;
 20 | import io.github.streamingwithflink.util.SensorTimeAssigner;
 21 | import org.apache.flink.api.common.functions.FilterFunction;
 22 | import org.apache.flink.api.common.functions.FlatMapFunction;
 23 | import org.apache.flink.api.common.functions.MapFunction;
 24 | import org.apache.flink.api.common.typeinfo.Types;
 25 | import org.apache.flink.streaming.api.TimeCharacteristic;
 26 | import org.apache.flink.streaming.api.datastream.DataStream;
 27 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 28 | import org.apache.flink.util.Collector;
 29 | 
 30 | /**
 31 |  * Example program to demonstrate simple transformation functions: filter, map, and flatMap.
 32 |  */
 33 | public class BasicTransformations {
 34 | 
 35 |     public static void main(String[] args) throws Exception {
 36 | 
 37 |         // set up the streaming execution environment
 38 |         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 39 | 
 40 |         // use event time for the application
 41 |         env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 42 |         // configure watermark interval
 43 |         env.getConfig().setAutoWatermarkInterval(1000L);
 44 | 
 45 |         // ingest sensor stream
 46 |         DataStream<SensorReading> readings = env
 47 |             // SensorSource generates random temperature readings
 48 |             .addSource(new SensorSource())
 49 |             // assign timestamps and watermarks which are required for event time
 50 |             .assignTimestampsAndWatermarks(new SensorTimeAssigner());
 51 | 
 52 |         // filter out sensor measurements with temperature below 25 degrees
 53 |         DataStream<SensorReading> filteredReadings = readings
 54 |             .filter(r -> r.temperature >= 25);
 55 | 
 56 |         // the above filter transformation using a FilterFunction instead of a lambda function
 57 |         // DataStream<SensorReading> filteredReadings = readings
 58 |         //     .filter(new TemperatureFilter(25));
 59 | 
 60 |         // project the reading to the id of the sensor
 61 |         DataStream<String> sensorIds = filteredReadings
 62 |             .map(r -> r.id);
 63 | 
 64 |         // the above map transformation using a MapFunction instead of a lambda function
 65 |         // DataStream<String> sensorIds = filteredReadings
 66 |         //     .map(new IdExtractor());
 67 | 
 68 |         // split the String id of each sensor to the prefix "sensor" and sensor number
 69 |         DataStream<String> splitIds = sensorIds
 70 |             .flatMap((FlatMapFunction<String, String>)
 71 |                     (id, out) -> { for (String s: id.split("_")) { out.collect(s);}})
 72 |             // provide result type because Java cannot infer return type of lambda function
 73 |             .returns(Types.STRING);
 74 | 
 75 |         // the above flatMap transformation using a FlatMapFunction instead of a lambda function
 76 |         // DataStream<String> splitIds = sensorIds
 77 |         //         .flatMap(new IdSplitter());
 78 | 
 79 |         // print result stream to standard out
 80 |         splitIds.print();
 81 | 
 82 |         // execute application
 83 |         env.execute("Basic Transformations Example");
 84 |     }
 85 | 
 86 |     /**
 87 |      * User-defined FilterFunction to filter out SensorReading with temperature below the threshold.
 88 |      */
 89 |     public static class TemperatureFilter implements FilterFunction<SensorReading> {
 90 | 
 91 |         private final double threshold;
 92 | 
 93 |         public TemperatureFilter(double threshold) {
 94 |             this.threshold = threshold;
 95 |         }
 96 | 
 97 |         @Override
 98 |         public boolean filter(SensorReading r) {
 99 |             return r.temperature >= threshold;
100 |         }
101 |     }
102 | 
103 |     /**
104 |      * User-defined MapFunction to extract a reading's sensor id.
105 |      */
106 |     public static class IdExtractor implements MapFunction<SensorReading, String> {
107 | 
108 |         @Override
109 |         public String map(SensorReading r) throws Exception {
110 |             return r.id;
111 |         }
112 |     }
113 | 
114 |     /**
115 |      * User-defined FlatMapFunction that splits a sensor's id String into a prefix and a number.
116 |      */
117 |     public static class IdSplitter implements FlatMapFunction<String, String> {
118 | 
119 |         @Override
120 |         public void flatMap(String id, Collector<String> out) {
121 | 
122 |             String[] splits = id.split("_");
123 | 
124 |             for (String split : splits) {
125 |                 out.collect(split);
126 |             }
127 |         }
128 |     }
129 | 
130 | }
131 | 


--------------------------------------------------------------------------------
/src/main/java/io/github/streamingwithflink/chapter6/CoProcessFunctionTimers.java:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright 2015 Fabian Hueske / Vasia Kalavri
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *  http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | package io.github.streamingwithflink.chapter6;
 17 | 
 18 | import io.github.streamingwithflink.util.SensorReading;
 19 | import io.github.streamingwithflink.util.SensorSource;
 20 | import org.apache.flink.api.common.state.ValueState;
 21 | import org.apache.flink.api.common.state.ValueStateDescriptor;
 22 | import org.apache.flink.api.common.typeinfo.Types;
 23 | import org.apache.flink.api.java.tuple.Tuple2;
 24 | import org.apache.flink.configuration.Configuration;
 25 | import org.apache.flink.streaming.api.TimeCharacteristic;
 26 | import org.apache.flink.streaming.api.datastream.DataStream;
 27 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 28 | import org.apache.flink.streaming.api.functions.co.CoProcessFunction;
 29 | import org.apache.flink.util.Collector;
 30 | 
 31 | /**
 32 |  * This example shows how to use a CoProcessFunction and Timers.
 33 |  */
 34 | public class CoProcessFunctionTimers {
 35 | 
 36 |     public static void main(String[] args) throws Exception {
 37 | 
 38 |         // set up the streaming execution environment
 39 |         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 40 | 
 41 |         // use event time for the application
 42 |         env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
 43 | 
 44 |         // switch messages disable filtering of sensor readings for a specific amount of time
 45 |         DataStream<Tuple2<String, Long>> filterSwitches = env
 46 |             .fromElements(
 47 |                 // forward readings of sensor_2 for 10 seconds
 48 |                 Tuple2.of("sensor_2", 10_000L),
 49 |                 // forward readings of sensor_7 for 1 minute
 50 |                 Tuple2.of("sensor_7", 60_000L));
 51 | 
 52 |         // ingest sensor stream
 53 |         DataStream<SensorReading> readings = env
 54 |             // SensorSource generates random temperature readings
 55 |             .addSource(new SensorSource());
 56 | 
 57 |         DataStream<SensorReading> forwardedReadings = readings
 58 |             // connect readings and switches
 59 |             .connect(filterSwitches)
 60 |             // key by sensor ids
 61 |             .keyBy(r -> r.id, s -> s.f0)
 62 |             // apply filtering CoProcessFunction
 63 |             .process(new ReadingFilter());
 64 | 
 65 |         forwardedReadings.print();
 66 | 
 67 |         env.execute("Filter sensor readings");
 68 |     }
 69 | 
 70 |     public static class ReadingFilter extends CoProcessFunction<SensorReading, Tuple2<String, Long>, SensorReading> {
 71 | 
 72 |         // switch to enable forwarding
 73 |         private ValueState<Boolean> forwardingEnabled;
 74 |         // timestamp to disable the currently active timer
 75 |         private ValueState<Long> disableTimer;
 76 | 
 77 |         @Override
 78 |         public void open(Configuration parameters) throws Exception {
 79 |             forwardingEnabled = getRuntimeContext().getState(
 80 |                 new ValueStateDescriptor<>("filterSwitch", Types.BOOLEAN));
 81 |             disableTimer = getRuntimeContext().getState(
 82 |                 new ValueStateDescriptor<Long>("timer", Types.LONG));
 83 |         }
 84 | 
 85 |         @Override
 86 |         public void processElement1(SensorReading r, Context ctx, Collector<SensorReading> out) throws Exception {
 87 |             // check if we need to forward the reading
 88 |             Boolean forward = forwardingEnabled.value();
 89 |             if (forward != null && forward) {
 90 |                 out.collect(r);
 91 |             }
 92 |         }
 93 | 
 94 |         @Override
 95 |         public void processElement2(Tuple2<String, Long> s, Context ctx, Collector<SensorReading> out) throws Exception {
 96 |             // enable forwarding of readings
 97 |             forwardingEnabled.update(true);
 98 |             // set timer to disable switch
 99 |             long timerTimestamp = ctx.timerService().currentProcessingTime() + s.f1;
100 |             Long curTimerTimestamp = disableTimer.value();
101 |             if (curTimerTimestamp == null || timerTimestamp > curTimerTimestamp) {
102 |                 // remove current timer
103 |                 if (curTimerTimestamp != null) {
104 |                     ctx.timerService().deleteProcessingTimeTimer(curTimerTimestamp);
105 |                 }
106 |                 // register new timer
107 |                 ctx.timerService().registerProcessingTimeTimer(timerTimestamp);
108 |                 disableTimer.update(timerTimestamp);
109 |             }
110 |         }
111 | 
112 |         @Override
113 |         public void onTimer(long ts, OnTimerContext ctx, Collector<SensorReading> out) throws Exception {
114 |             // remove all state
115 |             forwardingEnabled.clear();
116 |             disableTimer.clear();
117 |         }
118 |     }
119 | }
120 | 
121 | 
122 | 


--------------------------------------------------------------------------------
/src/main/java/io/github/streamingwithflink/chapter6/CustomWindow.java:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright 2015 Fabian Hueske / Vasia Kalavri
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *  http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | package io.github.streamingwithflink.chapter6;
 17 | 
 18 | import io.github.streamingwithflink.util.SensorReading;
 19 | import io.github.streamingwithflink.util.SensorSource;
 20 | import io.github.streamingwithflink.util.SensorTimeAssigner;
 21 | import org.apache.flink.api.common.ExecutionConfig;
 22 | import org.apache.flink.api.common.state.ValueState;
 23 | import org.apache.flink.api.common.state.ValueStateDescriptor;
 24 | import org.apache.flink.api.common.typeinfo.Types;
 25 | import org.apache.flink.api.common.typeutils.TypeSerializer;
 26 | import org.apache.flink.api.java.tuple.Tuple4;
 27 | import org.apache.flink.streaming.api.TimeCharacteristic;
 28 | import org.apache.flink.streaming.api.datastream.DataStream;
 29 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 30 | import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
 31 | import org.apache.flink.streaming.api.windowing.assigners.WindowAssigner;
 32 | import org.apache.flink.streaming.api.windowing.triggers.EventTimeTrigger;
 33 | import org.apache.flink.streaming.api.windowing.triggers.Trigger;
 34 | import org.apache.flink.streaming.api.windowing.triggers.TriggerResult;
 35 | import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
 36 | import org.apache.flink.util.Collector;
 37 | 
 38 | import java.util.Collection;
 39 | import java.util.Collections;
 40 | 
 41 | public class CustomWindow {
 42 | 
 43 |     public static void main(String[] args) throws Exception {
 44 | 
 45 |         // set up the streaming execution environment
 46 |         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 47 | 
 48 |         // checkpoint every 10 seconds
 49 |         env.getCheckpointConfig().setCheckpointInterval(10_000);
 50 | 
 51 |         // use event time for the application
 52 |         env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
 53 |         // configure watermark interval
 54 |         env.getConfig().setAutoWatermarkInterval(1000L);
 55 | 
 56 |         // ingest sensor stream
 57 |         DataStream<SensorReading> sensorData = env
 58 |                 // SensorSource generates random temperature readings
 59 |                 .addSource(new SensorSource())
 60 |                 // assign timestamps and watermarks which are required for event time
 61 |                 .assignTimestampsAndWatermarks(new SensorTimeAssigner());
 62 | 
 63 |         DataStream<Tuple4<String, Long, Long, Integer>> countsPerThirtySecs = sensorData
 64 |             .keyBy(r -> r.id)
 65 |             // a custom window assigner for 30 seconds tumbling windows
 66 |             .window(new ThirtySecondsWindows())
 67 |             // a custom trigger that fires early (at most) every second
 68 |             .trigger(new OneSecondIntervalTrigger())
 69 |             // count readings per window
 70 |             .process(new CountFunction());
 71 | 
 72 |         countsPerThirtySecs.print();
 73 | 
 74 |         env.execute("Run custom window example");
 75 |     }
 76 | 
 77 |     /**
 78 |      * A custom window that groups events in to 30 second tumbling windows.
 79 |      */
 80 |     public static class ThirtySecondsWindows extends WindowAssigner<Object, TimeWindow> {
 81 | 
 82 |         long windowSize = 30_000L;
 83 | 
 84 |         @Override
 85 |         public Collection<TimeWindow> assignWindows(Object e, long ts, WindowAssignerContext ctx) {
 86 | 
 87 |             // rounding down by 30 seconds
 88 |             long startTime = ts - (ts % windowSize);
 89 |             long endTime = startTime + windowSize;
 90 |             // emitting the corresponding time window
 91 |             return Collections.singletonList(new TimeWindow(startTime, endTime));
 92 |         }
 93 | 
 94 |         @Override
 95 |         public Trigger<Object, TimeWindow> getDefaultTrigger(StreamExecutionEnvironment env) {
 96 |             return EventTimeTrigger.create();
 97 |         }
 98 | 
 99 |         @Override
100 |         public TypeSerializer<TimeWindow> getWindowSerializer(ExecutionConfig executionConfig) {
101 |             return new TimeWindow.Serializer();
102 |         }
103 | 
104 |         @Override
105 |         public boolean isEventTime() {
106 |             return true;
107 |         }
108 |     }
109 | 
110 |     /**
111 |      * A trigger thet fires early. The trigger fires at most every second.
112 |      */
113 |     public static class OneSecondIntervalTrigger extends Trigger<SensorReading, TimeWindow> {
114 | 
115 |         @Override
116 |         public TriggerResult onElement(SensorReading r, long ts, TimeWindow w, TriggerContext ctx) throws Exception {
117 |             // firstSeen will be false if not set yet
118 |             ValueState<Boolean> firstSeen = ctx.getPartitionedState(
119 |                 new ValueStateDescriptor<>("firstSeen", Types.BOOLEAN));
120 | 
121 |             // register initial timer only for first element
122 |             if (firstSeen.value() == null) {
123 |                 // compute time for next early firing by rounding watermark to second
124 |                 long t = ctx.getCurrentWatermark() + (1000 - (ctx.getCurrentWatermark() % 1000));
125 |                 ctx.registerEventTimeTimer(t);
126 |                 // register timer for the end of the window
127 |                 ctx.registerEventTimeTimer(w.getEnd());
128 |                 firstSeen.update(true);
129 |             }
130 |             // Continue. Do not evaluate window per element
131 |             return TriggerResult.CONTINUE;
132 |         }
133 | 
134 |         @Override
135 |         public TriggerResult onEventTime(long ts, TimeWindow w, TriggerContext ctx) throws Exception {
136 |             if (ts == w.getEnd()) {
137 |                 // final evaluation and purge window state
138 |                 return TriggerResult.FIRE_AND_PURGE;
139 |             } else {
140 |                 // register next early firing timer
141 |                 long t = ctx.getCurrentWatermark() + (1000 - (ctx.getCurrentWatermark() % 1000));
142 |                 if (t < w.getEnd()) {
143 |                     ctx.registerEventTimeTimer(t);
144 |                 }
145 |                 // fire trigger to early evaluate window
146 |                 return TriggerResult.FIRE;
147 |             }
148 |         }
149 | 
150 |         @Override
151 |         public TriggerResult onProcessingTime(long ts, TimeWindow w, TriggerContext ctx) throws Exception {
152 |             // Continue. We don't use processing time timers
153 |             return TriggerResult.CONTINUE;
154 |         }
155 | 
156 |         @Override
157 |         public void clear(TimeWindow w, TriggerContext ctx) throws Exception {
158 |             // clear trigger state
159 |             ValueState<Boolean> firstSeen = ctx.getPartitionedState(
160 |                 new ValueStateDescriptor<>("firstSeen", Types.BOOLEAN));
161 |             firstSeen.clear();
162 |         }
163 |     }
164 | 
165 |     /**
166 |      * A window function that counts the readings per sensor and window.
167 |      * The function emits the sensor id, window end, tiem of function evaluation, and count.
168 |      */
169 |     public static class CountFunction
170 |             extends ProcessWindowFunction<SensorReading, Tuple4<String, Long, Long, Integer>, String, TimeWindow> {
171 | 
172 |         @Override
173 |         public void process(
174 |                 String id,
175 |                 Context ctx,
176 |                 Iterable<SensorReading> readings,
177 |                 Collector<Tuple4<String, Long, Long, Integer>> out) throws Exception {
178 |             // count readings
179 |             int cnt = 0;
180 |             for (SensorReading r : readings) {
181 |                 cnt++;
182 |             }
183 |             // get current watermark
184 |             long evalTime = ctx.currentWatermark();
185 |             // emit result
186 |             out.collect(Tuple4.of(id, ctx.window().getEnd(), evalTime, cnt));
187 |         }
188 |     }
189 | }
190 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <!--
  2 | Licensed to the Apache Software Foundation (ASF) under one
  3 | or more contributor license agreements.  See the NOTICE file
  4 | distributed with this work for additional information
  5 | regarding copyright ownership.  The ASF licenses this file
  6 | to you under the Apache License, Version 2.0 (the
  7 | "License"); you may not use this file except in compliance
  8 | with the License.  You may obtain a copy of the License at
  9 | 
 10 |   http://www.apache.org/licenses/LICENSE-2.0
 11 | 
 12 | Unless required by applicable law or agreed to in writing,
 13 | software distributed under the License is distributed on an
 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 15 | KIND, either express or implied.  See the License for the
 16 | specific language governing permissions and limitations
 17 | under the License.
 18 | -->
 19 | <project xmlns="http://maven.apache.org/POM/4.0.0" 
 20 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 21 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 22 | 	<modelVersion>4.0.0</modelVersion>
 23 | 
 24 | 	<groupId>io.github.streamingwithflink</groupId>
 25 | 	<artifactId>examples-java</artifactId>
 26 | 	<version>1.0</version>
 27 | 	<packaging>jar</packaging>
 28 | 
 29 | 	<name>Java Examples for Stream Processing with Apache Flink</name>
 30 | 	<url>http://streaming-with-flink.github.io/examples</url>
 31 | 
 32 | 	<properties>
 33 | 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 34 | 		<flink.version>1.7.1</flink.version>
 35 | 		<java.version>1.8</java.version>
 36 | 		<scala.binary.version>2.12</scala.binary.version>
 37 | 		<maven.compiler.source>${java.version}</maven.compiler.source>
 38 | 		<maven.compiler.target>${java.version}</maven.compiler.target>
 39 | 	</properties>
 40 | 
 41 | 	<dependencies>
 42 | 		<!-- Apache Flink dependencies -->
 43 | 		<!-- These dependencies are provided, because they should not be packaged into the JAR file. -->
 44 | 		<dependency>
 45 | 			<groupId>org.apache.flink</groupId>
 46 | 			<artifactId>flink-java</artifactId>
 47 | 			<version>${flink.version}</version>
 48 | 			<scope>provided</scope>
 49 | 		</dependency>
 50 | 		<dependency>
 51 | 			<groupId>org.apache.flink</groupId>
 52 | 			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
 53 | 			<version>${flink.version}</version>
 54 | 			<scope>provided</scope>
 55 | 		</dependency>
 56 | 
 57 | 		<!-- runtime-web dependency is need to start web UI from IDE -->
 58 |                 <dependency>
 59 |                         <groupId>org.apache.flink</groupId>
 60 |                         <artifactId>flink-runtime-web_${scala.binary.version}</artifactId>
 61 |                         <version>${flink.version}</version>
 62 |                         <scope>provided</scope>
 63 |                 </dependency>
 64 | 
 65 | 		<!-- queryable-state dependencies are needed for respective examples -->
 66 |                 <dependency>
 67 |                         <groupId>org.apache.flink</groupId>
 68 |                         <artifactId>flink-queryable-state-runtime_${scala.binary.version}</artifactId>
 69 |                         <version>${flink.version}</version>
 70 |                 </dependency>
 71 |                 <dependency>
 72 |                         <groupId>org.apache.flink</groupId>
 73 |                         <artifactId>flink-queryable-state-client-java_${scala.binary.version}</artifactId>
 74 |                         <version>${flink.version}</version>
 75 |                 </dependency>
 76 | 
 77 | 		<!--
 78 |                 Derby is used for a sink connector example.
 79 |                 Example only works in local mode, i.e, it is not possible to submit it to a running cluster.
 80 |                 The dependency is set to provided to reduce the size of the JAR file.
 81 |                 -->
 82 |                 <dependency>
 83 |                         <groupId>org.apache.derby</groupId>
 84 |                         <artifactId>derby</artifactId>
 85 |                         <version>10.13.1.1</version>
 86 |                         <scope>provided</scope>
 87 |                 </dependency>
 88 | 
 89 |                 <!-- Logging -->
 90 |                 <dependency>
 91 |                         <groupId>org.slf4j</groupId>
 92 |                         <artifactId>slf4j-log4j12</artifactId>
 93 |                         <version>1.7.25</version>
 94 |                         <scope>runtime</scope>
 95 |                 </dependency>
 96 |                 <dependency>
 97 |                         <groupId>log4j</groupId>
 98 |                         <artifactId>log4j</artifactId>
 99 |                         <version>1.2.17</version>
100 |                         <scope>runtime</scope>
101 |                 </dependency>
102 | 	</dependencies>
103 | 
104 | 	<build>
105 | 		<plugins>
106 | 			<!-- Java Compiler -->
107 | 			<plugin>
108 | 				<groupId>org.apache.maven.plugins</groupId>
109 | 				<artifactId>maven-compiler-plugin</artifactId>
110 | 				<version>3.1</version>
111 | 				<configuration>
112 | 					<source>${java.version}</source>
113 | 					<target>${java.version}</target>
114 | 				</configuration>
115 | 			</plugin>
116 | 
117 | 			<!-- We use the maven-shade plugin to create a fat jar that contains all necessary dependencies. -->
118 | 			<!-- Change the value of <mainClass>...</mainClass> if your program entry point changes. -->
119 | 			<plugin>
120 | 				<groupId>org.apache.maven.plugins</groupId>
121 | 				<artifactId>maven-shade-plugin</artifactId>
122 | 				<version>3.0.0</version>
123 | 				<executions>
124 | 					<!-- Run shade goal on package phase -->
125 | 					<execution>
126 | 						<phase>package</phase>
127 | 						<goals>
128 | 							<goal>shade</goal>
129 | 						</goals>
130 | 						<configuration>
131 | 							<artifactSet>
132 | 								<excludes>
133 | 									<exclude>org.apache.flink:force-shading</exclude>
134 | 									<exlude>org.apache.flink:flink-shaded-netty</exlude>
135 |                                                                         <exlude>org.apache.flink:flink-shaded-guava</exlude>
136 | 									<exclude>com.google.code.findbugs:jsr305</exclude>
137 | 									<exclude>org.slf4j:*</exclude>
138 | 									<exclude>log4j:*</exclude>
139 | 								</excludes>
140 | 							</artifactSet>
141 | 							<filters>
142 | 								<filter>
143 | 									<!-- Do not copy the signatures in the META-INF folder.
144 | 									Otherwise, this might cause SecurityExceptions when using the JAR. -->
145 | 									<artifact>*:*</artifact>
146 | 									<excludes>
147 | 										<exclude>META-INF/*.SF</exclude>
148 | 										<exclude>META-INF/*.DSA</exclude>
149 | 										<exclude>META-INF/*.RSA</exclude>
150 | 									</excludes>
151 | 								</filter>
152 | 							</filters>
153 | 							<transformers>
154 | 								<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
155 | 									<mainClass>io.github.streamingwithflink.StreamingJob</mainClass>
156 | 								</transformer>
157 | 							</transformers>
158 | 						</configuration>
159 | 					</execution>
160 | 				</executions>
161 | 			</plugin>
162 | 		</plugins>
163 | 
164 | 		<pluginManagement>
165 | 			<plugins>
166 | 
167 | 				<!-- This improves the out-of-the-box experience in Eclipse by resolving some warnings. -->
168 | 				<plugin>
169 | 					<groupId>org.eclipse.m2e</groupId>
170 | 					<artifactId>lifecycle-mapping</artifactId>
171 | 					<version>1.0.0</version>
172 | 					<configuration>
173 | 						<lifecycleMappingMetadata>
174 | 							<pluginExecutions>
175 | 								<pluginExecution>
176 | 									<pluginExecutionFilter>
177 | 										<groupId>org.apache.maven.plugins</groupId>
178 | 										<artifactId>maven-shade-plugin</artifactId>
179 | 										<versionRange>[3.0.0,)</versionRange>
180 | 										<goals>
181 | 											<goal>shade</goal>
182 | 										</goals>
183 | 									</pluginExecutionFilter>
184 | 									<action>
185 | 										<ignore/>
186 | 									</action>
187 | 								</pluginExecution>
188 | 								<pluginExecution>
189 | 									<pluginExecutionFilter>
190 | 										<groupId>org.apache.maven.plugins</groupId>
191 | 										<artifactId>maven-compiler-plugin</artifactId>
192 | 										<versionRange>[3.1,)</versionRange>
193 | 										<goals>
194 | 											<goal>testCompile</goal>
195 | 											<goal>compile</goal>
196 | 										</goals>
197 | 									</pluginExecutionFilter>
198 | 									<action>
199 | 										<ignore/>
200 | 									</action>
201 | 								</pluginExecution>
202 | 							</pluginExecutions>
203 | 						</lifecycleMappingMetadata>
204 | 					</configuration>
205 | 				</plugin>
206 | 			</plugins>
207 | 		</pluginManagement>
208 | 	</build>
209 | 
210 | 	<!-- This profile helps to make things run out of the box in IntelliJ -->
211 | 	<!-- Its adds Flink's core classes to the runtime class path. -->
212 | 	<!-- Otherwise they are missing in IntelliJ, because the dependency is 'provided' -->
213 | 	<profiles>
214 | 		<profile>
215 | 			<id>add-dependencies-for-IDEA</id>
216 | 
217 | 			<activation>
218 | 				<property>
219 | 					<name>idea.version</name>
220 | 				</property>
221 | 			</activation>
222 | 
223 | 			<dependencies>
224 | 				<dependency>
225 | 					<groupId>org.apache.flink</groupId>
226 | 					<artifactId>flink-java</artifactId>
227 | 					<version>${flink.version}</version>
228 | 					<scope>compile</scope>
229 | 				</dependency>
230 | 				<dependency>
231 | 					<groupId>org.apache.flink</groupId>
232 | 					<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
233 | 					<version>${flink.version}</version>
234 | 					<scope>compile</scope>
235 | 				</dependency>
236 | 				<dependency>
237 |                                         <groupId>org.apache.flink</groupId>
238 |                                         <artifactId>flink-runtime-web_${scala.binary.version}</artifactId>
239 |                                         <version>${flink.version}</version>
240 |                                         <scope>compile</scope>
241 |                                 </dependency>
242 |                                 <dependency>
243 |                                         <groupId>org.apache.derby</groupId>
244 |                                         <artifactId>derby</artifactId>
245 |                                         <version>10.13.1.1</version>
246 |                                         <scope>compile</scope>
247 |                                 </dependency>
248 | 			</dependencies>
249 | 		</profile>
250 | 	</profiles>
251 | 
252 | </project>
253 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "{}"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright {yyyy} {name of copyright owner}
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 


--------------------------------------------------------------------------------