├── .gitignore
├── README.md
├── pom.xml
└── src
    ├── main
        ├── java
        │   └── com
        │   │   └── dataartisans
        │   │       └── cookbook
        │   │           ├── multiaggregate
        │   │               ├── HostMetric.java
        │   │               ├── HostMetricGenerator.java
        │   │               ├── Hostname.java
        │   │               └── MultiLevelAggregation.java
        │   │           └── windows
        │   │               ├── CountWindows.java
        │   │               └── ProcessingTimeWindows.java
        ├── resources
        │   └── log4j.properties
        └── scala
        │   └── com
        │       └── dataartisans
        │           └── cookbook
        │               ├── dynamicstreaming
        │                   ├── data
        │                   │   ├── CompositeKey.scala
        │                   │   ├── Impression.scala
        │                   │   ├── Keyed.scala
        │                   │   └── Program.scala
        │                   ├── functions
        │                   │   ├── DynamicCountWindowFunction.scala
        │                   │   └── DynamicKeyFunction.scala
        │                   └── jobs
        │                   │   └── DynamicStreamingJob.scala
        │               ├── expiringstate
        │                   └── ExpiringStateJob.scala
        │               └── misc
        │                   └── StateTracking.scala
    └── test
        └── scala
            └── exampletest
                └── ExampleTest.scala


/.gitignore:
--------------------------------------------------------------------------------
1 | .idea
2 | cookbook.iml
3 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | ## Flink Cookbook
2 | 
3 | This repository contains common patterns for (Flink) streaming applications. Each pattern should be runnable and contain a description of the use case. The patterns maybe in Scala or Java.


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <!--
  2 | Licensed to the Apache Software Foundation (ASF) under one
  3 | or more contributor license agreements.  See the NOTICE file
  4 | distributed with this work for additional information
  5 | regarding copyright ownership.  The ASF licenses this file
  6 | to you under the Apache License, Version 2.0 (the
  7 | "License"); you may not use this file except in compliance
  8 | with the License.  You may obtain a copy of the License at
  9 | 
 10 |   http://www.apache.org/licenses/LICENSE-2.0
 11 | 
 12 | Unless required by applicable law or agreed to in writing,
 13 | software distributed under the License is distributed on an
 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 15 | KIND, either express or implied.  See the License for the
 16 | specific language governing permissions and limitations
 17 | under the License.
 18 | -->
 19 | <project xmlns="http://maven.apache.org/POM/4.0.0"
 20 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 21 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 22 |     <modelVersion>4.0.0</modelVersion>
 23 | 
 24 |     <groupId>com.dataartisans</groupId>
 25 |     <artifactId>flink-cookbook</artifactId>
 26 |     <version>0.1</version>
 27 |     <packaging>jar</packaging>
 28 | 
 29 |     <name>Flink Cookbook</name>
 30 |     <url>http://www.data-artisans.com</url>
 31 | 
 32 |     <repositories>
 33 |         <repository>
 34 |             <id>apache.snapshots</id>
 35 |             <name>Apache Development Snapshot Repository</name>
 36 |             <url>https://repository.apache.org/content/repositories/snapshots/</url>
 37 |             <releases>
 38 |                 <enabled>false</enabled>
 39 |             </releases>
 40 |             <snapshots>
 41 |                 <enabled>true</enabled>
 42 |             </snapshots>
 43 |         </repository>
 44 |     </repositories>
 45 | 
 46 |     <properties>
 47 |         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 48 |         <flink.version>1.5.2</flink.version>
 49 |         <scala.binary.version>2.11</scala.binary.version>
 50 |         <scala.version>2.11.12</scala.version>
 51 |     </properties>
 52 | 
 53 |     <dependencies>
 54 |         <!-- Apache Flink dependencies -->
 55 |         <!-- These dependencies are provided, because they should not be packaged into the JAR file. -->
 56 |         <dependency>
 57 |             <groupId>org.apache.flink</groupId>
 58 |             <artifactId>flink-scala_${scala.binary.version}</artifactId>
 59 |             <version>${flink.version}</version>
 60 |             <scope>provided</scope>
 61 |         </dependency>
 62 |         <dependency>
 63 |             <groupId>org.apache.flink</groupId>
 64 |             <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
 65 |             <version>${flink.version}</version>
 66 |             <scope>provided</scope>
 67 |         </dependency>
 68 | 
 69 |         <!-- Scala Library, provided by Flink as well. -->
 70 |         <dependency>
 71 |             <groupId>org.scala-lang</groupId>
 72 |             <artifactId>scala-library</artifactId>
 73 |             <version>${scala.version}</version>
 74 |             <scope>provided</scope>
 75 |         </dependency>
 76 | 
 77 |         <!-- Add connector dependencies here. They must be in the default scope (compile). -->
 78 | 
 79 |         <!-- Example:
 80 | 
 81 |         <dependency>
 82 |             <groupId>org.apache.flink</groupId>
 83 |             <artifactId>flink-connector-kafka-0.10_${scala.binary.version}</artifactId>
 84 |             <version>${flink.version}</version>
 85 |         </dependency>
 86 |         -->
 87 | 
 88 |         <!-- Add logging framework, to produce console output when running in the IDE. -->
 89 |         <!-- These dependencies are excluded from the application JAR by default. -->
 90 |         <dependency>
 91 |             <groupId>org.slf4j</groupId>
 92 |             <artifactId>slf4j-log4j12</artifactId>
 93 |             <version>1.7.7</version>
 94 |             <scope>runtime</scope>
 95 |         </dependency>
 96 |         <dependency>
 97 |             <groupId>log4j</groupId>
 98 |             <artifactId>log4j</artifactId>
 99 |             <version>1.2.17</version>
100 |             <scope>runtime</scope>
101 |         </dependency>
102 | 
103 | 
104 |         <dependency>
105 |             <groupId>org.apache.flink</groupId>
106 |             <artifactId>flink-test-utils_${scala.binary.version}</artifactId>
107 |             <version>${flink.version}</version>
108 |             <scope>test</scope>
109 |         </dependency>
110 |     </dependencies>
111 | 
112 |     <build>
113 |         <plugins>
114 |             <!-- We use the maven-shade plugin to create a fat jar that contains all necessary dependencies. -->
115 |             <!-- Change the value of <mainClass>...</mainClass> if your program entry point changes. -->
116 |             <plugin>
117 |                 <groupId>org.apache.maven.plugins</groupId>
118 |                 <artifactId>maven-shade-plugin</artifactId>
119 |                 <version>3.0.0</version>
120 |                 <executions>
121 |                     <!-- Run shade goal on package phase -->
122 |                     <execution>
123 |                         <phase>package</phase>
124 |                         <goals>
125 |                             <goal>shade</goal>
126 |                         </goals>
127 |                         <configuration>
128 |                             <artifactSet>
129 |                                 <excludes>
130 |                                     <exclude>org.apache.flink:force-shading</exclude>
131 |                                     <exclude>com.google.code.findbugs:jsr305</exclude>
132 |                                     <exclude>org.slf4j:*</exclude>
133 |                                     <exclude>log4j:*</exclude>
134 |                                 </excludes>
135 |                             </artifactSet>
136 |                             <filters>
137 |                                 <filter>
138 |                                     <!-- Do not copy the signatures in the META-INF folder.
139 |                                     Otherwise, this might cause SecurityExceptions when using the JAR. -->
140 |                                     <artifact>*:*</artifact>
141 |                                     <excludes>
142 |                                         <exclude>META-INF/*.SF</exclude>
143 |                                         <exclude>META-INF/*.DSA</exclude>
144 |                                         <exclude>META-INF/*.RSA</exclude>
145 |                                     </excludes>
146 |                                 </filter>
147 |                             </filters>
148 |                             <transformers>
149 |                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
150 |                                     <mainClass>org.myorg.quickstart.StreamingJob</mainClass>
151 |                                 </transformer>
152 |                             </transformers>
153 |                         </configuration>
154 |                     </execution>
155 |                 </executions>
156 |             </plugin>
157 | 
158 |             <!-- Java Compiler -->
159 |             <plugin>
160 |                 <groupId>org.apache.maven.plugins</groupId>
161 |                 <artifactId>maven-compiler-plugin</artifactId>
162 |                 <version>3.1</version>
163 |                 <configuration>
164 |                     <source>1.8</source>
165 |                     <target>1.8</target>
166 |                 </configuration>
167 |             </plugin>
168 | 
169 |             <!-- Scala Compiler -->
170 |             <plugin>
171 |                 <groupId>net.alchim31.maven</groupId>
172 |                 <artifactId>scala-maven-plugin</artifactId>
173 |                 <version>3.2.2</version>
174 |                 <executions>
175 |                     <execution>
176 |                         <goals>
177 |                             <goal>compile</goal>
178 |                             <goal>testCompile</goal>
179 |                         </goals>
180 |                     </execution>
181 |                 </executions>
182 |             </plugin>
183 | 
184 |             <!-- Eclipse Scala Integration -->
185 |             <plugin>
186 |                 <groupId>org.apache.maven.plugins</groupId>
187 |                 <artifactId>maven-eclipse-plugin</artifactId>
188 |                 <version>2.8</version>
189 |                 <configuration>
190 |                     <downloadSources>true</downloadSources>
191 |                     <projectnatures>
192 |                         <projectnature>org.scala-ide.sdt.core.scalanature</projectnature>
193 |                         <projectnature>org.eclipse.jdt.core.javanature</projectnature>
194 |                     </projectnatures>
195 |                     <buildcommands>
196 |                         <buildcommand>org.scala-ide.sdt.core.scalabuilder</buildcommand>
197 |                     </buildcommands>
198 |                     <classpathContainers>
199 |                         <classpathContainer>org.scala-ide.sdt.launching.SCALA_CONTAINER</classpathContainer>
200 |                         <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
201 |                     </classpathContainers>
202 |                     <excludes>
203 |                         <exclude>org.scala-lang:scala-library</exclude>
204 |                         <exclude>org.scala-lang:scala-compiler</exclude>
205 |                     </excludes>
206 |                     <sourceIncludes>
207 |                         <sourceInclude>**/*.scala</sourceInclude>
208 |                         <sourceInclude>**/*.java</sourceInclude>
209 |                     </sourceIncludes>
210 |                 </configuration>
211 |             </plugin>
212 |             <plugin>
213 |                 <groupId>org.codehaus.mojo</groupId>
214 |                 <artifactId>build-helper-maven-plugin</artifactId>
215 |                 <version>1.7</version>
216 |                 <executions>
217 |                     <!-- Add src/main/scala to eclipse build path -->
218 |                     <execution>
219 |                         <id>add-source</id>
220 |                         <phase>generate-sources</phase>
221 |                         <goals>
222 |                             <goal>add-source</goal>
223 |                         </goals>
224 |                         <configuration>
225 |                             <sources>
226 |                                 <source>src/main/scala</source>
227 |                             </sources>
228 |                         </configuration>
229 |                     </execution>
230 |                     <!-- Add src/test/scala to eclipse build path -->
231 |                     <execution>
232 |                         <id>add-test-source</id>
233 |                         <phase>generate-test-sources</phase>
234 |                         <goals>
235 |                             <goal>add-test-source</goal>
236 |                         </goals>
237 |                         <configuration>
238 |                             <sources>
239 |                                 <source>src/test/scala</source>
240 |                             </sources>
241 |                         </configuration>
242 |                     </execution>
243 |                 </executions>
244 |             </plugin>
245 |         </plugins>
246 |     </build>
247 | 
248 |     <!-- This profile helps to make things run out of the box in IntelliJ -->
249 |     <!-- Its adds Flink's core classes to the runtime class path. -->
250 |     <!-- Otherwise they are missing in IntelliJ, because the dependency is 'provided' -->
251 |     <profiles>
252 |         <profile>
253 |             <id>add-dependencies-for-IDEA</id>
254 | 
255 |             <activation>
256 |                 <property>
257 |                     <name>idea.version</name>
258 |                 </property>
259 |             </activation>
260 | 
261 |             <dependencies>
262 |                 <dependency>
263 |                     <groupId>org.apache.flink</groupId>
264 |                     <artifactId>flink-scala_${scala.binary.version}</artifactId>
265 |                     <version>${flink.version}</version>
266 |                     <scope>compile</scope>
267 |                 </dependency>
268 |                 <dependency>
269 |                     <groupId>org.apache.flink</groupId>
270 |                     <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
271 |                     <version>${flink.version}</version>
272 |                     <scope>compile</scope>
273 |                 </dependency>
274 |                 <dependency>
275 |                     <groupId>org.scala-lang</groupId>
276 |                     <artifactId>scala-library</artifactId>
277 |                     <version>${scala.version}</version>
278 |                     <scope>compile</scope>
279 |                 </dependency>
280 |             </dependencies>
281 |         </profile>
282 |     </profiles>
283 | 
284 | </project>
285 | 


--------------------------------------------------------------------------------
/src/main/java/com/dataartisans/cookbook/multiaggregate/HostMetric.java:
--------------------------------------------------------------------------------
 1 | package com.dataartisans.cookbook.multiaggregate;
 2 | 
 3 | public class HostMetric {
 4 |   private long timestamp = 0;
 5 |   private Hostname hostname = new Hostname();
 6 |   private String metricName = "";
 7 |   private double value = 0.0;
 8 | 
 9 |   public HostMetric() {
10 | 
11 |   }
12 | 
13 |   public HostMetric(long timestamp, Hostname hostname, String metricName, double value) {
14 |     this.timestamp = timestamp;
15 |     this.hostname = hostname;
16 |     this.metricName = metricName;
17 |     this.value = value;
18 |   }
19 | 
20 |   public long getTimestamp() {
21 |     return timestamp;
22 |   }
23 | 
24 |   public void setTimestamp(long timestamp) {
25 |     this.timestamp = timestamp;
26 |   }
27 | 
28 |   public Hostname getHostname() {
29 |     return hostname;
30 |   }
31 | 
32 |   public void setHostname(Hostname hostname) {
33 |     this.hostname = hostname;
34 |   }
35 | 
36 |   public String getMetricName() {
37 |     return metricName;
38 |   }
39 | 
40 |   public void setMetricName(String metricName) {
41 |     this.metricName = metricName;
42 |   }
43 | 
44 |   public double getValue() {
45 |     return value;
46 |   }
47 | 
48 |   public void setValue(double value) {
49 |     this.value = value;
50 |   }
51 | 
52 |   @Override
53 |   public String toString() {
54 |     return "HostMetric{" +
55 |       "timestamp=" + timestamp +
56 |       ", hostname=" + hostname +
57 |       ", metricName='" + metricName + '\'' +
58 |       ", value=" + value +
59 |       '}';
60 |   }
61 | 
62 | }
63 | 


--------------------------------------------------------------------------------
/src/main/java/com/dataartisans/cookbook/multiaggregate/HostMetricGenerator.java:
--------------------------------------------------------------------------------
 1 | package com.dataartisans.cookbook.multiaggregate;
 2 | 
 3 | import org.apache.flink.streaming.api.functions.source.SourceFunction;
 4 | import org.apache.flink.streaming.api.watermark.Watermark;
 5 | 
 6 | public class HostMetricGenerator implements SourceFunction<HostMetric> {
 7 |   private volatile boolean isRunning = true;
 8 | 
 9 |   private String[] hostnames = {
10 |     "vader",
11 |     "luke",
12 |     "hans",
13 |     "leia",
14 |     "r2-d2",
15 |     "c-3po",
16 |     "obi-wan",
17 |     "lando",
18 |     "yoda",
19 |     "chewbacca"
20 |   };
21 | 
22 |   private String[] domains = {
23 |     "wikipedia.org",
24 |     "google.com",
25 |     "flink.org",
26 |     "starwars.com",
27 |     "gamespot",
28 |     "timeout.com",
29 |     "galactic-voyage.com",
30 |     "screenrant.com",
31 |     "zimbio.com",
32 |     "telegraph.co.uk"
33 |   };
34 | 
35 |   private String[] metrics = {
36 |     "cpu_usage",
37 |     "memory_usage",
38 |     "jvm_heap_usage",
39 |     "gc_time",
40 |     "gc_collections",
41 |     "disk_read_time",
42 |     "disk_write_time",
43 |     "network_read_time",
44 |     "network_write_time",
45 |     "uptime"
46 |   };
47 | 
48 |   @Override
49 |   public void run(SourceContext<HostMetric> ctx) throws Exception {
50 | 
51 |     long timestamp = 0;
52 |     while(isRunning){
53 |       for(int i=0; i<domains.length; i++){
54 |         for(int j=0; j<hostnames.length; j++){
55 |           for(int k=0; k<metrics.length; k++) {
56 |             ctx.collectWithTimestamp(new HostMetric(timestamp, new Hostname(hostnames[j], domains[i]), metrics[k], k), timestamp);
57 |           }
58 |         }
59 |       }
60 |       ctx.emitWatermark(new Watermark(timestamp));
61 |       timestamp++;
62 |     }
63 |   }
64 | 
65 |   @Override
66 |   public void cancel() {
67 |     isRunning = false;
68 |   }
69 | }
70 | 


--------------------------------------------------------------------------------
/src/main/java/com/dataartisans/cookbook/multiaggregate/Hostname.java:
--------------------------------------------------------------------------------
 1 | package com.dataartisans.cookbook.multiaggregate;
 2 | 
 3 | public class Hostname {
 4 |   private String host = "";
 5 |   private String domain = "";
 6 | 
 7 |   public Hostname() {
 8 |   }
 9 | 
10 |   public Hostname(String host, String domain) {
11 |     this.host = host;
12 |     this.domain = domain;
13 |   }
14 | 
15 |   public String getDomain() {
16 |     return domain;
17 |   }
18 | 
19 |   public void setDomain(String domain) {
20 |     this.domain = domain;
21 |   }
22 | 
23 |   public String getHost() {
24 |     return host;
25 |   }
26 | 
27 |   public void setHost(String host) {
28 |     this.host = host;
29 |   }
30 | 
31 |   @Override
32 |   public String toString() {
33 |     return "Hostname{" +
34 |       "host='" + host + '\'' +
35 |       ", domain='" + domain + '\'' +
36 |       '}';
37 |   }
38 | 
39 |   @Override
40 |   public boolean equals(Object o) {
41 |     if (this == o) return true;
42 |     if (o == null || getClass() != o.getClass()) return false;
43 | 
44 |     Hostname hostname = (Hostname) o;
45 | 
46 |     if (!host.equals(hostname.host)) return false;
47 |     return domain.equals(hostname.domain);
48 | 
49 |   }
50 | 
51 |   @Override
52 |   public int hashCode() {
53 |     int result = host.hashCode();
54 |     result = 31 * result + domain.hashCode();
55 |     return result;
56 |   }
57 | }
58 | 


--------------------------------------------------------------------------------
/src/main/java/com/dataartisans/cookbook/multiaggregate/MultiLevelAggregation.java:
--------------------------------------------------------------------------------
 1 | package com.dataartisans.cookbook.multiaggregate;
 2 | 
 3 | import org.apache.flink.api.common.functions.MapFunction;
 4 | import org.apache.flink.streaming.api.TimeCharacteristic;
 5 | import org.apache.flink.streaming.api.datastream.DataStream;
 6 | import org.apache.flink.streaming.api.datastream.DataStreamSource;
 7 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 8 | import org.apache.flink.streaming.api.functions.co.CoFlatMapFunction;
 9 | import org.apache.flink.streaming.api.functions.sink.PrintSinkFunction;
10 | import org.apache.flink.streaming.api.windowing.time.Time;
11 | import org.apache.flink.util.Collector;
12 | 
13 | public class MultiLevelAggregation {
14 | 
15 |   public static void main(String[] args) throws Exception {
16 | 
17 |     // set up the execution environment
18 |     final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
19 |     env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
20 |     //env.setParallelism(1);
21 | 
22 |     DataStreamSource<HostMetric> sourceStream = env.addSource(new HostMetricGenerator());
23 | 
24 |     DataStream<HostMetric> hostStream = sourceStream
25 |       //.filter(hm -> hm.getMetricName().equals("memory_usage"))
26 |       .keyBy("hostname", "metricName")
27 |       .timeWindow(Time.seconds(1))
28 |       .sum("value");
29 | 
30 |     DataStream<HostMetric> domainStream = hostStream
31 |       .keyBy("hostname.domain", "metricName")
32 |       .timeWindow(Time.seconds(1))
33 |       .sum("value")
34 |       .map(elideHost());
35 | 
36 |     DataStream<HostMetric> metricStream = domainStream
37 |       .keyBy("metricName")
38 |       .timeWindow(Time.seconds(1))
39 |       .sum("value")
40 |       .map(elideHostAndDomain());
41 | 
42 |     boolean mergedOutput = false;
43 |     if (mergedOutput) {
44 |       union(hostStream, domainStream, metricStream).addSink(new PrintSinkFunction<>(false));
45 |     } else {
46 |       hostStream.addSink(new PrintSinkFunction(false));
47 |       domainStream.addSink(new PrintSinkFunction<>(false));
48 |       metricStream.addSink(new PrintSinkFunction<>(true));
49 |     }
50 | 
51 |     // execute program
52 |     env.execute("Name");
53 |   }
54 | 
55 |   private static DataStream<HostMetric> union(DataStream<HostMetric> hostStream, DataStream<HostMetric> domainStream, DataStream<HostMetric> metricStream) {
56 |     return hostStream
57 |       .union(domainStream)
58 |       .union(metricStream);
59 |   }
60 | 
61 |   private static MapFunction<HostMetric, HostMetric> elideHost() {
62 |     return hostMetric -> {
63 |       hostMetric.getHostname().setHost("*");
64 |       return hostMetric;
65 |     };
66 |   }
67 | 
68 |   private static MapFunction<HostMetric, HostMetric> elideHostAndDomain() {
69 |     return hostMetric -> {
70 |       hostMetric.getHostname().setHost("*");
71 |       hostMetric.getHostname().setDomain("*");
72 |       return hostMetric;
73 |     };
74 |   }
75 | }
76 | 


--------------------------------------------------------------------------------
/src/main/java/com/dataartisans/cookbook/windows/CountWindows.java:
--------------------------------------------------------------------------------
 1 | package com.dataartisans.cookbook.windows;
 2 | 
 3 | import org.apache.flink.api.java.tuple.Tuple2;
 4 | import org.apache.flink.streaming.api.TimeCharacteristic;
 5 | import org.apache.flink.streaming.api.datastream.DataStreamSource;
 6 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 7 | 
 8 | public class CountWindows {
 9 |   public static void main(String[] args) throws Exception {
10 | 
11 |     // set up the execution environment
12 |     final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
13 | 
14 |     DataStreamSource<Tuple2<String, Integer>> sourceStream = env.fromElements(
15 |       new Tuple2("a", 1),
16 |       new Tuple2("a", 1),
17 |       new Tuple2("a", 1),
18 |       new Tuple2("a", 1),
19 |       new Tuple2("a", 1),
20 |       new Tuple2("b", 2),
21 |       new Tuple2("b", 2),
22 |       new Tuple2("b", 2),
23 |       new Tuple2("b", 2),
24 |       new Tuple2("b", 2));
25 | 
26 |     sourceStream
27 |       .keyBy(0)
28 |       .countWindow(5)
29 |       .sum(1)
30 |       .print();
31 | 
32 |     // execute program
33 |     env.execute("Count Windows");
34 |   }
35 | }
36 | 


--------------------------------------------------------------------------------
/src/main/java/com/dataartisans/cookbook/windows/ProcessingTimeWindows.java:
--------------------------------------------------------------------------------
 1 | package com.dataartisans.cookbook.windows;
 2 | 
 3 | import org.apache.flink.api.common.typeinfo.TypeHint;
 4 | import org.apache.flink.api.java.tuple.Tuple2;
 5 | import org.apache.flink.streaming.api.TimeCharacteristic;
 6 | import org.apache.flink.streaming.api.datastream.DataStreamSource;
 7 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 8 | import org.apache.flink.streaming.api.functions.sink.PrintSinkFunction;
 9 | import org.apache.flink.streaming.api.windowing.time.Time;
10 | 
11 | public class ProcessingTimeWindows {
12 |   public static void main(String[] args) throws Exception {
13 | 
14 |     // set up the execution environment
15 |     final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
16 |     env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
17 |     env.setParallelism(1);
18 | 
19 |     DataStreamSource<Tuple2<String, Integer>> sourceStream = env.fromElements(
20 |       new Tuple2("a", 1),
21 |       new Tuple2("a", 1),
22 |       new Tuple2("a", 1),
23 |       new Tuple2("a", 1),
24 |       new Tuple2("a", 1),
25 |       new Tuple2("b", 2),
26 |       new Tuple2("b", 2),
27 |       new Tuple2("b", 2),
28 |       new Tuple2("b", 2),
29 |       new Tuple2("b", 2));
30 | 
31 |     sourceStream
32 |       .map( t -> {Thread.sleep(500); return t; })
33 |       .returns(new TypeHint<Tuple2<String, Integer>>(){})
34 |       .keyBy(0)
35 |       .timeWindow(Time.seconds(1))
36 |       .sum(1)
37 |       .addSink(new PrintSinkFunction<>(true));
38 | 
39 |     // execute program
40 |     env.execute("Processing Time Windows");
41 |   }
42 | }
43 | 


--------------------------------------------------------------------------------
/src/main/resources/log4j.properties:
--------------------------------------------------------------------------------
 1 | ################################################################################
 2 | #  Licensed to the Apache Software Foundation (ASF) under one
 3 | #  or more contributor license agreements.  See the NOTICE file
 4 | #  distributed with this work for additional information
 5 | #  regarding copyright ownership.  The ASF licenses this file
 6 | #  to you under the Apache License, Version 2.0 (the
 7 | #  "License"); you may not use this file except in compliance
 8 | #  with the License.  You may obtain a copy of the License at
 9 | #
10 | #      http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | #  Unless required by applicable law or agreed to in writing, software
13 | #  distributed under the License is distributed on an "AS IS" BASIS,
14 | #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | #  See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | ################################################################################
18 | 
19 | log4j.rootLogger=INFO, console
20 | 
21 | log4j.appender.console=org.apache.log4j.ConsoleAppender
22 | log4j.appender.console.layout=org.apache.log4j.PatternLayout
23 | log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
24 | 


--------------------------------------------------------------------------------
/src/main/scala/com/dataartisans/cookbook/dynamicstreaming/data/CompositeKey.scala:
--------------------------------------------------------------------------------
1 | package com.dataartisans.cookbook.dynamicstreaming.data
2 | 
3 | case class CompositeKey[KEY](name: String, value: KEY)
4 | 


--------------------------------------------------------------------------------
/src/main/scala/com/dataartisans/cookbook/dynamicstreaming/data/Impression.scala:
--------------------------------------------------------------------------------
1 | package com.dataartisans.cookbook.dynamicstreaming.data
2 | 
3 | case class Impression(tweetId: Long, userId: Long, count: Long)
4 | 


--------------------------------------------------------------------------------
/src/main/scala/com/dataartisans/cookbook/dynamicstreaming/data/Keyed.scala:
--------------------------------------------------------------------------------
1 | package com.dataartisans.cookbook.dynamicstreaming.data
2 | 
3 | case class Keyed[IN, KEY](wrapped: IN, key: CompositeKey[KEY])
4 | 


--------------------------------------------------------------------------------
/src/main/scala/com/dataartisans/cookbook/dynamicstreaming/data/Program.scala:
--------------------------------------------------------------------------------
1 | package com.dataartisans.cookbook.dynamicstreaming.data
2 | 
3 | import org.apache.flink.api.java.functions.KeySelector
4 | 
5 | case class Program[IN, KEY, OUT](
6 |                                   keySelector: KeySelector[IN, KEY],
7 |                                   keyName: String,
8 |                                   windowLength: Long,
9 |                                   windowFunction: Iterable[IN] => OUT)


--------------------------------------------------------------------------------
/src/main/scala/com/dataartisans/cookbook/dynamicstreaming/functions/DynamicCountWindowFunction.scala:
--------------------------------------------------------------------------------
 1 | package com.dataartisans.cookbook.dynamicstreaming.functions
 2 | 
 3 | import com.dataartisans.cookbook.dynamicstreaming.data.{CompositeKey, Program, Keyed}
 4 | import org.apache.flink.streaming.api.functions.co.RichCoFlatMapFunction
 5 | import org.apache.flink.util.Collector
 6 | 
 7 | import scala.collection.mutable
 8 | 
 9 | class DynamicCountWindowFunction[IN, KEY, OUT] extends RichCoFlatMapFunction[Keyed[IN, KEY], Program[IN, KEY, OUT], (CompositeKey[KEY], OUT)] {
10 | 
11 |   private case class KeyState(var count: Long, val items: mutable.Buffer[IN])
12 | 
13 |   // The program to run for each different keyed stream (extend to many)
14 |   private val programs = mutable.Map[String, Program[IN, KEY, OUT]]()
15 |   private val keyState = mutable.Map[CompositeKey[KEY], KeyState]()
16 | 
17 |   override def flatMap1(data: Keyed[IN, KEY], out: Collector[(CompositeKey[KEY], OUT)]): Unit = {
18 | 
19 |     // Get the program to run
20 |     val program = programs(data.key.name)
21 | 
22 |     // Retrieve the state for the current key
23 |     val state = stateForKey(data.key)
24 | 
25 |     // Fire the window if the program-specified count has been reached
26 |     if (state.count < program.windowLength) {
27 |       state.items += data.wrapped
28 |       state.count += 1
29 |     }
30 |     else {
31 |       keyState.remove(data.key)
32 |       // Fire the window function
33 |       out.collect(data.key, program.windowFunction(state.items))
34 |     }
35 |   }
36 | 
37 |   override def flatMap2(program: Program[IN, KEY, OUT], out: Collector[(CompositeKey[KEY], OUT)]): Unit = {
38 |     programs.put(program.keyName, program)
39 |   }
40 | 
41 |   private def stateForKey(key: CompositeKey[KEY]): KeyState = {
42 |     keyState.getOrElseUpdate(key, KeyState(0, emptyBuffer))
43 |   }
44 | 
45 |   private def emptyBuffer = mutable.Buffer.empty[IN]
46 | }
47 | 
48 | 


--------------------------------------------------------------------------------
/src/main/scala/com/dataartisans/cookbook/dynamicstreaming/functions/DynamicKeyFunction.scala:
--------------------------------------------------------------------------------
 1 | package com.dataartisans.cookbook.dynamicstreaming.functions
 2 | 
 3 | import com.dataartisans.cookbook.dynamicstreaming.data.{CompositeKey, Keyed, Program}
 4 | import org.apache.flink.streaming.api.functions.co.RichCoFlatMapFunction
 5 | import org.apache.flink.util.Collector
 6 | 
 7 | import scala.collection.mutable
 8 | 
 9 | class DynamicKeyFunction[IN, KEY, OUT] extends RichCoFlatMapFunction[IN, Program[IN, KEY, OUT], Keyed[IN, KEY]] {
10 | 
11 |   private val programs = mutable.Set[Program[IN, KEY, OUT]]()
12 | 
13 |   override def flatMap1(data: IN, out: Collector[Keyed[IN, KEY]]): Unit = {
14 |     // for each program output a message with keyed according to the needs of the program
15 |     programs
16 |       .foreach(p => {
17 |         out.collect(Keyed(data, CompositeKey(p.keyName, p.keySelector.getKey(data))))
18 |       })
19 |   }
20 | 
21 |   override def flatMap2(program: Program[IN, KEY, OUT], out: Collector[Keyed[IN, KEY]]): Unit = {
22 |     programs.add(program)
23 |   }
24 | }
25 | 


--------------------------------------------------------------------------------
/src/main/scala/com/dataartisans/cookbook/dynamicstreaming/jobs/DynamicStreamingJob.scala:
--------------------------------------------------------------------------------
  1 | package com.dataartisans.cookbook.dynamicstreaming.jobs
  2 | 
  3 | import com.dataartisans.cookbook.dynamicstreaming.data.{Impression, Program}
  4 | import com.dataartisans.cookbook.dynamicstreaming.functions.{DynamicKeyFunction, DynamicCountWindowFunction}
  5 | import org.apache.flink.api.java.functions.KeySelector
  6 | import org.apache.flink.api.scala._
  7 | import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
  8 | import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
  9 | 
 10 | 
 11 | /**
 12 |   * This program demonstrates how we can do "dynamic" streaming.  In this case
 13 |   * data is read from one input source and programs are read from another.  A program
 14 |   * publishes how it needs the input stream *keyed*, as well as a window length, and a
 15 |   * window function to run.  This is just an example.  You would make this more dynamic than
 16 |   * just simple window functions.  You'd expose all of TimelyFlatMap, for example.
 17 |   *
 18 |   * The idea here is this is how you would support many different users simultaneously that
 19 |   * all want to run simple programs over the same input data stream.
 20 |   */
 21 | object DynamicStreamingJob {
 22 |   def main(args: Array[String]) {
 23 |     val env = StreamExecutionEnvironment.getExecutionEnvironment
 24 |     env.setParallelism(1)
 25 | 
 26 |     val stream = env.addSource(dataSource)
 27 | 
 28 |     val programStream = env.addSource(programSource).broadcast
 29 | 
 30 |     stream
 31 |       .connect(programStream)
 32 |       .flatMap(new DynamicKeyFunction)
 33 |       .keyBy(_.key)
 34 |       .connect(programStream)
 35 |       .flatMap(new DynamicCountWindowFunction)
 36 |       .print()
 37 | 
 38 |     env.execute()
 39 |   }
 40 | 
 41 |   /**
 42 |     * Just some test data
 43 |     */
 44 |   def dataSource = {
 45 |     (sc: SourceContext[Impression]) => {
 46 |       while (true) {
 47 |         Thread.sleep(100)
 48 |         sc.collect(Impression(123456789, 24, 10))
 49 |         sc.collect(Impression(123456789, 25, 20))
 50 |       }
 51 |     }
 52 |   }
 53 | 
 54 |   /**
 55 |     * This is an example of a stream of programs.  They do different things
 56 |     * and they provide a keySelector that indicates how they want the input stream
 57 |     * keyed.  These are generated at runtime!  In a real system they would be programs
 58 |     * streamed in -- maybe groovy or other dynamic JVM language.
 59 |     */
 60 |   def programSource = {
 61 |     (sc: SourceContext[Program[Impression, Long, Long]]) => {
 62 | 
 63 |       /**
 64 |         * Key by TweetId, window into blocks of 10, and sum
 65 |         */
 66 |       sc.collect(
 67 |         Program(
 68 |           keySelector = keySelector(_.tweetId),
 69 |           keyName = "tweetId",
 70 |           windowLength = 10,
 71 |           windowFunction = (itor) => {
 72 |             itor.map(_.count).sum  // compute sum -- expect 150
 73 |           }
 74 |         )
 75 |       )
 76 | 
 77 |       /**
 78 |         * Key by userId, window into blocks of 5, and average
 79 |         */
 80 |       sc.collect(
 81 |         Program(
 82 |           keySelector = keySelector(_.userId),
 83 |           keyName = "userId",
 84 |           windowLength = 5,
 85 |           windowFunction = (itor) => {
 86 |             itor.map(_.count).sum / itor.size // compute average -- expect 10 & 20
 87 |           }
 88 |         )
 89 |       )
 90 | 
 91 |       while (true) {
 92 |         Thread.sleep(500)
 93 |       }
 94 |     }
 95 |   }
 96 | 
 97 |   private def keySelector[IN, KEY](function: IN => KEY): KeySelector[IN, KEY] = {
 98 |     new KeySelector[IN, KEY]() {
 99 |       override def getKey(value: IN): KEY = {
100 |         function(value)
101 |       }
102 |     }
103 |   }
104 | }
105 | 


--------------------------------------------------------------------------------
/src/main/scala/com/dataartisans/cookbook/expiringstate/ExpiringStateJob.scala:
--------------------------------------------------------------------------------
 1 | package com.dataartisans.cookbook.expiringstate
 2 | 
 3 | import org.apache.flink.api.common.state.ValueStateDescriptor
 4 | import org.apache.flink.api.common.typeinfo.{TypeHint, TypeInformation}
 5 | import org.apache.flink.streaming.api.TimeCharacteristic
 6 | import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
 7 | import org.apache.flink.streaming.api.scala.{StreamExecutionEnvironment, _}
 8 | import org.apache.flink.streaming.api.watermark.Watermark
 9 | import org.apache.flink.streaming.api.windowing.assigners.GlobalWindows
10 | import org.apache.flink.streaming.api.windowing.triggers.Trigger.TriggerContext
11 | import org.apache.flink.streaming.api.windowing.triggers.{Trigger, TriggerResult}
12 | import org.apache.flink.streaming.api.windowing.windows.GlobalWindow
13 | 
14 | object ExpiringStateJob {
15 |   def main(args: Array[String]) {
16 |     val env = StreamExecutionEnvironment.getExecutionEnvironment
17 |     env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
18 |     env.setParallelism(1)
19 | 
20 |     val stream = env.addSource((ctx: SourceContext[Data]) => {
21 |       val MessageIntervalMs = 100
22 |       var key = 0
23 |       var timestamp = 0L
24 |       while (true) {
25 |         for (i <- 0 until 10) {
26 |           ctx.collectWithTimestamp(Data(timestamp, key, 1), timestamp)
27 |           ctx.emitWatermark(new Watermark(timestamp))
28 |           Thread.sleep(MessageIntervalMs)
29 |           timestamp += MessageIntervalMs
30 |         }
31 |         // keyspace is always growing but we only see each key for 1 second and then never again
32 |         key += 1
33 |       }
34 |     })
35 | 
36 |     // Doing this checkpoints will grow larger and larger as the keyspace grows forever and will
37 |     // never get cleaned up.
38 | //        stream
39 | //          .keyBy("key")
40 | //          .reduce((a, b) => Data(a.timestamp, a.key, a.value + b.value))
41 | //          .print()
42 | 
43 |     // Here's the same thing with a custom trigger that cleans up state for each key after 2 seconds
44 |     stream
45 |       .keyBy("key")
46 |       .window(GlobalWindows.create())
47 |       .trigger(new StateTtlTrigger(2000))
48 |       .reduce((a, b) => Data(a.timestamp, a.key, a.value + b.value))
49 |       .print
50 | 
51 |     env.execute()
52 |   }
53 | 
54 |   case class Data(timestamp: Long, key: Long, value: Long)
55 | 
56 |   /**
57 |     * This trigger fires the window on every element and registers a timer when it's first created
58 |     * that will purge/destory the window after `expireTimeMs`.  Since the state kept is scoped by window
59 |     * and key the state will be cleaned up when the window is purged.
60 |     *
61 |     * This essentially has the effect of creating state with a TTL.
62 |     *
63 |     * @param expireTimeMs
64 |     */
65 |   class StateTtlTrigger(expireTimeMs: Long) extends Trigger[Data, GlobalWindow] {
66 | 
67 |     val stateDesc = new ValueStateDescriptor[Boolean]("timer_created", TypeInformation.of(new TypeHint[Boolean]() {}), false)
68 | 
69 |     override def onElement(t: Data, l: Long, w: GlobalWindow, triggerContext: TriggerContext): TriggerResult = {
70 |       val timerCreated = triggerContext.getPartitionedState(stateDesc)
71 |       if (!timerCreated.value()) {
72 |         println(s"Registering purge timer for key ${t.key}")
73 |         triggerContext.registerEventTimeTimer(t.timestamp + expireTimeMs)
74 |         timerCreated.update(true)
75 |       }
76 |       TriggerResult.FIRE
77 |     }
78 | 
79 |     override def onEventTime(l: Long, w: GlobalWindow, triggerContext: TriggerContext): TriggerResult = {
80 |       println(s"Purging state for key ${(l - expireTimeMs) / 1000} @ time ${l}")
81 |       triggerContext.getPartitionedState(stateDesc).clear()
82 |       triggerContext.deleteEventTimeTimer(l)
83 |       TriggerResult.PURGE
84 |     }
85 | 
86 |     override def onProcessingTime(l: Long, w: GlobalWindow, triggerContext: TriggerContext): TriggerResult = {
87 |       ???
88 |     }
89 | 
90 |     override def clear(w: GlobalWindow, triggerContext: TriggerContext): Unit = ???
91 |   }
92 | }
93 | 


--------------------------------------------------------------------------------
/src/main/scala/com/dataartisans/cookbook/misc/StateTracking.scala:
--------------------------------------------------------------------------------
 1 | package com.dataartisans.cookbook.misc
 2 | 
 3 | import org.apache.flink.api.common.state.ValueStateDescriptor
 4 | import org.apache.flink.api.scala._
 5 | import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
 6 | import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
 7 | import org.apache.flink.streaming.api.windowing.time.Time
 8 | import org.apache.flink.streaming.api.windowing.triggers.Trigger.TriggerContext
 9 | import org.apache.flink.streaming.api.windowing.triggers.{Trigger, TriggerResult}
10 | import org.apache.flink.streaming.api.windowing.windows.TimeWindow
11 | import org.apache.flink.util.Collector
12 | 
13 | case class Event(guid: String, state: String)
14 | 
15 | object ProcessingTimeTriggerWithPeriodicFirings {
16 |   def apply(intervalMs: Long) = {
17 |     new ProcessingTimeTriggerWithPeriodicFirings(intervalMs)
18 |   }
19 | }
20 | 
21 | class ProcessingTimeTriggerWithPeriodicFirings(intervalMs: Long) extends Trigger[Event, TimeWindow] {
22 |   private val startTimeDesc = new ValueStateDescriptor[Long]("start-time", classOf[Long], 0L)
23 | 
24 |   override def onElement(element: Event, timestamp: Long, window: TimeWindow, ctx: TriggerContext): TriggerResult = {
25 |     val startTime = ctx.getPartitionedState(startTimeDesc)
26 |     if (startTime.value == 0) {
27 |       startTime.update(window.getStart)
28 |       ctx.registerProcessingTimeTimer(window.getEnd)
29 |       ctx.registerProcessingTimeTimer(System.currentTimeMillis() + intervalMs)
30 |     }
31 |     TriggerResult.CONTINUE
32 |   }
33 | 
34 |   override def onProcessingTime(time: Long, window: TimeWindow, ctx: TriggerContext): TriggerResult = {
35 |     if (time == window.getEnd) {
36 |       TriggerResult.PURGE
37 |     }
38 |     else {
39 |       ctx.registerProcessingTimeTimer(time + intervalMs)
40 |       TriggerResult.FIRE
41 |     }
42 |   }
43 | 
44 |   override def onEventTime(time: Long, window: TimeWindow, ctx: TriggerContext): TriggerResult = {
45 |     TriggerResult.CONTINUE
46 |   }
47 | 
48 |   override def clear(w: TimeWindow, triggerContext: TriggerContext): Unit = ???
49 | }
50 | 
51 | object StateTracking {
52 |   def main(args: Array[String]) {
53 | 
54 |     val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
55 | 
56 |     // TODO: Remove
57 |     env.setParallelism(1)
58 | 
59 |     val stream = env.addSource((ctx: SourceContext[Event]) => {
60 |       var guid = 1
61 |       while (true) {
62 |         for (i <- 1 to 1) {
63 |           val state = s"state-$i"
64 |           ctx.collect(Event(s"guid-$guid", state))
65 |           Thread.sleep(1000)
66 |         }
67 |         guid = guid + 1
68 |       }
69 |     })
70 | 
71 |     val results = stream
72 |       .keyBy(_.guid)
73 |       .timeWindow(Time.seconds(30))
74 |       .trigger(ProcessingTimeTriggerWithPeriodicFirings(1000))
75 |       .apply(
76 |         (e1, e2) => e2,
77 |         (k, w, i, c: Collector[(String, Long)]) => {
78 |           if (i.head != null) c.collect((i.head.state, 1))
79 |         }
80 |       )
81 |       .keyBy(0)
82 |       .timeWindow(Time.seconds(1))
83 |       .sum(1)
84 | 
85 | 
86 |     results.printToErr()
87 | 
88 |     env.execute("Job")
89 |   }
90 | 
91 | }
92 | 


--------------------------------------------------------------------------------
/src/test/scala/exampletest/ExampleTest.scala:
--------------------------------------------------------------------------------
 1 | package exampletest
 2 | 
 3 | import java.util.concurrent.TimeUnit
 4 | 
 5 | import org.apache.flink.api.scala._
 6 | import org.apache.flink.streaming.api.TimeCharacteristic
 7 | import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks
 8 | import org.apache.flink.streaming.api.functions.sink.SinkFunction
 9 | import org.apache.flink.streaming.api.functions.source.SourceFunction
10 | import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
11 | import org.apache.flink.streaming.api.watermark.Watermark
12 | import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows
13 | import org.apache.flink.streaming.api.windowing.time.Time
14 | import org.apache.flink.test.util.AbstractTestBase
15 | import org.junit.Assert._
16 | import org.junit.Test
17 | 
18 | import scala.collection.mutable
19 | 
20 | /**
21 |   * Tests for Folds over windows. These also test whether OutputTypeConfigurable functions
22 |   * work for windows, because FoldWindowFunction is OutputTypeConfigurable.
23 |   */
24 | class ExampleTest extends AbstractTestBase {
25 | 
26 |   @Test
27 |   def testReduceWindow(): Unit = {
28 |     val env = StreamExecutionEnvironment.getExecutionEnvironment
29 |     env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
30 |     env.setParallelism(1)
31 | 
32 |     val source1 = env.addSource(new SourceFunction[(String, Int)]() {
33 |       def run(ctx: SourceFunction.SourceContext[(String, Int)]) {
34 |         ctx.collect(("a", 0))
35 |         ctx.collect(("a", 1))
36 |         ctx.collect(("a", 2))
37 |         ctx.collect(("b", 3))
38 |         ctx.collect(("b", 4))
39 |         ctx.collect(("b", 5))
40 |         ctx.collect(("a", 6))
41 |         ctx.collect(("a", 7))
42 |         ctx.collect(("a", 8))
43 | 
44 |         // source is finite, so it will have an implicit MAX watermark when it finishes
45 |       }
46 | 
47 |       override def cancel(): Unit = {}
48 |     }).assignTimestampsAndWatermarks(new ExampleTest.Tuple2TimestampExtractor)
49 | 
50 |     source1
51 |       .keyBy(0)
52 |       .window(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS)))
53 |       .reduce((a, b) => (a._1 + b._1, a._2 + b._2))
54 |       .addSink(new SinkFunction[(String, Int)]() {
55 |         def invoke(value: (String, Int)) {
56 |           ExampleTest.testResults += value.toString
57 |         }
58 |       })
59 | 
60 |     env.execute("Reduce Window Test")
61 | 
62 |     val expectedResult = mutable.MutableList(
63 |       "(aaa,3)",
64 |       "(aaa,21)",
65 |       "(bbb,12)")
66 | 
67 |     assertEquals(expectedResult.sorted, ExampleTest.testResults.sorted)
68 |   }
69 | }
70 | 
71 | object ExampleTest {
72 | 
73 |   val testResults = mutable.MutableList[String]()
74 | 
75 |   private class Tuple2TimestampExtractor extends AssignerWithPunctuatedWatermarks[(String, Int)] {
76 | 
77 |     private var currentTimestamp = -1L
78 | 
79 |     override def extractTimestamp(element: (String, Int), previousTimestamp: Long): Long = {
80 |       currentTimestamp = element._2
81 |       currentTimestamp
82 |     }
83 | 
84 |     def checkAndGetNextWatermark(lastElement: (String, Int), extractedTimestamp: Long): Watermark = {
85 |       new Watermark(lastElement._2 - 1)
86 |     }
87 |   }
88 | }
89 | 
90 | 
91 | 


--------------------------------------------------------------------------------