├── .gitignore ├── LICENSE ├── README.md ├── streaming-state-machine ├── README.md ├── pom.xml └── src │ └── main │ ├── resources │ └── log4j.properties │ └── scala │ └── com │ └── dataartisans │ └── flink │ └── example │ └── eventpattern │ ├── Event.scala │ ├── EventStateMachine.scala │ ├── EventsGenerator.scala │ ├── EventsGeneratorSource.scala │ ├── Socket │ ├── SocketGenerator.scala │ └── SocketSource.scala │ ├── StandaloneGeneratorBase.scala │ ├── StreamingDemo.scala │ └── kafka │ ├── EventDeSerializer.scala │ └── KafkaGenerator.scala └── timeout-monitoring ├── pom.xml └── src └── main ├── java └── com │ └── mythingy │ └── streaming │ ├── EventGenerator.java │ ├── EventStreamAnalysis.java │ ├── ProgressEvent.java │ ├── ProgressEventJSONSerializer.java │ └── Result.java └── resources └── log4j.properties /.gitignore: -------------------------------------------------------------------------------- 1 | .cache 2 | scalastyle-output.xml 3 | .classpath 4 | .idea 5 | .metadata 6 | .settings 7 | .project 8 | .version.properties 9 | filter.properties 10 | logs.zip 11 | target 12 | tmp 13 | *.class 14 | *.iml 15 | *.swp 16 | *.jar 17 | *.log 18 | .DS_Store 19 | _site 20 | docs/api 21 | build-target 22 | flink-staging/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/ 23 | atlassian-ide-plugin.xml 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Apache Flink Demo Examples 2 | 3 | This repository contains example applications (for tutorials or demos) for real-time analysis of 4 | streaming data with Apache Flink. 5 | 6 | 7 | ### Streaming event pattern recognition using state machines 8 | 9 | 10 | https://github.com/StephanEwen/flink-demos/tree/master/streaming-state-machine 11 | -------------------------------------------------------------------------------- /streaming-state-machine/README.md: -------------------------------------------------------------------------------- 1 | Example: Event pattern detection with Apache Flink 2 | ================================================== 3 | 4 | This example assumes a scenario inspired by IT security or network intrusion detection. 5 | 6 | Events in streams (generated by devices and services, such as firewalls login-, and 7 | authentication services) are expected to occur in certain patterns. Any deviation from 8 | these patterns indicates an anomaly (attempted intrusion) that the streaming system should 9 | recognize and that should trigger an alert. 10 | 11 | The event patterns are tracked per interacting party (here simplified per source IP address) 12 | and are validated by a state machine. The state machine's states define what possible 13 | events may occur next, and what new states these events will result in. 14 | 15 | The streaming program that analyzes the event stream is depicted in the diagram below. 16 | The core logic is in the `flatMap` function, which runs the state machines per IP address. 17 | 18 | The main class of this example program is `com.dataartisans.flink.streamingdemo.StreamingDemo` 19 | 20 | 21 | NOTE: The source may be Kafka, but to make this example self-contained, it comes with 22 | a data generator source that produces a sample event stream (with occasional anomalies) 23 | and that can be used without the need to have a Kafka installation. 24 | 25 | ``` 26 | [ KAFKA-PART-1] --> source --> partition -+---> flatMap(state machine) --> sink 27 | \/ 28 | /\ 29 | [ KAFKA-PART-2] --> source --> partition -+---> flatMap(state machine) --> sink 30 | ``` 31 | 32 | 33 | The following diagram depicts the state machine used in this example. 34 | 35 | ``` 36 | +----> W ----> Y -----+ 37 | | ^ | +--------> TERM 38 | INITIAL-+ | | | 39 | | | +--> (Z)----+ 40 | +----> X ------+ | ^ | 41 | | | | | 42 | +-----------------+ +--------+ 43 | ``` 44 | 45 | 46 | Feedback for this example can be sent to mailto:sewen@apache.org 47 | 48 | Docs on Apache Flink can be found at http://flink.apache.org 49 | 50 | Check Apache Flink's mailing lists for help on Flink: mailto:user@flink.apache.org 51 | -------------------------------------------------------------------------------- /streaming-state-machine/pom.xml: -------------------------------------------------------------------------------- 1 | 19 | 22 | 23 | 4.0.0 24 | 25 | com.dataartisans.flink 26 | streaming-state-demo 27 | 1.0-SNAPSHOT 28 | jar 29 | 30 | Flink Streaming Demo Job 31 | http://www.data-artisans.com 32 | 33 | 34 | UTF-8 35 | 36 | 37 | 38 | 39 | org.apache.flink 40 | flink-streaming-scala_2.10 41 | 1.2-SNAPSHOT 42 | 43 | 44 | org.apache.flink 45 | flink-connector-kafka-0.8_2.10 46 | 1.2-SNAPSHOT 47 | 48 | 49 | org.apache.flink 50 | flink-connector-elasticsearch_2.10 51 | 1.2-SNAPSHOT 52 | 53 | 54 | 55 | 56 | 57 | 59 | build-jar 60 | 61 | false 62 | 63 | 64 | 65 | org.apache.flink 66 | flink-streaming-scala_2.10 67 | 1.2-SNAPSHOT 68 | provided 69 | 70 | 71 | org.scala-lang 72 | scala-library 73 | 2.10.4 74 | provided 75 | 76 | 77 | 78 | org.slf4j 79 | slf4j-api 80 | 1.7.7 81 | provided 82 | 83 | 84 | 85 | org.slf4j 86 | slf4j-log4j12 87 | 1.7.7 88 | provided 89 | 90 | 91 | 92 | log4j 93 | log4j 94 | 1.2.17 95 | provided 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 106 | 107 | org.apache.maven.plugins 108 | maven-shade-plugin 109 | 2.4.1 110 | 111 | 112 | 113 | package 114 | 115 | shade 116 | 117 | 118 | 119 | 120 | 122 | *:* 123 | 124 | META-INF/*.SF 125 | META-INF/*.DSA 126 | META-INF/*.RSA 127 | 128 | 129 | 130 | 131 | 132 | 133 | com.dataartisans.flink.example.eventpattern.StreamingDemo 134 | 135 | 136 | 137 | false 138 | 139 | 140 | 141 | 142 | 143 | 144 | net.alchim31.maven 145 | scala-maven-plugin 146 | 3.1.4 147 | 148 | 149 | 150 | compile 151 | testCompile 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | org.apache.maven.plugins 160 | maven-eclipse-plugin 161 | 2.8 162 | 163 | true 164 | 165 | org.scala-ide.sdt.core.scalanature 166 | org.eclipse.jdt.core.javanature 167 | 168 | 169 | org.scala-ide.sdt.core.scalabuilder 170 | 171 | 172 | org.scala-ide.sdt.launching.SCALA_CONTAINER 173 | 174 | org.eclipse.jdt.launching.JRE_CONTAINER 175 | 176 | 177 | 178 | org.scala-lang:scala-library 179 | org.scala-lang:scala-compiler 180 | 181 | 182 | **/*.scala 183 | **/*.java 184 | 185 | 186 | 187 | 188 | 189 | 190 | org.codehaus.mojo 191 | build-helper-maven-plugin 192 | 1.7 193 | 194 | 195 | 196 | add-source 197 | generate-sources 198 | 199 | add-source 200 | 201 | 202 | 203 | src/main/scala 204 | 205 | 206 | 207 | 208 | 209 | add-test-source 210 | generate-test-sources 211 | 212 | add-test-source 213 | 214 | 215 | 216 | src/test/scala 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | -------------------------------------------------------------------------------- /streaming-state-machine/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | ################################################################################ 18 | 19 | log4j.rootLogger=INFO, console 20 | 21 | log4j.appender.console=org.apache.log4j.ConsoleAppender 22 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 23 | log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n 24 | 25 | log4j.logger.org.apache.flink.streaming.connectors.kafka.api.persistent.PersistentKafkaSource=OFF 26 | -------------------------------------------------------------------------------- /streaming-state-machine/src/main/scala/com/dataartisans/flink/example/eventpattern/Event.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Data Artisans GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.dataartisans.flink.example.eventpattern 18 | 19 | /** 20 | * Data type for events, consisting of the originating IP address and an event type. 21 | * 22 | * @param sourceAddress The originating address (think 32 bit IPv4 address). 23 | * @param event The event type. 24 | */ 25 | case class Event(sourceAddress: Int, event: Event.EventType) { 26 | 27 | override def toString: String = { 28 | s"Event ${Event.formatAddress(sourceAddress)} : ${Event.eventTypeName(event)}" 29 | } 30 | } 31 | 32 | /** 33 | * Data type for alerts. 34 | * 35 | * @param address The originating address (think 32 bit IPv4 address). 36 | * @param state The state that the event state machine found. 37 | * @param transition The transition that was considered invalid. 38 | */ 39 | case class Alert(address: Int, state: State, transition: Event.EventType) { 40 | 41 | override def toString: String = { 42 | s"ALERT ${Event.formatAddress(address)} : ${state.name} -> ${Event.eventTypeName(transition)}" 43 | } 44 | } 45 | 46 | 47 | /** 48 | * Companion object to the Event type with event type definition and utility methods. 49 | */ 50 | object Event { 51 | 52 | // NOTE: The reason why we are not using Scala Enums is that they are currently not 53 | // recognized by Flink's serialization stack (pending issue) and would be handled 54 | // as objects of a custom dynamic class hierarchy. That is vastly less efficient 55 | // than numbers, so we map the types to integers right now 56 | // 57 | // NOTE: Using a Java enum is efficiently supported right now 58 | 59 | type EventType = Int 60 | 61 | val a : EventType = 1 62 | val b : EventType = 2 63 | val c : EventType = 3 64 | val d : EventType = 4 65 | val e : EventType = 5 66 | val f : EventType = 6 67 | val g : EventType = 7 68 | 69 | /** 70 | * Util method to encode the type name. Part of the workaround to use integers rather 71 | * than an enumeration type. 72 | * 73 | * @param evt The event type. 74 | * @return The string name of the event type 75 | */ 76 | def eventTypeName(evt: EventType): String = { 77 | String.valueOf(('a' + evt - 1).asInstanceOf[Char]) 78 | } 79 | 80 | /** 81 | * Util method to create a string representation of a 32 bit integer representing 82 | * an IPv4 address. 83 | * 84 | * @param address The address, MSB first. 85 | * @return The IP address string. 86 | */ 87 | def formatAddress(address: Int): String = { 88 | val b1 = (address >>> 24) & 0xff 89 | val b2 = (address >>> 16) & 0xff 90 | val b3 = (address >>> 8) & 0xff 91 | val b4 = address & 0xff 92 | 93 | s"$b1.$b2.$b3.$b4" 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /streaming-state-machine/src/main/scala/com/dataartisans/flink/example/eventpattern/EventStateMachine.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Data Artisans GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.dataartisans.flink.example.eventpattern 18 | 19 | import java.util.Random 20 | 21 | import Event._ 22 | 23 | // 24 | // The State Machine implemented by the classes in this file. 25 | // 26 | // 27 | // +----> W ----> Y -----+ 28 | // | ^ | +--------> TERM 29 | // INITIAL-+ | | | 30 | // | | +--> (Z)----+ 31 | // +----> X ------+ | ^ | 32 | // | | | | 33 | // +-----------------+ +--------+ 34 | // 35 | 36 | /** 37 | * A possible transition on a given event into a target state. The transition 38 | * belongs to its originating state and has an associated probability that is 39 | * used to generate random transition events. 40 | * 41 | * @param event The event that triggers the transition. 42 | * @param targetState The target state after the transition. 43 | * @param prob The probability of the transition. 44 | */ 45 | case class Transition(event: EventType, targetState: State, prob: Float) 46 | 47 | 48 | /** 49 | * Base class for states in the state machine. 50 | * 51 | * @param name The name of the state. 52 | * @param transitions The transitions from this state into other states on events. May be empty, 53 | * in which case this state is a terminal state. 54 | */ 55 | sealed class State (val name: String, 56 | private val transitions: Transition*) extends java.io.Serializable { 57 | /** 58 | * Returns the name of the state. 59 | * @return The name of the state. 60 | */ 61 | override def toString: String = name 62 | 63 | /** 64 | * Checks if this state is a terminal state. 65 | * @return True if this state is a terminal state, false otherwise. 66 | */ 67 | def terminal: Boolean = transitions.isEmpty 68 | 69 | /** 70 | * Gets the state after transitioning from this state based on the given event. 71 | * If the transition is valid, this returns the new state, and if this transition 72 | * is illegal, it returns [[InvalidTransition]]. 73 | * 74 | * @param evt The event that defined the transition. 75 | * @return The new state, or [[InvalidTransition]]. 76 | */ 77 | def transition(evt: EventType): State = 78 | transitions 79 | .find( transition => transition.event == evt ) 80 | .map(_.targetState) 81 | .getOrElse(InvalidTransition) 82 | 83 | /** 84 | * Picks a random transition, based on the probabilities of the outgoing transitions 85 | * of this state. 86 | * 87 | * @param rnd The random number generator to use. 88 | * @return A pair ot (transition event , new state). 89 | */ 90 | def randomTransition(rnd: Random): (EventType, State) = { 91 | if (transitions.isEmpty) { 92 | throw new RuntimeException("Cannot transition from state " + name) 93 | } 94 | else { 95 | val p = rnd.nextFloat() 96 | var mass = 0.0f 97 | var transition: Transition = null 98 | transitions.foreach { trans => 99 | mass += trans.prob 100 | if (transition == null && p <= mass) { 101 | transition = trans 102 | } 103 | } 104 | (transition.event, transition.targetState) 105 | } 106 | } 107 | 108 | /** 109 | * Returns an event type that, if applied as a transition on this state, will result 110 | * in an illegal state transition. 111 | * 112 | * @param rnd The random number generator to use. 113 | * @return And event type for an illegal state transition. 114 | */ 115 | def randomInvalidTransition(rnd: Random): EventType = { 116 | var value: EventType = -1 117 | 118 | while (value == -1) { 119 | val candidate = rnd.nextInt(g + 1) 120 | value = if (transition(candidate) == InvalidTransition) candidate else -1 121 | } 122 | value 123 | } 124 | } 125 | 126 | /** 127 | * The initial state from which all state sequences start. 128 | */ 129 | object InitialState extends State("Initial", Transition(a, W, 0.6f), Transition(c, X, 0.4f)) 130 | 131 | object W extends State("W", Transition(b, Y, 1.0f)) 132 | 133 | object X extends State("X", Transition(b, Y, 0.2f), Transition(d, Z, 0.8f)) 134 | 135 | object Y extends State("Y", Transition(e, Z, 1.0f)) 136 | 137 | object Z extends State("Z", Transition(g, TerminalState, 1.0f) ) 138 | 139 | /** 140 | * The terminal state in the state machine. 141 | */ 142 | object TerminalState extends State("Terminal") 143 | 144 | /** 145 | * Special state returned by the State.transition(...) function when attempting 146 | * an illegal state transition. 147 | */ 148 | object InvalidTransition extends State("Invalid Transition") -------------------------------------------------------------------------------- /streaming-state-machine/src/main/scala/com/dataartisans/flink/example/eventpattern/EventsGenerator.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Data Artisans GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.dataartisans.flink.example.eventpattern 18 | 19 | import java.util.Random 20 | 21 | /** 22 | * A generator for events. The generator internally maintains a series of state 23 | * machines (addresses and current associated state) and returns transition events 24 | * from those state machines. Each time the next event is generators, this 25 | * generator picks a random state machine and creates a random transition on that 26 | * state machine. 27 | * 28 | * The generator randomly adds new state machines, and removes state machines as 29 | * soon as they reach the terminal state. This implementation maintains up to 30 | * 1000 state machines concurrently. 31 | */ 32 | class EventsGenerator { 33 | 34 | /** Probability with this generator generates an illegal state transition */ 35 | private[this] val errorProb: Double = 0.0000000 36 | 37 | /** The random number generator */ 38 | private[this] val rnd = new Random() 39 | 40 | /** The currently active state machines */ 41 | private[this] val states = new java.util.LinkedHashMap[Int, State]() 42 | 43 | /** 44 | * Creates a new random event. This method randomly pick either 45 | * one of its currently running state machines, or start a new state machine for 46 | * a random IP address. 47 | * 48 | * With [[errorProb]] probability, the generated event will be from an illegal state 49 | * transition of one of the currently running state machines. 50 | * 51 | * @param minIp The lower bound for the range from which a new IP address may be picked. 52 | * @param maxIp The upper bound for the range from which a new IP address may be picked. 53 | * @return A next random 54 | */ 55 | def next(minIp: Int, maxIp: Int): Event = { 56 | 57 | val p = rnd.nextDouble() 58 | 59 | if (p * 1000 >= states.size()) { 60 | // create a new state machine 61 | val nextIP = rnd.nextInt(maxIp - minIp) + minIp 62 | 63 | if (!states.containsKey(nextIP)) { 64 | val (transition, state) = InitialState.randomTransition(rnd) 65 | states.put(nextIP, state) 66 | Event(nextIP, transition) 67 | } 68 | else { 69 | // collision on IP address, try again 70 | next(minIp, maxIp) 71 | } 72 | } 73 | else { 74 | // pick an existing state machine 75 | 76 | // skip over some elements in the linked map, then take the next 77 | // update it, and insert it at the end 78 | 79 | val numToSkip = Math.min(20, rnd.nextInt(states.size())) 80 | val iter = states.entrySet().iterator() 81 | var i = 0 82 | while (i < numToSkip) { 83 | i += 1 84 | iter.next() 85 | } 86 | 87 | val entry = iter.next() 88 | val address = entry.getKey 89 | val currentState = entry.getValue 90 | iter.remove() 91 | 92 | if (p < errorProb) { 93 | val event = currentState.randomInvalidTransition(rnd) 94 | Event(address, event) 95 | } 96 | else { 97 | val (event, newState) = currentState.randomTransition(rnd) 98 | if (!newState.terminal) { 99 | // reinsert 100 | states.put(address, newState) 101 | } 102 | 103 | Event(address, event) 104 | } 105 | } 106 | } 107 | 108 | /** 109 | * Creates an event for an illegal state transition of one of the internal 110 | * state machines. If the generator has not yet started any state machines 111 | * (for example, because no call to [[next(Int, Int)]] was made, yet), this 112 | * will return [[None]]. 113 | * 114 | * @return An event for a illegal state transition, or [[None]], if not possible. 115 | */ 116 | def nextInvalid(): Option[Event] = { 117 | val iter = states.entrySet().iterator() 118 | if (iter.hasNext) { 119 | val entry = iter.next() 120 | val address = entry.getKey 121 | val currentState = entry.getValue 122 | iter.remove() 123 | 124 | val event = currentState.randomInvalidTransition(rnd) 125 | Some(Event(address, event)) 126 | } 127 | else { 128 | None 129 | } 130 | } 131 | 132 | def numActiveEntries: Int = states.size() 133 | } 134 | -------------------------------------------------------------------------------- /streaming-state-machine/src/main/scala/com/dataartisans/flink/example/eventpattern/EventsGeneratorSource.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Data Artisans GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.dataartisans.flink.example.eventpattern 18 | 19 | import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction 20 | import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext 21 | 22 | /** 23 | * A Flink data stream source that uses the [[EventsGenerator]] to produce a stream 24 | * of events. 25 | */ 26 | class EventsGeneratorSource(val printSpeed: Boolean = false) extends RichParallelSourceFunction[Event] { 27 | 28 | protected[this] var running = true 29 | 30 | protected[this] var count = 0 31 | 32 | override def run(sourceContext: SourceContext[Event]): Unit = { 33 | 34 | if (printSpeed) { 35 | val logger = new Thread("Throughput Logger") { 36 | override def run(): Unit = { 37 | 38 | var lastCount = 0 39 | var lastTimeStamp = System.currentTimeMillis() 40 | 41 | while (running) { 42 | Thread.sleep(1000) 43 | 44 | val ts = System.currentTimeMillis() 45 | val currCount = count 46 | val factor: Double = (ts - lastTimeStamp) / 1000 47 | val perSec = (currCount - lastCount) / factor 48 | lastTimeStamp = ts 49 | lastCount = currCount 50 | 51 | System.out.println(perSec + " / sec") 52 | } 53 | } 54 | } 55 | logger.setDaemon(true) 56 | logger.start() 57 | } 58 | 59 | val generator = new EventsGenerator() 60 | 61 | val range = Integer.MAX_VALUE / getRuntimeContext.getNumberOfParallelSubtasks() 62 | val min = range * getRuntimeContext.getIndexOfThisSubtask() 63 | val max = min + range 64 | 65 | while (running) { 66 | sourceContext.collect(generator.next(min, max)) 67 | count += 1 68 | } 69 | 70 | // set running to false to stop the logger 71 | running = false 72 | } 73 | 74 | override def cancel(): Unit = { 75 | running = false 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /streaming-state-machine/src/main/scala/com/dataartisans/flink/example/eventpattern/Socket/SocketGenerator.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Data Artisans GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.dataartisans.flink.example.eventpattern.Socket 18 | 19 | import java.net.{InetAddress, InetSocketAddress} 20 | import java.nio.{ByteOrder, ByteBuffer} 21 | import java.nio.channels.SocketChannel 22 | 23 | import com.dataartisans.flink.example.eventpattern.{StandaloneGeneratorBase, Event} 24 | import org.apache.flink.util.Collector 25 | 26 | object SocketGenerator extends StandaloneGeneratorBase { 27 | 28 | val BASE_PORT = 51762 29 | 30 | def main(args: Array[String]): Unit = { 31 | 32 | val numPartitions = 4 //args(0).toInt 33 | val collectors = new Array[SocketCollector](numPartitions) 34 | 35 | // create the generator threads 36 | for (i <- 0 until collectors.length) { 37 | collectors(i) = new SocketCollector(BASE_PORT + i) 38 | } 39 | 40 | runGenerator(collectors) 41 | } 42 | } 43 | 44 | class SocketCollector(val port: Int) extends Collector[Event] { 45 | 46 | val channel = SocketChannel.open(new InetSocketAddress(InetAddress.getByName("localhost"), port)) 47 | channel.configureBlocking(true) 48 | channel.finishConnect() 49 | 50 | val buffer = ByteBuffer.allocateDirect(4096).order(ByteOrder.LITTLE_ENDIAN) 51 | 52 | override def collect(t: Event): Unit = { 53 | if (buffer.remaining() < 8) { 54 | buffer.flip() 55 | channel.write(buffer) 56 | buffer.clear() 57 | } 58 | 59 | buffer.putInt(t.sourceAddress) 60 | buffer.putInt(t.event) 61 | } 62 | 63 | override def close(): Unit = { 64 | if (buffer.position() > 0) { 65 | buffer.flip() 66 | channel.write(buffer) 67 | } 68 | channel.close() 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /streaming-state-machine/src/main/scala/com/dataartisans/flink/example/eventpattern/Socket/SocketSource.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Data Artisans GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.dataartisans.flink.example.eventpattern.Socket 18 | 19 | import java.net.{InetAddress, InetSocketAddress} 20 | import java.nio.channels.ServerSocketChannel 21 | import java.nio.{ByteBuffer, ByteOrder} 22 | import com.dataartisans.flink.example.eventpattern.Event 23 | import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction 24 | import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext 25 | 26 | /** 27 | * Flink stream source that reads the Event class elements from a socket. 28 | * It open a server socket and waits for the generator to connect and push data. 29 | */ 30 | class SocketSource extends RichParallelSourceFunction[Event] { 31 | 32 | private[this] var running = true 33 | 34 | override def run(sourceContext: SourceContext[Event]): Unit = { 35 | 36 | val port = SocketGenerator.BASE_PORT + getRuntimeContext.getIndexOfThisSubtask() 37 | 38 | val server = ServerSocketChannel.open() 39 | server.configureBlocking(true) 40 | server.bind(new InetSocketAddress(InetAddress.getByName("localhost"), port)) 41 | 42 | val channel = server.accept() 43 | 44 | val buffer = ByteBuffer.allocateDirect(4096) 45 | buffer.order(ByteOrder.LITTLE_ENDIAN) 46 | 47 | while (running) { 48 | channel.read(buffer) 49 | buffer.flip() 50 | 51 | while (running && buffer.remaining() >= 8) { 52 | val address = buffer.getInt() 53 | val event = buffer.getInt() 54 | 55 | sourceContext.collect(Event(address, event)) 56 | } 57 | buffer.compact() 58 | } 59 | 60 | channel.close() 61 | } 62 | 63 | override def cancel(): Unit = { 64 | running = false 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /streaming-state-machine/src/main/scala/com/dataartisans/flink/example/eventpattern/StandaloneGeneratorBase.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Data Artisans GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.dataartisans.flink.example.eventpattern 18 | 19 | import org.apache.flink.util.Collector 20 | 21 | /** 22 | * Base for standalone generators that use the state machine to create event 23 | * sequences and push them for example into Kafka. 24 | */ 25 | abstract class StandaloneGeneratorBase { 26 | 27 | def runGenerator(collectors: Array[_ <: Collector[Event]]): Unit = { 28 | 29 | val threads = new Array[GeneratorThread](collectors.length) 30 | val range = Integer.MAX_VALUE / collectors.length 31 | 32 | // create the generator threads 33 | for (i <- threads.indices) { 34 | 35 | val min = range * i 36 | val max = min + range 37 | val thread = new GeneratorThread(collectors(i), min, max) 38 | 39 | threads(i) = thread 40 | thread.setName("Generator " + i) 41 | } 42 | 43 | var delay: Long = 2L 44 | var nextErroneous: Int = 0 45 | var running: Boolean = true 46 | 47 | threads.foreach( _.setDelay(delay) ) 48 | threads.foreach( _.start() ) 49 | 50 | // val throughputLogger = new ThroughputLogger(threads) 51 | // throughputLogger.start() 52 | 53 | println("Commands:") 54 | println(" -> q : Quit") 55 | println(" -> + : increase latency") 56 | println(" -> - : decrease latency") 57 | println(" -> e : inject invalid state transition") 58 | 59 | // input loop 60 | 61 | while (running) { 62 | val next: Int = System.in.read() 63 | 64 | next match { 65 | case 'q' => 66 | println("Quitting...") 67 | running = false 68 | 69 | case 'e' => 70 | println("Injecting erroneous transition ...") 71 | threads(nextErroneous).sendInvalidStateTransition() 72 | nextErroneous = (nextErroneous + 1) % threads.length 73 | 74 | case '+' => 75 | delay = Math.max(delay * 2, 1) 76 | println("Delay is " + delay) 77 | threads.foreach( _.setDelay(delay) ) 78 | 79 | case '-' => 80 | delay /= 2 81 | println("Delay is " + delay) 82 | threads.foreach( _.setDelay(delay) ) 83 | 84 | case _ => 85 | } 86 | } 87 | 88 | // shutdown 89 | // throughputLogger.shutdown() 90 | threads.foreach( _.shutdown() ) 91 | threads.foreach( _.join() ) 92 | } 93 | } 94 | 95 | 96 | 97 | /** 98 | * A thread running a [[EventsGenerator]] and pushes generated events to the given collector 99 | * (such as Kafka / Socket / ...) 100 | * 101 | * @param out The collector to push the generated records to. 102 | * @param minAddress The lower bound for the range from which a new IP address may be picked. 103 | * @param maxAddress The upper bound for the range from which a new IP address may be picked. 104 | */ 105 | class GeneratorThread(private[this] val out: Collector[Event], 106 | private[this] val minAddress: Int, 107 | private[this] val maxAddress: Int) extends Thread { 108 | 109 | private[this] var runningThread: Thread = _ 110 | 111 | private[this] var delay: Long = 0 112 | 113 | private[this] var count: Long = 0 114 | 115 | private[this] var running: Boolean = true 116 | 117 | private[this] var injectInvalid: Boolean = false 118 | 119 | 120 | override def run(): Unit = { 121 | runningThread = Thread.currentThread() 122 | val generator = new EventsGenerator() 123 | 124 | while (running) { 125 | if (injectInvalid) { 126 | injectInvalid = false 127 | generator.nextInvalid() match { 128 | case Some(evt) => out.collect(evt) 129 | case None => 130 | } 131 | } 132 | else { 133 | out.collect(generator.next(minAddress, maxAddress)) 134 | } 135 | 136 | count += 1 137 | 138 | // sleep the delay to throttle 139 | if (delay > 0) { 140 | try { 141 | Thread.sleep(delay) 142 | } 143 | catch { 144 | case e: InterruptedException => 145 | } 146 | } 147 | } 148 | } 149 | 150 | def currentCount: Long = count 151 | 152 | def shutdown(): Unit = { 153 | running = false 154 | 155 | if (runningThread != null) { 156 | runningThread.interrupt() 157 | } 158 | } 159 | 160 | def setDelay(delay: Long): Unit = { 161 | this.delay = delay 162 | } 163 | 164 | def sendInvalidStateTransition(): Unit = { 165 | this.injectInvalid = true 166 | } 167 | } 168 | 169 | /** 170 | * Thread that periodically print the number of elements generated per second. 171 | * 172 | * @param generators The generator threads whose aggregate throughput should be logged. 173 | */ 174 | class ThroughputLogger(private[this] val generators: Array[GeneratorThread]) extends Thread { 175 | 176 | private[this] var running = true 177 | 178 | override def run(): Unit = { 179 | var lastCount: Long = 0L 180 | var lastTimeStamp: Long = System.currentTimeMillis() 181 | 182 | while (running) { 183 | Thread.sleep(1000) 184 | 185 | val ts = System.currentTimeMillis() 186 | val currCount: Long = generators.foldLeft(0L)( (v, gen) => v + gen.currentCount) 187 | val factor: Double = (ts - lastTimeStamp) / 1000 188 | val perSec: Double = (currCount - lastCount) / factor 189 | lastTimeStamp = ts 190 | lastCount = currCount 191 | 192 | System.out.println(perSec + " / sec") 193 | } 194 | } 195 | 196 | def shutdown(): Unit = { 197 | running = false 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /streaming-state-machine/src/main/scala/com/dataartisans/flink/example/eventpattern/StreamingDemo.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Data Artisans GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.dataartisans.flink.example.eventpattern 18 | 19 | import java.text.SimpleDateFormat 20 | import java.util 21 | import java.util.{Calendar, Properties, UUID} 22 | 23 | import com.dataartisans.flink.example.eventpattern.kafka.EventDeSerializer 24 | 25 | import org.apache.flink.api.common.functions.{RuntimeContext, RichFlatMapFunction} 26 | import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor} 27 | import org.apache.flink.configuration.Configuration 28 | import org.apache.flink.streaming.api.scala._ 29 | import org.apache.flink.streaming.connectors.elasticsearch.{IndexRequestBuilder, ElasticsearchSink} 30 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer08 31 | import org.apache.flink.util.Collector 32 | 33 | import org.elasticsearch.action.index.IndexRequest 34 | import org.elasticsearch.client.Requests 35 | 36 | /** 37 | * Demo streaming program that receives (or generates) a stream of events and evaluates 38 | * a state machine (per originating IP address) to validate that the events follow 39 | * the state machine's rules. 40 | */ 41 | object StreamingDemo { 42 | 43 | def main(args: Array[String]): Unit = { 44 | 45 | // create the environment to create streams and configure execution 46 | val env = StreamExecutionEnvironment.getExecutionEnvironment 47 | env.enableCheckpointing(5000) 48 | 49 | // data stream from kafka topic. 50 | val kafkaProps = new Properties() 51 | kafkaProps.setProperty("zookeeper.connect", "localhost:2181") 52 | kafkaProps.setProperty("bootstrap.servers", "localhost:9092") 53 | kafkaProps.setProperty("group.id", UUID.randomUUID().toString) 54 | kafkaProps.setProperty("auto.commit.enable", "false") 55 | kafkaProps.setProperty("auto.offset.reset", "largest") 56 | 57 | val elasticConfig = new java.util.HashMap[String, String] 58 | elasticConfig.put("bulk.flush.max.actions", "1") 59 | elasticConfig.put("cluster.name", "elasticsearch") 60 | 61 | 62 | val stream = env.addSource(new FlinkKafkaConsumer08[Event]( 63 | "flink-demo-topic-1", new EventDeSerializer(), kafkaProps)) 64 | val alerts = stream 65 | // partition on the address to make sure equal addresses 66 | // end up in the same state machine flatMap function 67 | .keyBy("sourceAddress") 68 | 69 | // the function that evaluates the state machine over the sequence of events 70 | .flatMap(new StateMachineMapper()) 71 | 72 | 73 | alerts.print() 74 | 75 | alerts.addSink(new ElasticsearchSink[Alert](elasticConfig, new IndexRequestBuilder[Alert]() { 76 | 77 | override def createIndexRequest(element: Alert, ctx: RuntimeContext): IndexRequest = { 78 | 79 | val now: AnyRef = System.currentTimeMillis().asInstanceOf[AnyRef] 80 | 81 | val json = new util.HashMap[String, AnyRef]() 82 | json.put("message", element.toString) 83 | json.put("time", now) 84 | 85 | Requests.indexRequest() 86 | .index("alerts-idx") 87 | .`type`("numalerts") 88 | .source(json) 89 | } 90 | })) 91 | // // output to standard-out 92 | // .print() 93 | 94 | // trigger program execution 95 | env.execute() 96 | } 97 | } 98 | 99 | /** 100 | * The function that maintains the per-IP-address state machines and verifies that the 101 | * events are consistent with the current state of the state machine. If the event is not 102 | * consistent with the current state, the function produces an alert. 103 | */ 104 | class StateMachineMapper extends RichFlatMapFunction[Event, Alert] { 105 | 106 | private[this] var currentState: ValueState[State] = _ 107 | 108 | override def open(config: Configuration): Unit = { 109 | currentState = getRuntimeContext.getState( 110 | new ValueStateDescriptor("state", classOf[State], InitialState)) 111 | } 112 | 113 | override def flatMap(t: Event, out: Collector[Alert]): Unit = { 114 | val state = currentState.value() 115 | val nextState = state.transition(t.event) 116 | 117 | nextState match { 118 | case InvalidTransition => 119 | out.collect(Alert(t.sourceAddress, state, t.event)) 120 | case x if x.terminal => 121 | currentState.clear() 122 | case x => 123 | currentState.update(nextState) 124 | } 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /streaming-state-machine/src/main/scala/com/dataartisans/flink/example/eventpattern/kafka/EventDeSerializer.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Data Artisans GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.dataartisans.flink.example.eventpattern.kafka 18 | 19 | import java.nio.{ByteBuffer, ByteOrder} 20 | 21 | import com.dataartisans.flink.example.eventpattern.Event 22 | import org.apache.flink.api.common.typeinfo.TypeInformation 23 | import org.apache.flink.streaming.api.scala._ 24 | import org.apache.flink.streaming.util.serialization.{DeserializationSchema, SerializationSchema} 25 | 26 | /** 27 | * A serializer / Deserializer for converting [[Event]] objects from/to byte sequences 28 | * for Kafka. 29 | */ 30 | class EventDeSerializer extends DeserializationSchema[Event] with SerializationSchema[Event] { 31 | 32 | override def deserialize(bytes: Array[Byte]): Event = { 33 | val buffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN) 34 | val address: Int = buffer.getInt(0) 35 | val eventType: Int = buffer.getInt(4) 36 | Event(address, eventType) 37 | } 38 | 39 | override def serialize(t: Event): Array[Byte] = { 40 | val byteBuffer = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN) 41 | byteBuffer.putInt(0, t.sourceAddress) 42 | byteBuffer.putInt(4, t.event) 43 | byteBuffer.array() 44 | } 45 | 46 | override def isEndOfStream(t: Event): Boolean = false 47 | 48 | override def getProducedType: TypeInformation[Event] = { 49 | createTypeInformation[Event] 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /streaming-state-machine/src/main/scala/com/dataartisans/flink/example/eventpattern/kafka/KafkaGenerator.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Data Artisans GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.dataartisans.flink.example.eventpattern.kafka 18 | 19 | import java.util.Properties 20 | 21 | import com.dataartisans.flink.example.eventpattern.{StandaloneGeneratorBase, Event} 22 | import kafka.producer.{KeyedMessage, Producer, ProducerConfig} 23 | import kafka.serializer.DefaultEncoder 24 | import org.apache.flink.util.Collector 25 | 26 | /** 27 | * A generator that pushes the data into Kafka. 28 | */ 29 | object KafkaGenerator extends StandaloneGeneratorBase { 30 | 31 | val TOPIC = "flink-demo-topic-1" 32 | 33 | def main(args: Array[String]): Unit = { 34 | 35 | val numPartitions = 1 36 | val collectors = new Array[KafkaCollector](numPartitions) 37 | 38 | // create the generator threads 39 | for (i <- collectors.indices) { 40 | collectors(i) = new KafkaCollector(i) 41 | } 42 | 43 | runGenerator(collectors) 44 | } 45 | } 46 | 47 | class KafkaCollector(private[this] val partition: Int) extends Collector[Event] { 48 | 49 | // create Kafka producer 50 | val properties = new Properties() 51 | properties.put("metadata.broker.list", "localhost:9092") 52 | properties.put("serializer.class", classOf[DefaultEncoder].getCanonicalName) 53 | properties.put("key.serializer.class", classOf[DefaultEncoder].getCanonicalName) 54 | 55 | val config: ProducerConfig = new ProducerConfig(properties) 56 | 57 | val producer = new Producer[Event, Array[Byte]](config) 58 | 59 | val serializer = new EventDeSerializer() 60 | 61 | override def collect(t: Event): Unit = { 62 | val serialized = serializer.serialize(t) 63 | 64 | producer.send(new KeyedMessage[Event, Array[Byte]]( 65 | KafkaGenerator.TOPIC, null, partition, serialized)) 66 | } 67 | 68 | override def close(): Unit = { 69 | producer.close() 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /timeout-monitoring/pom.xml: -------------------------------------------------------------------------------- 1 | 19 | 21 | 4.0.0 22 | 23 | com.mythingy.streaming 24 | flink 25 | 1.0-SNAPSHOT 26 | jar 27 | 28 | Flink Sample Job for timeout monitoring 29 | 30 | 31 | UTF-8 32 | 0.10.0 33 | 34 | 35 | 36 | 37 | apache.snapshots 38 | Apache Development Snapshot Repository 39 | https://repository.apache.org/content/repositories/snapshots/ 40 | 41 | false 42 | 43 | 44 | true 45 | 46 | 47 | 48 | 49 | 72 | 73 | 74 | 75 | org.apache.flink 76 | flink-java 77 | ${flink.version} 78 | 79 | 80 | org.apache.flink 81 | flink-streaming-java 82 | ${flink.version} 83 | 84 | 85 | org.apache.flink 86 | flink-clients 87 | ${flink.version} 88 | 89 | 90 | org.apache.flink 91 | flink-connector-kafka 92 | ${flink.version} 93 | 94 | 95 | 96 | 97 | 98 | 101 | 102 | org.apache.maven.plugins 103 | maven-shade-plugin 104 | 2.4.1 105 | 106 | 107 | 108 | package 109 | 110 | shade 111 | 112 | 113 | 114 | 115 | 118 | org.apache.flink:flink-shaded-* 119 | org.apache.flink:flink-core 120 | org.apache.flink:flink-java 121 | org.apache.flink:flink-scala 122 | org.apache.flink:flink-runtime 123 | org.apache.flink:flink-optimizer 124 | org.apache.flink:flink-clients 125 | org.apache.flink:flink-avro 126 | org.apache.flink:flink-java-examples 127 | org.apache.flink:flink-scala-examples 128 | org.apache.flink:flink-streaming-examples 129 | org.apache.flink:flink-streaming-java 130 | 131 | 137 | org.scala-lang:scala-library 138 | org.scala-lang:scala-compiler 139 | org.scala-lang:scala-reflect 140 | com.amazonaws:aws-java-sdk 141 | com.typesafe.akka:akka-actor_* 142 | com.typesafe.akka:akka-remote_* 143 | com.typesafe.akka:akka-slf4j_* 144 | io.netty:netty-all 145 | io.netty:netty 146 | org.eclipse.jetty:jetty-server 147 | org.eclipse.jetty:jetty-continuation 148 | org.eclipse.jetty:jetty-http 149 | org.eclipse.jetty:jetty-io 150 | org.eclipse.jetty:jetty-util 151 | org.eclipse.jetty:jetty-security 152 | org.eclipse.jetty:jetty-servlet 153 | commons-fileupload:commons-fileupload 154 | org.apache.avro:avro 155 | commons-collections:commons-collections 156 | org.codehaus.jackson:jackson-core-asl 157 | org.codehaus.jackson:jackson-mapper-asl 158 | com.thoughtworks.paranamer:paranamer 159 | org.xerial.snappy:snappy-java 160 | org.apache.commons:commons-compress 161 | org.tukaani:xz 162 | com.esotericsoftware.kryo:kryo 163 | com.esotericsoftware.minlog:minlog 164 | org.objenesis:objenesis 165 | com.twitter:chill_* 166 | com.twitter:chill-java 167 | com.twitter:chill-avro_* 168 | com.twitter:chill-bijection_* 169 | com.twitter:bijection-core_* 170 | com.twitter:bijection-avro_* 171 | commons-lang:commons-lang 172 | junit:junit 173 | de.javakaffee:kryo-serializers 174 | joda-time:joda-time 175 | org.apache.commons:commons-lang3 176 | org.slf4j:slf4j-api 177 | org.slf4j:slf4j-log4j12 178 | log4j:log4j 179 | org.apache.commons:commons-math 180 | org.apache.sling:org.apache.sling.commons.json 181 | commons-logging:commons-logging 182 | org.apache.httpcomponents:httpclient 183 | org.apache.httpcomponents:httpcore 184 | commons-codec:commons-codec 185 | com.fasterxml.jackson.core:jackson-core 186 | com.fasterxml.jackson.core:jackson-databind 187 | com.fasterxml.jackson.core:jackson-annotations 188 | org.codehaus.jettison:jettison 189 | stax:stax-api 190 | com.typesafe:config 191 | org.uncommons.maths:uncommons-maths 192 | com.github.scopt:scopt_* 193 | org.mortbay.jetty:servlet-api 194 | commons-io:commons-io 195 | commons-cli:commons-cli 196 | 197 | 198 | 199 | 200 | org.apache.flink:* 201 | 202 | org/apache/flink/shaded/** 203 | web-docs/** 204 | 205 | 206 | 207 | 209 | *:* 210 | 211 | META-INF/*.SF 212 | META-INF/*.DSA 213 | META-INF/*.RSA 214 | 215 | 216 | 217 | 218 | 219 | 220 | com.mythingy.streaming.EventStreamAnalysis 221 | 222 | 223 | false 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | org.apache.maven.plugins 232 | maven-jar-plugin 233 | 2.5 234 | 235 | 236 | 237 | com.mythingy.streaming.EventStreamAnalysis 238 | 239 | 240 | 241 | 242 | 243 | 244 | org.apache.maven.plugins 245 | maven-compiler-plugin 246 | 3.1 247 | 248 | 1.7 249 | 1.7 250 | 251 | 252 | 253 | 254 | 255 | 256 | 316 | 317 | 318 | 319 | 320 | 322 | build-jar 323 | 324 | false 325 | 326 | 327 | 328 | org.apache.flink 329 | flink-java 330 | ${flink.version} 331 | provided 332 | 333 | 334 | org.apache.flink 335 | flink-streaming-java 336 | ${flink.version} 337 | provided 338 | 339 | 340 | org.apache.flink 341 | flink-clients 342 | ${flink.version} 343 | provided 344 | 345 | 346 | 347 | 348 | 349 | -------------------------------------------------------------------------------- /timeout-monitoring/src/main/java/com/mythingy/streaming/EventGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.mythingy.streaming; 20 | 21 | import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction; 22 | 23 | import java.util.Random; 24 | import java.util.UUID; 25 | import java.util.concurrent.Executors; 26 | import java.util.concurrent.ScheduledExecutorService; 27 | import java.util.concurrent.TimeUnit; 28 | 29 | 30 | public class EventGenerator extends RichParallelSourceFunction { 31 | 32 | private static final long DELAY_BETWEEN_EVENTS = 100; 33 | private static final long MIN_EVENT_TIME_SPAN = 2000; 34 | private static final long MAX_EVENT_TIME_SPAN = 6000; 35 | 36 | private volatile boolean running = true; 37 | 38 | 39 | @Override 40 | public void run(final SourceContext sourceContext) throws Exception { 41 | 42 | final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); 43 | final Random rnd = new Random(); 44 | 45 | try { 46 | while (running) { 47 | // schedule the next sequence of events 48 | 49 | final UUID uuid = UUID.randomUUID(); 50 | final String imageName = generateImageName(); 51 | final String[] types = generateEventTypeSequence(); 52 | 53 | final long delay = MIN_EVENT_TIME_SPAN + rnd.nextLong() % (MAX_EVENT_TIME_SPAN - MIN_EVENT_TIME_SPAN); 54 | final long now = System.currentTimeMillis(); 55 | 56 | // emit the first event 57 | sourceContext.collect(new ProgressEvent(types[0], imageName, now, uuid)); 58 | 59 | // schedule the successive events 60 | for (int i = 1; i < types.length; i++) { 61 | long thisDelay = delay / (types.length - 1) * i; 62 | ProgressEvent evt = new ProgressEvent(types[i], imageName, now + thisDelay, uuid); 63 | executor.schedule(new EventEmitter(sourceContext, evt), thisDelay, TimeUnit.MILLISECONDS); 64 | } 65 | 66 | // throttle the throughput 67 | Thread.sleep(DELAY_BETWEEN_EVENTS); 68 | } 69 | } 70 | finally { 71 | executor.shutdownNow(); 72 | executor.awaitTermination(5, TimeUnit.SECONDS); 73 | } 74 | } 75 | 76 | @Override 77 | public void cancel() { 78 | running = false; 79 | } 80 | 81 | // ------------------------------------------------------------------------ 82 | 83 | private static String generateImageName() { 84 | return "trove-mysql-5.6"; 85 | } 86 | 87 | private static String[] generateEventTypeSequence() { 88 | return new String[] { 89 | "compute.instance.create.start", 90 | "compute.instance.create.end", 91 | "trove.instance.create" 92 | }; 93 | } 94 | 95 | // ------------------------------------------------------------------------ 96 | 97 | private static final class EventEmitter implements Runnable { 98 | 99 | private final SourceContext context; 100 | private final ProgressEvent event; 101 | 102 | private EventEmitter(SourceContext context, ProgressEvent event) { 103 | this.context = context; 104 | this.event = event; 105 | } 106 | 107 | @Override 108 | public void run() { 109 | context.collect(event); 110 | } 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /timeout-monitoring/src/main/java/com/mythingy/streaming/EventStreamAnalysis.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package com.mythingy.streaming; 19 | 20 | import org.apache.flink.api.common.state.OperatorState; 21 | import org.apache.flink.api.java.tuple.Tuple; 22 | import org.apache.flink.streaming.api.datastream.DataStream; 23 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 24 | import org.apache.flink.streaming.api.functions.windowing.WindowFunction; 25 | import org.apache.flink.streaming.api.windowing.assigners.GlobalWindows; 26 | import org.apache.flink.streaming.api.windowing.triggers.Trigger; 27 | import org.apache.flink.streaming.api.windowing.windows.GlobalWindow; 28 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer082; 29 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; 30 | import org.apache.flink.util.Collector; 31 | 32 | import java.io.IOException; 33 | import java.util.Properties; 34 | 35 | 36 | public class EventStreamAnalysis { 37 | 38 | public static final String KAFKA_TOPIC = "vm_event_topic"; 39 | public static final String KAFKA_BROKERS = "localhost:9092"; 40 | public static final String ZOOKEEPER_CONNECTION = "localhost:2181"; 41 | 42 | public static final long TIMEOUT = 5000; 43 | 44 | public static void main(String[] args) throws Exception { 45 | 46 | final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 47 | env.setParallelism(5); 48 | env.enableCheckpointing(1000); 49 | 50 | DataStream eventStream = env.addSource(new EventGenerator()); 51 | 52 | // variant 1: local generator - nothing extra needed 53 | 54 | // variant 2: Kafka as a queue between generator and consumer 55 | // Properties props = new Properties(); 56 | // props.setProperty("bootstrap.servers", KAFKA_BROKERS); 57 | // props.setProperty("zookeeper.connect", ZOOKEEPER_CONNECTION); 58 | // props.setProperty("group.id", "myGroup"); 59 | // 60 | // eventStream.addSink(new FlinkKafkaProducer<>(KAFKA_TOPIC, 61 | // new ProgressEventJSONSerializer(), props)); 62 | // 63 | // eventStream = env.addSource(new FlinkKafkaConsumer082<>( 64 | // KAFKA_TOPIC, new ProgressEventJSONSerializer(), props)); 65 | 66 | // variant 3: RabbitMQ as a queue between event producer and event consumer 67 | 68 | 69 | 70 | DataStream results = eventStream 71 | .keyBy("instanceId") 72 | .window(GlobalWindows.create()) 73 | .trigger(new TimeoutAndClosingTrigger()) 74 | .apply(new WindowFunction() { 75 | 76 | @Override 77 | public void apply(Tuple key, 78 | GlobalWindow window, 79 | Iterable events, 80 | Collector out) { 81 | 82 | // find the last event, which tells us whether we completed OK or expired 83 | ProgressEvent last = null; 84 | for (ProgressEvent evt : events) { 85 | last = evt; 86 | } 87 | 88 | if (last != null) { 89 | boolean expired = !("trove.instance.create".equals(last.eventType)); 90 | out.collect(new Result(last.instanceId, last.eventType, expired)); 91 | } 92 | } 93 | }); 94 | 95 | results.print(); 96 | 97 | env.execute("Timeout Detection"); 98 | } 99 | 100 | // ------------------------------------------------------------------------ 101 | // The trigger that implements the session logic 102 | // ------------------------------------------------------------------------ 103 | 104 | 105 | private static class TimeoutAndClosingTrigger implements Trigger { 106 | 107 | @Override 108 | public TriggerResult onElement(ProgressEvent progressEvent, 109 | long timestamp, 110 | GlobalWindow globalWindow, 111 | TriggerContext triggerContext) throws IOException { 112 | 113 | OperatorState inProgress = triggerContext.getKeyValueState("progress_flag", false); 114 | 115 | if ("compute.instance.create.start".equals(progressEvent.eventType)) { 116 | // first event, schedule session timeout and go on 117 | triggerContext.registerProcessingTimeTimer(progressEvent.timestamp + TIMEOUT); 118 | 119 | // mark the window as in progress. this lets us discard late elements later 120 | inProgress.update(true); 121 | return TriggerResult.CONTINUE; 122 | } 123 | else if (inProgress.value()) { 124 | // the window is still maintained, so we react to these elements 125 | 126 | if ("trove.instance.create".equals(progressEvent.eventType)) { 127 | // last event, completed successfully, evaluate and throw away window 128 | return TriggerResult.FIRE_AND_PURGE; 129 | } 130 | else { 131 | // all else, simply go on 132 | return TriggerResult.CONTINUE; 133 | } 134 | } 135 | else { 136 | // window is no longer in progress, throw away late element 137 | return TriggerResult.PURGE; 138 | } 139 | } 140 | 141 | @Override 142 | public TriggerResult onProcessingTime(long timestamp, 143 | GlobalWindow globalWindow, 144 | TriggerContext triggerContext) throws IOException { 145 | 146 | return TriggerResult.FIRE_AND_PURGE; 147 | } 148 | 149 | @Override 150 | public TriggerResult onEventTime(long timestamp, 151 | GlobalWindow globalWindow, 152 | TriggerContext triggerContext) { 153 | throw new IllegalStateException("Should never have any event time triggers"); 154 | } 155 | } 156 | } 157 | 158 | -------------------------------------------------------------------------------- /timeout-monitoring/src/main/java/com/mythingy/streaming/ProgressEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.mythingy.streaming; 20 | 21 | import java.text.SimpleDateFormat; 22 | import java.util.UUID; 23 | 24 | public class ProgressEvent { 25 | 26 | public String eventType; 27 | 28 | public String imageName; 29 | 30 | public long timestamp; 31 | 32 | public UUID instanceId; 33 | 34 | 35 | public ProgressEvent() { 36 | } 37 | 38 | public ProgressEvent(String eventType, String imageName, long timestamp, UUID instanceId) { 39 | this.eventType = eventType; 40 | this.imageName = imageName; 41 | this.timestamp = timestamp; 42 | this.instanceId = instanceId; 43 | } 44 | 45 | // ------------------------------------------------------------------------ 46 | // Utilities 47 | // ------------------------------------------------------------------------ 48 | 49 | @Override 50 | public boolean equals(Object o) { 51 | if (this == o) { 52 | return true; 53 | } else if (o != null && getClass() == o.getClass()) { 54 | ProgressEvent that = (ProgressEvent) o; 55 | 56 | return timestamp == that.timestamp && 57 | eventType.equals(that.eventType) && 58 | imageName.equals(that.imageName) && 59 | instanceId.equals(that.instanceId); 60 | } else { 61 | return false; 62 | } 63 | } 64 | 65 | @Override 66 | public int hashCode() { 67 | int result = eventType != null ? eventType.hashCode() : 0; 68 | result = 31 * result + (imageName != null ? imageName.hashCode() : 0); 69 | result = 31 * result + (int) (timestamp ^ (timestamp >>> 32)); 70 | result = 31 * result + (instanceId != null ? instanceId.hashCode() : 0); 71 | return result; 72 | } 73 | 74 | @Override 75 | public String toString() { 76 | SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); 77 | String formattedTimestamp = format.format(timestamp); 78 | 79 | return "{\n" + 80 | " \"event_type\": \"" + eventType + "\"\n," + 81 | " \"image_name\": \"" + imageName + "\"\n," + 82 | " \"timestamp\": \"" + formattedTimestamp + "\"\n," + 83 | " \"instance_id\": \"" + instanceId + "\"\n" + 84 | "}"; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /timeout-monitoring/src/main/java/com/mythingy/streaming/ProgressEventJSONSerializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.mythingy.streaming; 20 | 21 | import com.fasterxml.jackson.core.JsonProcessingException; 22 | import com.fasterxml.jackson.databind.ObjectMapper; 23 | 24 | import org.apache.flink.api.common.typeinfo.TypeInformation; 25 | import org.apache.flink.api.java.typeutils.TypeExtractor; 26 | import org.apache.flink.streaming.util.serialization.DeserializationSchema; 27 | import org.apache.flink.streaming.util.serialization.SerializationSchema; 28 | 29 | import java.io.IOException; 30 | 31 | public class ProgressEventJSONSerializer 32 | implements SerializationSchema, DeserializationSchema { 33 | 34 | private final ObjectMapper mapper = new ObjectMapper(); 35 | 36 | @Override 37 | public byte[] serialize(ProgressEvent progressEvent) { 38 | try { 39 | return mapper.writeValueAsBytes(progressEvent); 40 | } catch (JsonProcessingException e) { 41 | throw new RuntimeException(e); 42 | } 43 | } 44 | 45 | @Override 46 | public ProgressEvent deserialize(byte[] bytes) throws IOException { 47 | return mapper.readValue(bytes, ProgressEvent.class); 48 | } 49 | 50 | @Override 51 | public boolean isEndOfStream(ProgressEvent progressEvent) { 52 | return false; 53 | } 54 | 55 | @Override 56 | public TypeInformation getProducedType() { 57 | return TypeExtractor.getForClass(ProgressEvent.class); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /timeout-monitoring/src/main/java/com/mythingy/streaming/Result.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.mythingy.streaming; 20 | 21 | import java.util.UUID; 22 | 23 | public class Result { 24 | 25 | public UUID instanceId; 26 | 27 | public String lastEventType; 28 | 29 | public boolean isExpired; 30 | 31 | 32 | public Result() {} 33 | 34 | public Result(UUID instanceId, String lastEventType, boolean isExpired) { 35 | this.instanceId = instanceId; 36 | this.lastEventType = lastEventType; 37 | this.isExpired = isExpired; 38 | } 39 | 40 | 41 | // ------------------------------------------------------------------------ 42 | // utilities 43 | // ------------------------------------------------------------------------ 44 | 45 | @Override 46 | public String toString() { 47 | return instanceId + " : " + 48 | (isExpired ? "EXPIRED (last status = " + lastEventType + ")": "SUCCESS"); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /timeout-monitoring/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | ################################################################################ 18 | 19 | log4j.rootLogger=INFO, console 20 | 21 | log4j.appender.console=org.apache.log4j.ConsoleAppender 22 | log4j.appender.console.target=System.err 23 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 24 | log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n 25 | --------------------------------------------------------------------------------