├── .gitignore ├── project.clj ├── LICENSE ├── src └── main │ ├── java │ └── io │ │ └── kosong │ │ └── flink │ │ └── clojure │ │ ├── CljTimestampAssigner.java │ │ ├── NippySerializer.java │ │ └── functions │ │ ├── CljSimpleReduceFunction.java │ │ ├── CljKeySelector.java │ │ ├── CljSimpleAggregateFunction.java │ │ ├── CljWindowFunction.java │ │ ├── CljReduceFunction.java │ │ ├── CljMapFunction.java │ │ ├── CljFlatMapFunction.java │ │ ├── CljFilterFunction.java │ │ ├── CljCoMapFunction.java │ │ ├── CljProcessJoinFunction.java │ │ ├── CljCoFlatMapFunction.java │ │ ├── CljSinkFunction.java │ │ ├── CljSourceFunction.java │ │ ├── CljParallelSourceFunction.java │ │ ├── CljProcessFunction.java │ │ ├── CljKeyedProcessFunction.java │ │ ├── CljProcessWindowFunction.java │ │ ├── CljProcessAllWindowFunction.java │ │ ├── CljBroadcastProcessFunction.java │ │ ├── CljAsyncFunction.java │ │ ├── CljKeyedCoProcessFunction.java │ │ ├── CljCoProcessFunction.java │ │ └── CljKeyedBroadcastProcessFunction.java │ └── clojure │ └── io │ └── kosong │ └── flink │ └── clojure │ └── core.clj └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | pom.xml 2 | pom.xml.asc 3 | *.jar 4 | *.class 5 | *.iml 6 | /lib/ 7 | /classes/ 8 | /target/ 9 | /checkouts/ 10 | .lein-deps-sum 11 | .lein-repl-history 12 | .lein-plugins/ 13 | .lein-failures 14 | .nrepl-port 15 | .cpcache/ 16 | /.idea/ 17 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject io.kosong.flink/flink-clojure "0.3.0-SNAPSHOT" 2 | :description "Clojure wrapper for Apache Flink" 3 | :url "https://github.com/keytiong/flink-clojure" 4 | :license {:name "The MIT License" 5 | :url "http://opensource.org/licenses/MIT"} 6 | :plugins [] 7 | 8 | :source-paths ^:replace ["src/main/clojure"] 9 | :test-paths ["src/test/clojure" "src/test/java"] 10 | :resource-paths ["src/main/resource"] 11 | 12 | :java-source-paths ["src/main/java"] 13 | 14 | :dependencies [[org.clojure/clojure "1.11.1"] 15 | [com.taoensso/nippy "3.1.1"] 16 | [org.apache.logging.log4j/log4j-api "2.17.2"] 17 | [com.esotericsoftware.kryo/kryo "2.24.0"]] 18 | 19 | :profiles {:provided 20 | {:dependencies 21 | [[org.apache.flink/flink-streaming-java "1.15.0"]]}} 22 | ) 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 keytiong 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/CljTimestampAssigner.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure; 2 | 3 | import clojure.java.api.Clojure; 4 | import clojure.lang.*; 5 | import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner; 6 | 7 | import java.time.Instant; 8 | 9 | public class CljTimestampAssigner implements SerializableTimestampAssigner { 10 | 11 | private final AFunction extractTimestampFn; 12 | private final Namespace namespace; 13 | 14 | private transient boolean initialized; 15 | 16 | public CljTimestampAssigner(APersistentMap args) { 17 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 18 | extractTimestampFn = (AFunction) Keyword.intern("extractTimestamp").invoke(args); 19 | } 20 | 21 | private void init() { 22 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 23 | initialized = true; 24 | } 25 | 26 | @Override 27 | public long extractTimestamp(T element, long recordTimestamp) { 28 | if (!initialized) { 29 | init(); 30 | } 31 | /* 32 | Instant eventTime = (Instant) Keyword.intern("start-time").invoke(element); 33 | return eventTime.toEpochMilli(); 34 | */ 35 | 36 | return (long) extractTimestampFn.invoke(element, recordTimestamp); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # flink-clojure 2 | 3 | Clojure wrapper for Apache Flink 4 | 5 | ## Usage 6 | 7 | ### Leiningen 8 | 9 | ```clojure 10 | [io.kosong.flink/flink-clojure "0.1.0"] 11 | ``` 12 | 13 | ## Example 14 | 15 | ```clojure 16 | (require '[io.kosong.flink.clojure.core :as fk]) 17 | 18 | (import 'org.apache.flink.streaming.api.environment.StreamExecutionEnvironment) 19 | 20 | (def env (StreamExecutionEnvironment/getExecutionEnvironment)) 21 | 22 | (fk/register-clojure-types env) 23 | 24 | (def word-count-data 25 | ["To be, or not to be,--that is the question:--" 26 | "Whether 'tis nobler in the mind to suffer" 27 | ;... 28 | ]) 29 | 30 | (def tokenizer 31 | (fk/flink-fn 32 | {:fn :flat-map 33 | :returns (fk/type-info-of []) 34 | :flatMap (fn [this line collector] 35 | (doseq [word (-> line .toLowerCase (.split "\\W+"))] 36 | (.collect collector [word 1])))})) 37 | 38 | (def counter 39 | (fk/flink-fn 40 | {:fn :reduce 41 | :returns (fk/type-info-of []) 42 | :reduce (fn [this [word-1 count-1] [word-2 count-2]] 43 | [word-1 (+ count-1 count-2)])})) 44 | 45 | (def word-selector 46 | (fk/flink-fn 47 | {:fn :key-selector 48 | :returns (fk/type-info-of String) 49 | :getKey (fn [this [word count]] 50 | word)})) 51 | 52 | (-> env 53 | (.fromCollection word-count-data) 54 | (.flatMap tokenizer) 55 | (.keyBy word-selector) 56 | (.reduce counter) 57 | (.print)) 58 | 59 | (.execute env "Word Count") 60 | ``` 61 | 62 | ## Build 63 | 64 | ```shell 65 | lein clean 66 | lein install 67 | ``` -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/NippySerializer.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure; 2 | 3 | 4 | import clojure.java.api.Clojure; 5 | import clojure.lang.IFn; 6 | import com.esotericsoftware.kryo.Kryo; 7 | import com.esotericsoftware.kryo.Serializer; 8 | import com.esotericsoftware.kryo.io.Input; 9 | import com.esotericsoftware.kryo.io.Output; 10 | 11 | public class NippySerializer extends Serializer { 12 | 13 | private static final IFn freeze; 14 | private static final IFn thaw; 15 | 16 | static { 17 | Clojure.var("clojure.core/require").invoke(Clojure.read("taoensso.nippy")); 18 | freeze = Clojure.var("taoensso.nippy/freeze"); 19 | thaw = Clojure.var("taoensso.nippy/thaw"); 20 | } 21 | 22 | public NippySerializer() { 23 | } 24 | 25 | @Override 26 | public void write(Kryo kryo, Output output, Object object) { 27 | try { 28 | byte[] barr = (byte[]) freeze.invoke(object); 29 | output.writeInt(barr.length, true); 30 | output.writeBytes(barr); 31 | output.flush(); 32 | } catch (Exception e) { 33 | throw new RuntimeException(e); 34 | } 35 | } 36 | 37 | @Override 38 | public Object read(Kryo kryo, Input input, Class type) { 39 | try { 40 | int bsize = input.readInt(true); 41 | byte[] barr = new byte[bsize]; 42 | input.readBytes(barr); 43 | return thaw.invoke(barr); 44 | } catch (Exception e) { 45 | throw new RuntimeException("Could not create " + type, e); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljSimpleReduceFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | import clojure.java.api.Clojure; 4 | import clojure.lang.APersistentMap; 5 | import clojure.lang.IFn; 6 | import clojure.lang.Keyword; 7 | import clojure.lang.Namespace; 8 | import org.apache.flink.api.common.functions.ReduceFunction; 9 | import org.apache.flink.api.common.typeinfo.TypeInformation; 10 | 11 | public class CljSimpleReduceFunction implements ReduceFunction { 12 | 13 | private final Namespace namespace; 14 | private final IFn initFn; 15 | private final IFn reduceFn; 16 | private final TypeInformation returnType; 17 | 18 | private transient Object state; 19 | private transient boolean initialized; 20 | 21 | public CljSimpleReduceFunction(APersistentMap args) { 22 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 23 | initFn = (IFn) Keyword.intern("init").invoke(args); 24 | reduceFn = (IFn) Keyword.intern("reduce").invoke(args); 25 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 26 | } 27 | 28 | private void init() { 29 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 30 | if (initFn != null) { 31 | state = initFn.invoke(this); 32 | } 33 | initialized = true; 34 | } 35 | 36 | @Override 37 | public T reduce(T value1, T value2) throws Exception { 38 | if (!initialized) { 39 | init(); 40 | } 41 | return (T) reduceFn.invoke(this, value1, value2); 42 | } 43 | 44 | public TypeInformation getProducedType() { 45 | return returnType; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljKeySelector.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | import clojure.java.api.Clojure; 4 | import clojure.lang.*; 5 | import org.apache.flink.api.common.typeinfo.TypeInformation; 6 | import org.apache.flink.api.java.functions.KeySelector; 7 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 8 | 9 | public class CljKeySelector implements KeySelector, ResultTypeQueryable { 10 | 11 | private final TypeInformation returnType; 12 | private final IFn getKeyFn; 13 | private final Namespace namespace; 14 | private final IFn initFn; 15 | 16 | private transient Object state; 17 | private transient boolean initialized; 18 | 19 | public CljKeySelector(APersistentMap args) { 20 | getKeyFn = (IFn) Keyword.intern("getKey").invoke(args); 21 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 22 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 23 | initFn = (IFn) Keyword.intern("init").invoke(args); 24 | } 25 | 26 | public Object state() { 27 | return state; 28 | } 29 | 30 | private void init() { 31 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 32 | if (initFn != null) { 33 | this.state = initFn.invoke(this); 34 | } 35 | initialized = true; 36 | } 37 | 38 | @Override 39 | public KEY getKey(IN value) throws Exception { 40 | if (!initialized) { 41 | init(); 42 | } 43 | return (KEY) getKeyFn.invoke(this, value); 44 | } 45 | 46 | @Override 47 | public TypeInformation getProducedType() { 48 | return returnType; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljSimpleAggregateFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | import clojure.java.api.Clojure; 4 | import clojure.lang.APersistentMap; 5 | import clojure.lang.IFn; 6 | import clojure.lang.Keyword; 7 | import clojure.lang.Namespace; 8 | import org.apache.flink.api.common.functions.AggregateFunction; 9 | import org.apache.flink.api.common.typeinfo.TypeInformation; 10 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 11 | 12 | public class CljSimpleAggregateFunction implements AggregateFunction, ResultTypeQueryable { 13 | 14 | private final Namespace namespace; 15 | private final IFn initFn; 16 | private final IFn createAccumulatorFn; 17 | private final IFn addFn; 18 | private final IFn getResultFn; 19 | private final IFn mergeFn; 20 | private final TypeInformation returnType; 21 | 22 | private transient Object state; 23 | private transient boolean initialized; 24 | 25 | public CljSimpleAggregateFunction(APersistentMap args) { 26 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 27 | initFn = (IFn) Keyword.intern("init").invoke(args); 28 | createAccumulatorFn = (IFn) Keyword.intern("createAccumulator").invoke(args); 29 | addFn = (IFn) Keyword.intern("add").invoke(args); 30 | getResultFn = (IFn) Keyword.intern("getResult").invoke(args); 31 | mergeFn = (IFn) Keyword.intern("merge").invoke(args); 32 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 33 | } 34 | 35 | private void init() { 36 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 37 | if (initFn != null) { 38 | state = initFn.invoke(this); 39 | } 40 | initialized = true; 41 | } 42 | 43 | public Object state() { 44 | return state; 45 | } 46 | 47 | @Override 48 | public ACC createAccumulator() { 49 | if (!initialized) { 50 | init(); 51 | } 52 | return (ACC) createAccumulatorFn.invoke(this); 53 | } 54 | 55 | @Override 56 | public ACC add(IN value, ACC accumulator) { 57 | if (!initialized) { 58 | init(); 59 | } 60 | return (ACC) addFn.invoke(this, value, accumulator); 61 | } 62 | 63 | @Override 64 | public OUT getResult(ACC accumulator) { 65 | if (!initialized) { 66 | init(); 67 | } 68 | return (OUT) getResultFn.invoke(this, accumulator); 69 | } 70 | 71 | @Override 72 | public ACC merge(ACC a, ACC b) { 73 | if (!initialized) { 74 | init(); 75 | } 76 | return (ACC) mergeFn.invoke(this, a, b); 77 | } 78 | 79 | @Override 80 | public TypeInformation getProducedType() { 81 | return returnType; 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljWindowFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | import clojure.java.api.Clojure; 4 | import clojure.lang.APersistentMap; 5 | import clojure.lang.IFn; 6 | import clojure.lang.Keyword; 7 | import clojure.lang.Namespace; 8 | import org.apache.flink.api.common.typeinfo.TypeInformation; 9 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 10 | import org.apache.flink.runtime.state.FunctionInitializationContext; 11 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 12 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 13 | import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction; 14 | import org.apache.flink.streaming.api.windowing.windows.Window; 15 | import org.apache.flink.util.Collector; 16 | 17 | public class CljWindowFunction extends RichWindowFunction 18 | implements ResultTypeQueryable, CheckpointedFunction { 19 | 20 | 21 | private final Namespace namespace; 22 | private final TypeInformation returnType; 23 | private final IFn initFn; 24 | private final IFn initializeStateFn; 25 | private final IFn applyFn; 26 | private final IFn snapshotStateFn; 27 | 28 | private transient Object state; 29 | private transient boolean initialized; 30 | 31 | public CljWindowFunction(APersistentMap args) { 32 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 33 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 34 | initFn = (IFn) Keyword.intern("init").invoke(args); 35 | applyFn = (IFn) Keyword.intern("apply").invoke(args); 36 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 37 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 38 | } 39 | 40 | private void init() { 41 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 42 | if (initFn != null) { 43 | state = initFn.invoke(this); 44 | } 45 | initialized = true; 46 | } 47 | 48 | @Override 49 | public TypeInformation getProducedType() { 50 | return returnType; 51 | } 52 | 53 | @Override 54 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 55 | if (snapshotStateFn != null) { 56 | snapshotStateFn.invoke(this, context); 57 | } 58 | } 59 | 60 | @Override 61 | public void initializeState(FunctionInitializationContext context) throws Exception { 62 | if (!initialized) { 63 | init(); 64 | } 65 | if (initializeStateFn != null) { 66 | initializeStateFn.invoke(this, context); 67 | } 68 | } 69 | 70 | @Override 71 | public void apply(KEY key, W window, Iterable input, Collector out) throws Exception { 72 | applyFn.invoke(this, key, window, input, out); 73 | } 74 | } -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljReduceFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | import clojure.java.api.Clojure; 4 | import clojure.lang.APersistentMap; 5 | import clojure.lang.IFn; 6 | import clojure.lang.Keyword; 7 | import clojure.lang.Namespace; 8 | import org.apache.flink.api.common.functions.RichReduceFunction; 9 | import org.apache.flink.api.common.typeinfo.TypeInformation; 10 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 11 | import org.apache.flink.configuration.Configuration; 12 | import org.apache.flink.runtime.state.FunctionInitializationContext; 13 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 14 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 15 | 16 | public class CljReduceFunction extends RichReduceFunction implements ResultTypeQueryable, CheckpointedFunction { 17 | 18 | private final Namespace namespace; 19 | private final IFn initFn; 20 | private final IFn initializeStateFn; 21 | private final IFn openFn; 22 | private final IFn closeFn; 23 | private final IFn reduceFn; 24 | private final IFn snapshotStateFn; 25 | private final TypeInformation returnType; 26 | 27 | private transient Object state; 28 | private transient boolean initialized; 29 | 30 | public CljReduceFunction(APersistentMap args) { 31 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 32 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 33 | initFn = (IFn) Keyword.intern("init").invoke(args); 34 | openFn = (IFn) Keyword.intern("open").invoke(args); 35 | closeFn = (IFn) Keyword.intern("close").invoke(args); 36 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 37 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 38 | reduceFn = (IFn) Keyword.intern("reduce").invoke(args); 39 | } 40 | 41 | private void init() { 42 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 43 | if (initFn != null) { 44 | state = initFn.invoke(this); 45 | } 46 | initialized = true; 47 | } 48 | 49 | @Override 50 | public void open(Configuration parameters) throws Exception { 51 | super.open(parameters); 52 | if (!initialized) { 53 | init(); 54 | } 55 | if (openFn != null) { 56 | openFn.invoke(this, parameters); 57 | } 58 | } 59 | 60 | @Override 61 | public void close() throws Exception { 62 | if (closeFn != null) { 63 | closeFn.invoke(this); 64 | } 65 | super.close(); 66 | } 67 | 68 | public TypeInformation getProducedType() { 69 | return returnType; 70 | } 71 | 72 | 73 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 74 | if (snapshotStateFn != null) { 75 | snapshotStateFn.invoke(this, context); 76 | } 77 | } 78 | 79 | public void initializeState(FunctionInitializationContext context) throws Exception { 80 | if (!initialized) { 81 | init(); 82 | } 83 | if (initializeStateFn != null) { 84 | initializeStateFn.invoke(this, context); 85 | } 86 | } 87 | 88 | @Override 89 | public T reduce(T value1, T value2) throws Exception { 90 | return (T) reduceFn.invoke(this, value1, value2); 91 | } 92 | } -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljMapFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | import clojure.java.api.Clojure; 4 | import clojure.lang.*; 5 | import org.apache.flink.api.common.functions.RichMapFunction; 6 | import org.apache.flink.api.common.typeinfo.TypeInformation; 7 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 8 | import org.apache.flink.configuration.Configuration; 9 | import org.apache.flink.runtime.state.FunctionInitializationContext; 10 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 11 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 12 | 13 | 14 | public class CljMapFunction extends RichMapFunction 15 | implements ResultTypeQueryable, CheckpointedFunction { 16 | 17 | private transient Object state; 18 | private transient boolean initialized; 19 | private final Namespace namespace; 20 | private final TypeInformation returnType; 21 | private final IFn initFn; 22 | private final IFn openFn; 23 | private final IFn closeFn; 24 | private final IFn initializeStateFn; 25 | private final IFn snapshotStateFn; 26 | private final IFn mapFn; 27 | 28 | public CljMapFunction(APersistentMap args) { 29 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 30 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 31 | initFn = (IFn) Keyword.intern("init").invoke(args); 32 | openFn = (IFn) Keyword.intern("open").invoke(args); 33 | closeFn = (IFn) Keyword.intern("close").invoke(args); 34 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 35 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 36 | mapFn = (IFn) Keyword.intern("map").invoke(args); 37 | } 38 | 39 | @Override 40 | public OUT map(IN value) throws Exception { 41 | return (OUT) mapFn.invoke(this, value); 42 | } 43 | 44 | private void init() { 45 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 46 | if (initFn != null) { 47 | state = initFn.invoke(this); 48 | } 49 | initialized = true; 50 | } 51 | 52 | public Object state() { 53 | return this.state; 54 | } 55 | 56 | @Override 57 | public void open(Configuration parameters) throws Exception { 58 | super.open(parameters); 59 | if (!initialized) { 60 | init(); 61 | } 62 | if (openFn != null) { 63 | openFn.invoke(this, parameters); 64 | } 65 | } 66 | 67 | @Override 68 | public void close() throws Exception { 69 | if (closeFn != null) { 70 | closeFn.invoke(this); 71 | } 72 | super.close(); 73 | } 74 | 75 | @Override 76 | public TypeInformation getProducedType() { 77 | return returnType; 78 | } 79 | 80 | @Override 81 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 82 | if (snapshotStateFn != null) { 83 | snapshotStateFn.invoke(this, context); 84 | } 85 | } 86 | 87 | @Override 88 | public void initializeState(FunctionInitializationContext context) throws Exception { 89 | if (!initialized) { 90 | init(); 91 | } 92 | if (initializeStateFn != null) { 93 | initializeStateFn.invoke(this, context); 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/main/clojure/io/kosong/flink/clojure/core.clj: -------------------------------------------------------------------------------- 1 | (ns io.kosong.flink.clojure.core 2 | (:import (io.kosong.flink.clojure NippySerializer CljTimestampAssigner) 3 | (clojure.lang PersistentVector PersistentHashMap PersistentHashSet PersistentArrayMap PersistentStructMap 4 | PersistentTreeMap PersistentTreeSet PersistentList APersistentMap) 5 | (io.kosong.flink.clojure.functions CljMapFunction CljFlatMapFunction CljKeyedProcessFunction 6 | CljProcessFunction CljSinkFunction CljSourceFunction CljKeySelector 7 | CljReduceFunction CljWindowFunction CljFilterFunction CljCoFlatMapFunction CljProcessWindowFunction CljSimpleReduceFunction CljSimpleAggregateFunction CljBroadcastProcessFunction CljCoProcessFunction CljKeyedBroadcastProcessFunction CljProcessJoinFunction CljKeyedCoProcessFunction CljCoMapFunction CljAsyncFunction CljProcessAllWindowFunction CljParallelSourceFunction) 8 | (org.apache.flink.api.common.typeinfo TypeInformation))) 9 | 10 | (def ^:private clojure-collection-types 11 | [PersistentList 12 | PersistentVector PersistentHashSet #_PersistentTreeSet 13 | PersistentHashMap PersistentArrayMap PersistentStructMap #_PersistentTreeMap]) 14 | 15 | (defn register-clojure-types [env] 16 | (let [exec-config (.getConfig env)] 17 | (doseq [type clojure-collection-types] 18 | (.registerTypeWithKryoSerializer exec-config type NippySerializer)) 19 | env)) 20 | 21 | (defn- ensure-namespace [args] 22 | (if (:ns args) 23 | args 24 | (assoc args :ns *ns*))) 25 | 26 | (def keyword->fn-class 27 | {:source CljSourceFunction 28 | :parallel-source CljParallelSourceFunction 29 | 30 | :map CljMapFunction 31 | :flat-map CljFlatMapFunction 32 | :filter CljFilterFunction 33 | :reduce CljReduceFunction 34 | :process CljProcessFunction 35 | 36 | :key-selector CljKeySelector 37 | :keyed-process CljKeyedProcessFunction 38 | :keyed-broadcast-process CljKeyedBroadcastProcessFunction 39 | :keyed-co-process CljKeyedCoProcessFunction 40 | 41 | :window CljWindowFunction 42 | :process-window CljProcessWindowFunction 43 | :process-all-window CljProcessAllWindowFunction 44 | :simple-reduce CljSimpleReduceFunction 45 | :simple-aggregate CljSimpleAggregateFunction 46 | 47 | :co-map CljCoMapFunction 48 | :co-flat-map CljCoFlatMapFunction 49 | :co-process CljCoProcessFunction 50 | :broadcast-process CljBroadcastProcessFunction 51 | :process-join CljProcessJoinFunction 52 | 53 | :async CljAsyncFunction 54 | 55 | :sink CljSinkFunction}) 56 | 57 | (defn flink-fn [& {:as args}] 58 | (let [fn-class (keyword->fn-class (:fn args)) 59 | ctor (.getConstructor fn-class (into-array Class [APersistentMap])) 60 | args (ensure-namespace args)] 61 | (.newInstance ctor (into-array [args])))) 62 | 63 | (defn type-info-of [class-or-obj] 64 | (let [cls (if (class? class-or-obj) 65 | class-or-obj 66 | (type class-or-obj))] 67 | (TypeInformation/of ^Class cls))) 68 | 69 | (defmacro fdef [name & {:as body}] 70 | `(def ~name (flink-fn ~body))) 71 | 72 | (defn timestamp-assigner [& {:as body}] 73 | (let [body (ensure-namespace body)] 74 | (CljTimestampAssigner. body))) 75 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljFlatMapFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | import clojure.java.api.Clojure; 4 | import clojure.lang.APersistentMap; 5 | import clojure.lang.IFn; 6 | import clojure.lang.Keyword; 7 | import clojure.lang.Namespace; 8 | import org.apache.flink.api.common.functions.RichFlatMapFunction; 9 | import org.apache.flink.api.common.typeinfo.TypeInformation; 10 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 11 | import org.apache.flink.configuration.Configuration; 12 | import org.apache.flink.runtime.state.FunctionInitializationContext; 13 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 14 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 15 | import org.apache.flink.util.Collector; 16 | 17 | public class CljFlatMapFunction extends RichFlatMapFunction implements ResultTypeQueryable, CheckpointedFunction { 18 | 19 | private final Namespace namespace; 20 | private final IFn initFn; 21 | private final IFn openFn; 22 | private final IFn closeFn; 23 | private final IFn initializeStateFn; 24 | private final IFn flatMapFn; 25 | private final IFn snapshotStateFn; 26 | private final TypeInformation returnType; 27 | 28 | private transient Object state; 29 | private transient boolean initialized; 30 | 31 | public CljFlatMapFunction(APersistentMap args) { 32 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 33 | initFn = (IFn) Keyword.intern("init").invoke(args); 34 | openFn = (IFn) Keyword.intern("open").invoke(args); 35 | closeFn = (IFn) Keyword.intern("close").invoke(args); 36 | flatMapFn = (IFn) Keyword.intern("flatMap").invoke(args); 37 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 38 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 39 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 40 | } 41 | 42 | @Override 43 | public void flatMap(IN value, Collector out) throws Exception { 44 | flatMapFn.invoke(this, value, out); 45 | } 46 | 47 | private void init() { 48 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 49 | if (initFn != null) { 50 | state = initFn.invoke(this); 51 | } 52 | initialized = true; 53 | } 54 | 55 | public Object state() { 56 | return this.state; 57 | } 58 | 59 | @Override 60 | public void open(Configuration parameters) throws Exception { 61 | if (!initialized) { 62 | init(); 63 | } 64 | if (openFn != null) { 65 | openFn.invoke(this, parameters); 66 | } 67 | } 68 | 69 | @Override 70 | public void close() throws Exception { 71 | if (closeFn != null) { 72 | closeFn.invoke(this); 73 | } 74 | } 75 | 76 | @Override 77 | public TypeInformation getProducedType() { 78 | return returnType; 79 | } 80 | 81 | @Override 82 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 83 | if (snapshotStateFn != null) { 84 | snapshotStateFn.invoke(this, context); 85 | } 86 | } 87 | 88 | @Override 89 | public void initializeState(FunctionInitializationContext context) throws Exception { 90 | if (!initialized) { 91 | init(); 92 | } 93 | if (initializeStateFn != null) { 94 | initializeStateFn.invoke(this, context); 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljFilterFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | import clojure.java.api.Clojure; 4 | import clojure.lang.APersistentMap; 5 | import clojure.lang.IFn; 6 | import clojure.lang.Keyword; 7 | import clojure.lang.Namespace; 8 | import org.apache.flink.api.common.functions.RichFilterFunction; 9 | import org.apache.flink.api.common.typeinfo.TypeInformation; 10 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 11 | import org.apache.flink.configuration.Configuration; 12 | import org.apache.flink.runtime.state.FunctionInitializationContext; 13 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 14 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 15 | 16 | 17 | public class CljFilterFunction extends RichFilterFunction 18 | implements ResultTypeQueryable, CheckpointedFunction { 19 | 20 | private final Namespace namespace; 21 | private final IFn initFn; 22 | private final IFn openFn; 23 | private final IFn closeFn; 24 | private final IFn initializeStateFn; 25 | private final IFn filterFn; 26 | private final IFn snapshotStateFn; 27 | private final TypeInformation returnType; 28 | 29 | private transient Object state; 30 | private transient boolean initialized; 31 | 32 | public CljFilterFunction(APersistentMap args) { 33 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 34 | initFn = (IFn) Keyword.intern("init").invoke(args); 35 | openFn = (IFn) Keyword.intern("open").invoke(args); 36 | closeFn = (IFn) Keyword.intern("close").invoke(args); 37 | filterFn = (IFn) Keyword.intern("filter").invoke(args); 38 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 39 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 40 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 41 | } 42 | 43 | @Override 44 | public boolean filter(T value) throws Exception { 45 | return (Boolean) filterFn.invoke(this, value); 46 | } 47 | 48 | private void init() { 49 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 50 | if (initFn != null) { 51 | state = initFn.invoke(this); 52 | } 53 | initialized = true; 54 | } 55 | 56 | public Object state() { 57 | return this.state; 58 | } 59 | 60 | @Override 61 | public void open(Configuration parameters) throws Exception { 62 | super.open(parameters); 63 | if (!initialized) { 64 | init(); 65 | } 66 | if (openFn != null) { 67 | openFn.invoke(this, parameters); 68 | } 69 | } 70 | 71 | @Override 72 | public void close() throws Exception { 73 | if (closeFn != null) { 74 | closeFn.invoke(this); 75 | } 76 | super.close(); 77 | } 78 | 79 | @Override 80 | public TypeInformation getProducedType() { 81 | return returnType; 82 | } 83 | 84 | @Override 85 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 86 | if (snapshotStateFn != null) { 87 | snapshotStateFn.invoke(this, context); 88 | } 89 | } 90 | 91 | @Override 92 | public void initializeState(FunctionInitializationContext context) throws Exception { 93 | if (!initialized) { 94 | init(); 95 | } 96 | if (initializeStateFn != null) { 97 | initializeStateFn.invoke(this, context); 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljCoMapFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | import clojure.java.api.Clojure; 4 | import clojure.lang.APersistentMap; 5 | import clojure.lang.IFn; 6 | import clojure.lang.Keyword; 7 | import clojure.lang.Namespace; 8 | import org.apache.flink.api.common.typeinfo.TypeInformation; 9 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 10 | import org.apache.flink.configuration.Configuration; 11 | import org.apache.flink.runtime.state.FunctionInitializationContext; 12 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 13 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 14 | import org.apache.flink.streaming.api.functions.co.RichCoMapFunction; 15 | 16 | public class CljCoMapFunction extends RichCoMapFunction implements ResultTypeQueryable, CheckpointedFunction { 17 | 18 | private final Namespace namespace; 19 | private final IFn initFn; 20 | private final IFn openFn; 21 | private final IFn closeFn; 22 | private final IFn initializeStateFn; 23 | private final IFn map1Fn; 24 | private final IFn map2Fn; 25 | private final IFn snapshotStateFn; 26 | private final TypeInformation returnType; 27 | 28 | private transient Object state; 29 | private transient boolean initialized; 30 | 31 | public CljCoMapFunction(APersistentMap args) { 32 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 33 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 34 | initFn = (IFn) Keyword.intern("init").invoke(args); 35 | openFn = (IFn) Keyword.intern("open").invoke(args); 36 | closeFn = (IFn) Keyword.intern("close").invoke(args); 37 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 38 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 39 | map1Fn = (IFn) Keyword.intern("map1").invoke(args); 40 | map2Fn = (IFn) Keyword.intern("map2").invoke(args); 41 | } 42 | 43 | private void init() { 44 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 45 | if (initFn != null) { 46 | state = initFn.invoke(this); 47 | } 48 | initialized = true; 49 | } 50 | 51 | public Object state() { 52 | return this.state; 53 | } 54 | 55 | @Override 56 | public void open(Configuration parameters) throws Exception { 57 | if (!initialized) { 58 | init(); 59 | } 60 | if (openFn != null) { 61 | openFn.invoke(this, parameters); 62 | } 63 | } 64 | 65 | @Override 66 | public void close() throws Exception { 67 | if (closeFn != null) { 68 | closeFn.invoke(this); 69 | } 70 | } 71 | 72 | @Override 73 | public TypeInformation getProducedType() { 74 | return returnType; 75 | } 76 | 77 | @Override 78 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 79 | if (snapshotStateFn != null) { 80 | snapshotStateFn.invoke(this, context); 81 | } 82 | } 83 | 84 | @Override 85 | public void initializeState(FunctionInitializationContext context) throws Exception { 86 | if (!initialized) { 87 | init(); 88 | } 89 | if (initializeStateFn != null) { 90 | initializeStateFn.invoke(this, context); 91 | } 92 | } 93 | 94 | @Override 95 | public OUT map1(IN1 value) throws Exception { 96 | return (OUT) map1Fn.invoke(this, value); 97 | } 98 | 99 | @Override 100 | public OUT map2(IN2 value) throws Exception { 101 | return (OUT) map2Fn.invoke(this, value); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljProcessJoinFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | import clojure.java.api.Clojure; 4 | import clojure.lang.APersistentMap; 5 | import clojure.lang.IFn; 6 | import clojure.lang.Keyword; 7 | import clojure.lang.Namespace; 8 | import org.apache.flink.api.common.typeinfo.TypeInformation; 9 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 10 | import org.apache.flink.configuration.Configuration; 11 | import org.apache.flink.runtime.state.FunctionInitializationContext; 12 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 13 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 14 | import org.apache.flink.streaming.api.functions.co.ProcessJoinFunction; 15 | import org.apache.flink.util.Collector; 16 | 17 | public class CljProcessJoinFunction extends ProcessJoinFunction 18 | implements ResultTypeQueryable, CheckpointedFunction { 19 | 20 | private transient Object state; 21 | private transient boolean initialized; 22 | 23 | private final Namespace namespace; 24 | private final TypeInformation returnType; 25 | private final IFn initFn; 26 | private final IFn openFn; 27 | private final IFn closeFn; 28 | private final IFn snapshotStateFn; 29 | private final IFn initializeStateFn; 30 | private final IFn processElementFn; 31 | private final IFn onTimerFn; 32 | 33 | public CljProcessJoinFunction(APersistentMap args) { 34 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 35 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 36 | initFn = (IFn) Keyword.intern("init").invoke(args); 37 | openFn = (IFn) Keyword.intern("open").invoke(args); 38 | closeFn = (IFn) Keyword.intern("close").invoke(args); 39 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 40 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 41 | processElementFn = (IFn) Keyword.intern("processElement").invoke(args); 42 | onTimerFn = (IFn) Keyword.intern("onTimer").invoke(args); 43 | } 44 | 45 | private void init() { 46 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 47 | if (initFn != null) { 48 | this.state = initFn.invoke(this); 49 | } 50 | initialized = true; 51 | } 52 | 53 | public Object state() { 54 | return this.state; 55 | } 56 | 57 | @Override 58 | public void open(Configuration parameters) throws Exception { 59 | if (!initialized) { 60 | init(); 61 | } 62 | if (openFn != null) { 63 | openFn.invoke(this, parameters); 64 | } 65 | } 66 | 67 | @Override 68 | public void close() throws Exception { 69 | if (closeFn != null) { 70 | closeFn.invoke(this); 71 | } 72 | } 73 | 74 | @Override 75 | public TypeInformation getProducedType() { 76 | return returnType; 77 | } 78 | 79 | @Override 80 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 81 | if (snapshotStateFn != null) { 82 | snapshotStateFn.invoke(this, context); 83 | } 84 | } 85 | 86 | @Override 87 | public void initializeState(FunctionInitializationContext context) throws Exception { 88 | if (!initialized) { 89 | init(); 90 | } 91 | if (initializeStateFn != null) { 92 | initializeStateFn.invoke(this, context); 93 | } 94 | } 95 | 96 | @Override 97 | public void processElement(IN1 left, IN2 right, ProcessJoinFunction.Context ctx, Collector out) throws Exception { 98 | processElementFn.invoke(this, left, right, ctx, out); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljCoFlatMapFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | import clojure.java.api.Clojure; 4 | import clojure.lang.APersistentMap; 5 | import clojure.lang.IFn; 6 | import clojure.lang.Keyword; 7 | import clojure.lang.Namespace; 8 | import org.apache.flink.api.common.typeinfo.TypeInformation; 9 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 10 | import org.apache.flink.configuration.Configuration; 11 | import org.apache.flink.runtime.state.FunctionInitializationContext; 12 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 13 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 14 | import org.apache.flink.streaming.api.functions.co.RichCoFlatMapFunction; 15 | import org.apache.flink.util.Collector; 16 | 17 | public class CljCoFlatMapFunction extends RichCoFlatMapFunction implements ResultTypeQueryable, CheckpointedFunction { 18 | 19 | private final Namespace namespace; 20 | private final IFn initFn; 21 | private final IFn openFn; 22 | private final IFn closeFn; 23 | private final IFn initializeStateFn; 24 | private final IFn flatMap1Fn; 25 | private final IFn flatMap2Fn; 26 | private final IFn snapshotStateFn; 27 | private final TypeInformation returnType; 28 | 29 | private transient Object state; 30 | private transient boolean initialized; 31 | 32 | public CljCoFlatMapFunction(APersistentMap args) { 33 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 34 | initFn = (IFn) Keyword.intern("init").invoke(args); 35 | openFn = (IFn) Keyword.intern("open").invoke(args); 36 | closeFn = (IFn) Keyword.intern("close").invoke(args); 37 | flatMap1Fn = (IFn) Keyword.intern("flatMap1").invoke(args); 38 | flatMap2Fn = (IFn) Keyword.intern("flatMap2").invoke(args); 39 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 40 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 41 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 42 | } 43 | 44 | @Override 45 | public void flatMap1(IN1 value, Collector out) throws Exception { 46 | flatMap1Fn.invoke(this, value, out); 47 | } 48 | 49 | @Override 50 | public void flatMap2(IN2 value, Collector out) throws Exception { 51 | flatMap2Fn.invoke(this, value, out); 52 | } 53 | 54 | private void init() { 55 | 56 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 57 | 58 | if (initFn != null) { 59 | state = initFn.invoke(this); 60 | } 61 | initialized = true; 62 | } 63 | 64 | public Object state() { 65 | return this.state; 66 | } 67 | 68 | @Override 69 | public void open(Configuration parameters) throws Exception { 70 | if (!initialized) { 71 | init(); 72 | } 73 | if (openFn != null) { 74 | openFn.invoke(this, parameters); 75 | } 76 | } 77 | 78 | @Override 79 | public void close() throws Exception { 80 | if (closeFn != null) { 81 | closeFn.invoke(this); 82 | } 83 | } 84 | 85 | @Override 86 | public TypeInformation getProducedType() { 87 | return returnType; 88 | } 89 | 90 | @Override 91 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 92 | if (snapshotStateFn != null) { 93 | snapshotStateFn.invoke(this, context); 94 | } 95 | } 96 | 97 | @Override 98 | public void initializeState(FunctionInitializationContext context) throws Exception { 99 | if (!initialized) { 100 | init(); 101 | } 102 | if (initializeStateFn != null) { 103 | initializeStateFn.invoke(this, context); 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljSinkFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | 4 | import clojure.java.api.Clojure; 5 | import clojure.lang.APersistentMap; 6 | import clojure.lang.IFn; 7 | import clojure.lang.Keyword; 8 | import clojure.lang.Namespace; 9 | import org.apache.flink.api.common.eventtime.Watermark; 10 | import org.apache.flink.configuration.Configuration; 11 | import org.apache.flink.runtime.state.FunctionInitializationContext; 12 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 13 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 14 | import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; 15 | 16 | public class CljSinkFunction extends RichSinkFunction 17 | implements CheckpointedFunction { 18 | 19 | private transient boolean initialized = false; 20 | private transient Object state; 21 | 22 | private final Namespace namespace; 23 | private final IFn initFn; 24 | private final IFn openFn; 25 | private final IFn closeFn; 26 | private final IFn invokeFn; 27 | private final IFn initializeStateFn; 28 | private final IFn snapshotStateFn; 29 | private final IFn writeWatermarkFn; 30 | private final IFn finishFn; 31 | 32 | 33 | public CljSinkFunction(APersistentMap args) { 34 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 35 | initFn = (IFn) Keyword.intern("init").invoke(args); 36 | invokeFn = (IFn) Keyword.intern("invoke").invoke(args); 37 | openFn = (IFn) Keyword.intern("open").invoke(args); 38 | closeFn = (IFn) Keyword.intern("close").invoke(args); 39 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 40 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 41 | writeWatermarkFn = (IFn) Keyword.intern("writeWatermark").invoke(args); 42 | finishFn = (IFn) Keyword.intern("finish").invoke(args); 43 | } 44 | 45 | private void init() { 46 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 47 | if (initFn != null) { 48 | state = initFn.invoke(this); 49 | } 50 | initialized = true; 51 | } 52 | 53 | @Override 54 | public void open(Configuration parameters) throws Exception { 55 | if (!initialized) { 56 | init(); 57 | } 58 | 59 | if (openFn != null) { 60 | openFn.invoke(this, parameters); 61 | } 62 | } 63 | 64 | @Override 65 | public void close() throws Exception { 66 | if (closeFn != null) { 67 | closeFn.invoke(this); 68 | } 69 | } 70 | 71 | @Override 72 | public void invoke(IN value, Context context) throws Exception { 73 | invokeFn.invoke(this, value, context); 74 | } 75 | 76 | @Override 77 | public void writeWatermark(Watermark watermark) throws Exception { 78 | super.writeWatermark(watermark); 79 | if (writeWatermarkFn != null) { 80 | writeWatermarkFn.invoke(this, watermark); 81 | } 82 | } 83 | 84 | @Override 85 | public void finish() throws Exception { 86 | super.finish(); 87 | if (finishFn != null) { 88 | finishFn.invoke(this); 89 | } 90 | } 91 | 92 | public Object state() { 93 | return this.state; 94 | } 95 | 96 | @Override 97 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 98 | if (snapshotStateFn != null) { 99 | snapshotStateFn.invoke(this, context); 100 | } 101 | } 102 | 103 | @Override 104 | public void initializeState(FunctionInitializationContext context) throws Exception { 105 | if (!initialized) { 106 | init(); 107 | } 108 | if (initializeStateFn != null) { 109 | initializeStateFn.invoke(this, context); 110 | } 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljSourceFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | 4 | import clojure.java.api.Clojure; 5 | import clojure.lang.APersistentMap; 6 | import clojure.lang.IFn; 7 | import clojure.lang.Keyword; 8 | import clojure.lang.Namespace; 9 | import org.apache.flink.api.common.typeinfo.TypeInformation; 10 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 11 | import org.apache.flink.configuration.Configuration; 12 | import org.apache.flink.runtime.state.FunctionInitializationContext; 13 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 14 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 15 | import org.apache.flink.streaming.api.functions.source.RichSourceFunction; 16 | 17 | public class CljSourceFunction extends RichSourceFunction 18 | implements ResultTypeQueryable, CheckpointedFunction { 19 | 20 | private transient boolean initialized = false; 21 | private transient Object state; 22 | 23 | private final Namespace namespace; 24 | private TypeInformation returnType; 25 | private final IFn initFn; 26 | private final IFn openFn; 27 | private final IFn closeFn; 28 | private final IFn initializeStateFn; 29 | private final IFn snapshotStateFn; 30 | private final IFn runFn ; 31 | private final IFn cancelFn; 32 | 33 | public CljSourceFunction(APersistentMap args) { 34 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 35 | initFn = (IFn) Keyword.intern("init").invoke(args); 36 | openFn = (IFn) Keyword.intern("open").invoke(args); 37 | closeFn = (IFn) Keyword.intern("close").invoke(args); 38 | runFn = (IFn) Keyword.intern("run").invoke(args); 39 | cancelFn = (IFn) Keyword.intern("cancel").invoke(args); 40 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 41 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 42 | 43 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 44 | } 45 | 46 | private void init() { 47 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 48 | if (initFn != null) { 49 | state = initFn.invoke(this); 50 | } 51 | initialized = true; 52 | } 53 | 54 | @Override 55 | public void open(Configuration parameters) throws Exception { 56 | if (!initialized) { 57 | init(); 58 | } 59 | if (openFn != null) { 60 | openFn.invoke(this, parameters); 61 | } 62 | } 63 | 64 | @Override 65 | public void close() throws Exception { 66 | if (closeFn != null) { 67 | closeFn.invoke(this); 68 | } 69 | } 70 | 71 | public Object state() { 72 | return this.state; 73 | } 74 | 75 | @Override 76 | public void run(SourceContext ctx) throws Exception { 77 | if (runFn == null) { 78 | throw new IllegalStateException("run function is null"); 79 | } 80 | runFn.invoke(this, ctx); 81 | } 82 | 83 | @Override 84 | public void cancel() { 85 | if (cancelFn != null) { 86 | cancelFn.invoke(this); 87 | } 88 | } 89 | 90 | @Override 91 | public TypeInformation getProducedType() { 92 | return returnType; 93 | } 94 | 95 | @Override 96 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 97 | if (snapshotStateFn != null) { 98 | snapshotStateFn.invoke(this, context); 99 | } 100 | } 101 | 102 | @Override 103 | public void initializeState(FunctionInitializationContext context) throws Exception { 104 | if (!initialized) { 105 | init(); 106 | } 107 | 108 | if (initializeStateFn != null) { 109 | initializeStateFn.invoke(this, context); 110 | } 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljParallelSourceFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | 4 | import clojure.java.api.Clojure; 5 | import clojure.lang.APersistentMap; 6 | import clojure.lang.IFn; 7 | import clojure.lang.Keyword; 8 | import clojure.lang.Namespace; 9 | import org.apache.flink.api.common.typeinfo.TypeInformation; 10 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 11 | import org.apache.flink.configuration.Configuration; 12 | import org.apache.flink.runtime.state.FunctionInitializationContext; 13 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 14 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 15 | import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction; 16 | 17 | public class CljParallelSourceFunction extends RichParallelSourceFunction 18 | implements ResultTypeQueryable, CheckpointedFunction { 19 | 20 | private transient boolean initialized = false; 21 | private transient Object state; 22 | 23 | private final Namespace namespace; 24 | private TypeInformation returnType; 25 | private final IFn initFn; 26 | private final IFn openFn; 27 | private final IFn closeFn; 28 | private final IFn initializeStateFn; 29 | private final IFn snapshotStateFn; 30 | private final IFn runFn ; 31 | private final IFn cancelFn; 32 | 33 | public CljParallelSourceFunction(APersistentMap args) { 34 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 35 | initFn = (IFn) Keyword.intern("init").invoke(args); 36 | openFn = (IFn) Keyword.intern("open").invoke(args); 37 | closeFn = (IFn) Keyword.intern("close").invoke(args); 38 | runFn = (IFn) Keyword.intern("run").invoke(args); 39 | cancelFn = (IFn) Keyword.intern("cancel").invoke(args); 40 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 41 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 42 | 43 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 44 | } 45 | 46 | private void init() { 47 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 48 | if (initFn != null) { 49 | state = initFn.invoke(this); 50 | } 51 | initialized = true; 52 | } 53 | 54 | @Override 55 | public void open(Configuration parameters) throws Exception { 56 | if (!initialized) { 57 | init(); 58 | } 59 | if (openFn != null) { 60 | openFn.invoke(this, parameters); 61 | } 62 | } 63 | 64 | @Override 65 | public void close() throws Exception { 66 | if (closeFn != null) { 67 | closeFn.invoke(this); 68 | } 69 | } 70 | 71 | public Object state() { 72 | return this.state; 73 | } 74 | 75 | @Override 76 | public void run(SourceContext ctx) throws Exception { 77 | if (runFn == null) { 78 | throw new IllegalStateException("run function is null"); 79 | } 80 | runFn.invoke(this, ctx); 81 | } 82 | 83 | @Override 84 | public void cancel() { 85 | if (cancelFn != null) { 86 | cancelFn.invoke(this); 87 | } 88 | } 89 | 90 | @Override 91 | public TypeInformation getProducedType() { 92 | return returnType; 93 | } 94 | 95 | @Override 96 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 97 | if (snapshotStateFn != null) { 98 | snapshotStateFn.invoke(this, context); 99 | } 100 | } 101 | 102 | @Override 103 | public void initializeState(FunctionInitializationContext context) throws Exception { 104 | if (!initialized) { 105 | init(); 106 | } 107 | 108 | if (initializeStateFn != null) { 109 | initializeStateFn.invoke(this, context); 110 | } 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljProcessFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | 4 | import clojure.java.api.Clojure; 5 | import clojure.lang.APersistentMap; 6 | import clojure.lang.IFn; 7 | import clojure.lang.Keyword; 8 | import clojure.lang.Namespace; 9 | import org.apache.flink.api.common.typeinfo.TypeInformation; 10 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 11 | import org.apache.flink.configuration.Configuration; 12 | import org.apache.flink.runtime.state.FunctionInitializationContext; 13 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 14 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 15 | import org.apache.flink.streaming.api.functions.ProcessFunction; 16 | import org.apache.flink.util.Collector; 17 | 18 | public class CljProcessFunction extends ProcessFunction 19 | implements ResultTypeQueryable, CheckpointedFunction { 20 | 21 | private transient boolean initialized = false; 22 | private transient Object state; 23 | 24 | private final Namespace namespace; 25 | private final IFn initFn; 26 | private final IFn openFn; 27 | private final IFn closeFn; 28 | private final IFn onTimerFn; 29 | private final IFn processElementFn; 30 | private final IFn initializeStateFn; 31 | private final IFn snapshotStateFn; 32 | 33 | private TypeInformation returnType; 34 | 35 | public CljProcessFunction(APersistentMap args) { 36 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 37 | initFn = (IFn) Keyword.intern("init").invoke(args); 38 | openFn = (IFn) Keyword.intern("open").invoke(args); 39 | processElementFn = (IFn) Keyword.intern("processElement").invoke(args); 40 | onTimerFn = (IFn) Keyword.intern("onTimer").invoke(args); 41 | closeFn = (IFn) Keyword.intern("close").invoke(args); 42 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 43 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 44 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 45 | } 46 | 47 | @Override 48 | public void processElement(I value, ProcessFunction.Context ctx, Collector out) throws Exception { 49 | processElementFn.invoke(this, value, ctx, out); 50 | } 51 | 52 | @Override 53 | public void onTimer(long timestamp, OnTimerContext ctx, Collector out) throws Exception { 54 | onTimerFn.invoke(this, timestamp, ctx, out); 55 | } 56 | 57 | private void init() { 58 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 59 | if (initFn != null) { 60 | state = initFn.invoke(this); 61 | } 62 | initialized = true; 63 | } 64 | 65 | public Object state() { 66 | return this.state; 67 | } 68 | 69 | @Override 70 | public void open(Configuration parameters) throws Exception { 71 | if (!initialized) { 72 | init(); 73 | } 74 | if (openFn != null) { 75 | openFn.invoke(this, parameters); 76 | } 77 | } 78 | 79 | @Override 80 | public void close() throws Exception { 81 | if (closeFn != null) { 82 | closeFn.invoke(this); 83 | } 84 | } 85 | 86 | @Override 87 | public TypeInformation getProducedType() { 88 | return returnType; 89 | } 90 | 91 | @Override 92 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 93 | if (snapshotStateFn != null) { 94 | snapshotStateFn.invoke(this, context); 95 | } 96 | } 97 | 98 | @Override 99 | public void initializeState(FunctionInitializationContext context) throws Exception { 100 | if (!initialized) { 101 | init(); 102 | } 103 | if (initializeStateFn != null) { 104 | initializeStateFn.invoke(this, context); 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljKeyedProcessFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | 4 | import clojure.java.api.Clojure; 5 | import clojure.lang.APersistentMap; 6 | import clojure.lang.IFn; 7 | import clojure.lang.Keyword; 8 | import clojure.lang.Namespace; 9 | import org.apache.flink.api.common.typeinfo.TypeInformation; 10 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 11 | import org.apache.flink.configuration.Configuration; 12 | import org.apache.flink.runtime.state.FunctionInitializationContext; 13 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 14 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 15 | import org.apache.flink.streaming.api.functions.KeyedProcessFunction; 16 | import org.apache.flink.util.Collector; 17 | 18 | public class CljKeyedProcessFunction extends KeyedProcessFunction 19 | implements ResultTypeQueryable, CheckpointedFunction { 20 | 21 | private final TypeInformation returnType; 22 | 23 | private transient Object state; 24 | private transient boolean initialized; 25 | 26 | private final IFn openFn; 27 | private final IFn closeFn; 28 | private final IFn onTimerFn; 29 | private final IFn processElementFn; 30 | private final Namespace namespace; 31 | private final IFn initFn; 32 | private final IFn initializeStateFn; 33 | private final IFn snapshotStateFn; 34 | 35 | public CljKeyedProcessFunction(APersistentMap args) { 36 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 37 | initFn = (IFn) Keyword.intern("init").invoke(args); 38 | openFn = (IFn) Keyword.intern("open").invoke(args); 39 | closeFn = (IFn) Keyword.intern("close").invoke(args); 40 | processElementFn = (IFn) Keyword.intern("processElement").invoke(args); 41 | onTimerFn = (IFn) Keyword.intern("onTimer").invoke(args); 42 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 43 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 44 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 45 | } 46 | 47 | public Object state() { 48 | return this.state; 49 | } 50 | 51 | private void init() { 52 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 53 | if (initFn != null) { 54 | this.state = initFn.invoke(this); 55 | } 56 | initialized = true; 57 | } 58 | 59 | @Override 60 | public void open(Configuration parameters) throws Exception { 61 | 62 | if (!initialized) { 63 | init(); 64 | } 65 | 66 | if (openFn != null) { 67 | openFn.invoke(this, parameters); 68 | } 69 | } 70 | 71 | @Override 72 | public void close() throws Exception { 73 | if (closeFn != null) { 74 | closeFn.invoke(this); 75 | } 76 | } 77 | 78 | @Override 79 | public void processElement(I value, Context ctx, Collector out) throws Exception { 80 | processElementFn.invoke(this, value, ctx, out); 81 | } 82 | 83 | @Override 84 | public void onTimer(long timestamp, OnTimerContext ctx, Collector out) throws Exception { 85 | onTimerFn.invoke(this, timestamp, ctx, out); 86 | } 87 | 88 | @Override 89 | public TypeInformation getProducedType() { 90 | return returnType; 91 | } 92 | 93 | @Override 94 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 95 | if (snapshotStateFn != null) { 96 | snapshotStateFn.invoke(this, context); 97 | } 98 | } 99 | 100 | @Override 101 | public void initializeState(FunctionInitializationContext context) throws Exception { 102 | if (!initialized) { 103 | init(); 104 | } 105 | if (initializeStateFn != null) { 106 | initializeStateFn.invoke(this, context); 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljProcessWindowFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | 4 | import clojure.java.api.Clojure; 5 | import clojure.lang.APersistentMap; 6 | import clojure.lang.IFn; 7 | import clojure.lang.Keyword; 8 | import clojure.lang.Namespace; 9 | import org.apache.flink.api.common.typeinfo.TypeInformation; 10 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 11 | import org.apache.flink.configuration.Configuration; 12 | import org.apache.flink.runtime.state.FunctionInitializationContext; 13 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 14 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 15 | import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction; 16 | import org.apache.flink.streaming.api.windowing.windows.Window; 17 | import org.apache.flink.util.Collector; 18 | 19 | public class CljProcessWindowFunction extends ProcessWindowFunction 20 | implements ResultTypeQueryable, CheckpointedFunction { 21 | 22 | private transient boolean initialized = false; 23 | private transient Object state; 24 | 25 | private final Namespace namespace; 26 | private TypeInformation returnType; 27 | private final IFn initFn; 28 | private final IFn openFn; 29 | private final IFn closeFn; 30 | private final IFn initializeStateFn; 31 | private final IFn snapshotStateFn; 32 | private final IFn processFn; 33 | private final IFn clearFn; 34 | 35 | public CljProcessWindowFunction(APersistentMap args) { 36 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 37 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 38 | initFn = (IFn) Keyword.intern("init").invoke(args); 39 | openFn = (IFn) Keyword.intern("open").invoke(args); 40 | closeFn = (IFn) Keyword.intern("close").invoke(args); 41 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 42 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 43 | processFn = (IFn) Keyword.intern("process").invoke(args); 44 | clearFn = (IFn) Keyword.intern("clear").invoke(args); 45 | } 46 | 47 | private void init() { 48 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 49 | if (initFn != null) { 50 | state = initFn.invoke(this); 51 | } 52 | initialized = true; 53 | } 54 | 55 | public Object state() { 56 | return this.state; 57 | } 58 | 59 | @Override 60 | public void open(Configuration parameters) throws Exception { 61 | if (!initialized) { 62 | init(); 63 | } 64 | if (openFn != null) { 65 | openFn.invoke(this, parameters); 66 | } 67 | } 68 | 69 | @Override 70 | public void close() throws Exception { 71 | if (closeFn != null) { 72 | closeFn.invoke(this); 73 | } 74 | } 75 | 76 | @Override 77 | public TypeInformation getProducedType() { 78 | return returnType; 79 | } 80 | 81 | @Override 82 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 83 | if (snapshotStateFn != null) { 84 | snapshotStateFn.invoke(this, context); 85 | } 86 | } 87 | 88 | @Override 89 | public void initializeState(FunctionInitializationContext context) throws Exception { 90 | if (!initialized) { 91 | init(); 92 | } 93 | if (initializeStateFn != null) { 94 | initializeStateFn.invoke(this, context); 95 | } 96 | } 97 | 98 | @Override 99 | public void process(KEY key, ProcessWindowFunction.Context ctx, Iterable elements, Collector out) throws Exception { 100 | processFn.invoke(this, key, ctx, elements, out); 101 | } 102 | 103 | @Override 104 | public void clear(ProcessWindowFunction.Context ctx) { 105 | if (clearFn != null) { 106 | clearFn.invoke(this, ctx); 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljProcessAllWindowFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | 4 | import clojure.java.api.Clojure; 5 | import clojure.lang.APersistentMap; 6 | import clojure.lang.IFn; 7 | import clojure.lang.Keyword; 8 | import clojure.lang.Namespace; 9 | import org.apache.flink.api.common.typeinfo.TypeInformation; 10 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 11 | import org.apache.flink.configuration.Configuration; 12 | import org.apache.flink.runtime.state.FunctionInitializationContext; 13 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 14 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 15 | import org.apache.flink.streaming.api.functions.windowing.ProcessAllWindowFunction; 16 | import org.apache.flink.streaming.api.windowing.windows.Window; 17 | import org.apache.flink.util.Collector; 18 | 19 | public class CljProcessAllWindowFunction extends ProcessAllWindowFunction 20 | implements ResultTypeQueryable, CheckpointedFunction { 21 | 22 | private transient boolean initialized = false; 23 | private transient Object state; 24 | 25 | private final Namespace namespace; 26 | private TypeInformation returnType; 27 | private final IFn initFn; 28 | private final IFn openFn; 29 | private final IFn closeFn; 30 | private final IFn initializeStateFn; 31 | private final IFn snapshotStateFn; 32 | private final IFn processFn; 33 | private final IFn clearFn; 34 | 35 | public CljProcessAllWindowFunction(APersistentMap args) { 36 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 37 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 38 | initFn = (IFn) Keyword.intern("init").invoke(args); 39 | openFn = (IFn) Keyword.intern("open").invoke(args); 40 | closeFn = (IFn) Keyword.intern("close").invoke(args); 41 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 42 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 43 | processFn = (IFn) Keyword.intern("process").invoke(args); 44 | clearFn = (IFn) Keyword.intern("clear").invoke(args); 45 | } 46 | 47 | private void init() { 48 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 49 | if (initFn != null) { 50 | state = initFn.invoke(this); 51 | } 52 | initialized = true; 53 | } 54 | 55 | public Object state() { 56 | return this.state; 57 | } 58 | 59 | @Override 60 | public void open(Configuration parameters) throws Exception { 61 | if (!initialized) { 62 | init(); 63 | } 64 | if (openFn != null) { 65 | openFn.invoke(this, parameters); 66 | } 67 | } 68 | 69 | @Override 70 | public void close() throws Exception { 71 | if (closeFn != null) { 72 | closeFn.invoke(this); 73 | } 74 | } 75 | 76 | @Override 77 | public TypeInformation getProducedType() { 78 | return returnType; 79 | } 80 | 81 | @Override 82 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 83 | if (snapshotStateFn != null) { 84 | snapshotStateFn.invoke(this, context); 85 | } 86 | } 87 | 88 | @Override 89 | public void initializeState(FunctionInitializationContext context) throws Exception { 90 | if (!initialized) { 91 | init(); 92 | } 93 | if (initializeStateFn != null) { 94 | initializeStateFn.invoke(this, context); 95 | } 96 | } 97 | 98 | @Override 99 | public void process(ProcessAllWindowFunction.Context context, Iterable elements, Collector out) throws Exception { 100 | processFn.invoke(this, context, elements, out); 101 | } 102 | 103 | @Override 104 | public void clear(ProcessAllWindowFunction.Context context) throws Exception { 105 | if (clearFn != null) { 106 | clearFn.invoke(this, context); 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljBroadcastProcessFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | import clojure.java.api.Clojure; 4 | import clojure.lang.APersistentMap; 5 | import clojure.lang.IFn; 6 | import clojure.lang.Keyword; 7 | import clojure.lang.Namespace; 8 | import org.apache.flink.api.common.typeinfo.TypeInformation; 9 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 10 | import org.apache.flink.configuration.Configuration; 11 | import org.apache.flink.runtime.state.FunctionInitializationContext; 12 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 13 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 14 | import org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction; 15 | import org.apache.flink.util.Collector; 16 | 17 | public class CljBroadcastProcessFunction extends BroadcastProcessFunction 18 | implements ResultTypeQueryable, CheckpointedFunction { 19 | 20 | private transient Object state; 21 | private transient boolean initialized; 22 | 23 | private final Namespace namespace; 24 | private final TypeInformation returnType; 25 | private final IFn initFn; 26 | private final IFn openFn; 27 | private final IFn closeFn; 28 | private final IFn snapshotStateFn; 29 | private final IFn initializeStateFn; 30 | private final IFn processElementFn; 31 | private final IFn processBroadcastElementFn; 32 | 33 | public CljBroadcastProcessFunction(APersistentMap args) { 34 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 35 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 36 | initFn = (IFn) Keyword.intern("init").invoke(args); 37 | openFn = (IFn) Keyword.intern("open").invoke(args); 38 | closeFn = (IFn) Keyword.intern("close").invoke(args); 39 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 40 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 41 | processElementFn = (IFn) Keyword.intern("processElement").invoke(args); 42 | processBroadcastElementFn = (IFn) Keyword.intern("processBroadcastElement").invoke(args); 43 | } 44 | 45 | private void init() { 46 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 47 | if (initFn != null) { 48 | this.state = initFn.invoke(this); 49 | } 50 | initialized = true; 51 | } 52 | 53 | public Object state() { 54 | return this.state; 55 | } 56 | 57 | @Override 58 | public void open(Configuration parameters) throws Exception { 59 | if (!initialized) { 60 | init(); 61 | } 62 | if (openFn != null) { 63 | openFn.invoke(this, parameters); 64 | } 65 | } 66 | 67 | @Override 68 | public void close() throws Exception { 69 | if (closeFn != null) { 70 | closeFn.invoke(this); 71 | } 72 | } 73 | 74 | @Override 75 | public TypeInformation getProducedType() { 76 | return returnType; 77 | } 78 | 79 | @Override 80 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 81 | if (snapshotStateFn != null) { 82 | snapshotStateFn.invoke(this, context); 83 | } 84 | } 85 | 86 | @Override 87 | public void initializeState(FunctionInitializationContext context) throws Exception { 88 | if (!initialized) { 89 | init(); 90 | } 91 | if (initializeStateFn != null) { 92 | initializeStateFn.invoke(this, context); 93 | } 94 | } 95 | 96 | @Override 97 | public void processElement(IN1 value, BroadcastProcessFunction.ReadOnlyContext ctx, Collector out) throws Exception { 98 | processElementFn.invoke(this, value, ctx, out); 99 | } 100 | 101 | @Override 102 | public void processBroadcastElement(IN2 value, BroadcastProcessFunction.Context ctx, Collector out) throws Exception { 103 | processBroadcastElementFn.invoke(this, value, ctx, out); 104 | } 105 | 106 | } 107 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljAsyncFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | import clojure.java.api.Clojure; 4 | import clojure.lang.APersistentMap; 5 | import clojure.lang.IFn; 6 | import clojure.lang.Keyword; 7 | import clojure.lang.Namespace; 8 | import org.apache.flink.api.common.typeinfo.TypeInformation; 9 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 10 | import org.apache.flink.configuration.Configuration; 11 | import org.apache.flink.runtime.state.FunctionInitializationContext; 12 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 13 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 14 | import org.apache.flink.streaming.api.functions.async.ResultFuture; 15 | import org.apache.flink.streaming.api.functions.async.RichAsyncFunction; 16 | 17 | import java.util.concurrent.TimeoutException; 18 | 19 | 20 | public class CljAsyncFunction extends RichAsyncFunction 21 | implements ResultTypeQueryable, CheckpointedFunction { 22 | 23 | private transient Object state; 24 | private transient boolean initialized; 25 | private final Namespace namespace; 26 | private final TypeInformation returnType; 27 | private final IFn initFn; 28 | private final IFn openFn; 29 | private final IFn closeFn; 30 | private final IFn initializeStateFn; 31 | private final IFn snapshotStateFn; 32 | 33 | private final IFn asyncInvokeFn; 34 | private final IFn timeoutFn; 35 | 36 | 37 | public CljAsyncFunction(APersistentMap args) { 38 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 39 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 40 | initFn = (IFn) Keyword.intern("init").invoke(args); 41 | openFn = (IFn) Keyword.intern("open").invoke(args); 42 | closeFn = (IFn) Keyword.intern("close").invoke(args); 43 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 44 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 45 | asyncInvokeFn = (IFn) Keyword.intern("asyncInvoke").invoke(args); 46 | timeoutFn = (IFn) Keyword.intern("timeout").invoke(args); 47 | } 48 | 49 | private void init() { 50 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 51 | if (initFn != null) { 52 | state = initFn.invoke(this); 53 | } 54 | initialized = true; 55 | } 56 | 57 | public Object state() { 58 | return this.state; 59 | } 60 | 61 | @Override 62 | public void open(Configuration parameters) throws Exception { 63 | super.open(parameters); 64 | if (!initialized) { 65 | init(); 66 | } 67 | if (openFn != null) { 68 | openFn.invoke(this, parameters); 69 | } 70 | } 71 | 72 | @Override 73 | public void close() throws Exception { 74 | if (closeFn != null) { 75 | closeFn.invoke(this); 76 | } 77 | super.close(); 78 | } 79 | 80 | @Override 81 | public TypeInformation getProducedType() { 82 | return returnType; 83 | } 84 | 85 | @Override 86 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 87 | if (snapshotStateFn != null) { 88 | snapshotStateFn.invoke(this, context); 89 | } 90 | } 91 | 92 | @Override 93 | public void initializeState(FunctionInitializationContext context) throws Exception { 94 | if (!initialized) { 95 | init(); 96 | } 97 | if (initializeStateFn != null) { 98 | initializeStateFn.invoke(this, context); 99 | } 100 | } 101 | 102 | @Override 103 | public void asyncInvoke(IN input, ResultFuture resultFuture) throws Exception { 104 | asyncInvokeFn.invoke(this, input, resultFuture); 105 | } 106 | 107 | @Override 108 | public void timeout(IN input, ResultFuture resultFuture) throws Exception { 109 | if (timeoutFn != null) { 110 | timeoutFn.invoke(this, input, resultFuture); 111 | } else { 112 | resultFuture.completeExceptionally( 113 | new TimeoutException("Async function call has timed out.")); 114 | } 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljKeyedCoProcessFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | 4 | import clojure.java.api.Clojure; 5 | import clojure.lang.APersistentMap; 6 | import clojure.lang.IFn; 7 | import clojure.lang.Keyword; 8 | import clojure.lang.Namespace; 9 | import org.apache.flink.api.common.typeinfo.TypeInformation; 10 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 11 | import org.apache.flink.configuration.Configuration; 12 | import org.apache.flink.runtime.state.FunctionInitializationContext; 13 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 14 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 15 | import org.apache.flink.streaming.api.functions.co.KeyedCoProcessFunction; 16 | import org.apache.flink.util.Collector; 17 | 18 | public class CljKeyedCoProcessFunction extends KeyedCoProcessFunction 19 | implements ResultTypeQueryable, CheckpointedFunction { 20 | 21 | private final TypeInformation returnType; 22 | 23 | private transient Object state; 24 | private transient boolean initialized; 25 | 26 | private final Namespace namespace; 27 | private final IFn initFn; 28 | private final IFn openFn; 29 | private final IFn closeFn; 30 | private final IFn initializeStateFn; 31 | private final IFn snapshotStateFn; 32 | private final IFn processElement1Fn; 33 | private final IFn processElement2Fn; 34 | private final IFn onTimerFn; 35 | 36 | public CljKeyedCoProcessFunction(APersistentMap args) { 37 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 38 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 39 | initFn = (IFn) Keyword.intern("init").invoke(args); 40 | openFn = (IFn) Keyword.intern("open").invoke(args); 41 | closeFn = (IFn) Keyword.intern("close").invoke(args); 42 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 43 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 44 | processElement1Fn = (IFn) Keyword.intern("processElement1").invoke(args); 45 | processElement2Fn = (IFn) Keyword.intern("processElement2").invoke(args); 46 | onTimerFn = (IFn) Keyword.intern("onTimer").invoke(args); 47 | 48 | } 49 | 50 | public Object state() { 51 | return this.state; 52 | } 53 | 54 | private void init() { 55 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 56 | if (initFn != null) { 57 | this.state = initFn.invoke(this); 58 | } 59 | initialized = true; 60 | } 61 | 62 | @Override 63 | public void open(Configuration parameters) throws Exception { 64 | 65 | if (!initialized) { 66 | init(); 67 | } 68 | 69 | if (openFn != null) { 70 | openFn.invoke(this, parameters); 71 | } 72 | } 73 | 74 | @Override 75 | public void close() throws Exception { 76 | if (closeFn != null) { 77 | closeFn.invoke(this); 78 | } 79 | } 80 | 81 | @Override 82 | public TypeInformation getProducedType() { 83 | return returnType; 84 | } 85 | 86 | @Override 87 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 88 | if (snapshotStateFn != null) { 89 | snapshotStateFn.invoke(this, context); 90 | } 91 | } 92 | 93 | @Override 94 | public void initializeState(FunctionInitializationContext context) throws Exception { 95 | if (!initialized) { 96 | init(); 97 | } 98 | if (initializeStateFn != null) { 99 | initializeStateFn.invoke(this, context); 100 | } 101 | } 102 | @Override 103 | public void processElement1(IN1 value, Context ctx, Collector out) throws Exception { 104 | processElement1Fn.invoke(this, value, ctx, out); 105 | } 106 | 107 | @Override 108 | public void processElement2(IN2 value, Context ctx, Collector out) throws Exception { 109 | processElement2Fn.invoke(this, value, ctx, out); 110 | } 111 | 112 | @Override 113 | public void onTimer(long timestamp, OnTimerContext ctx, Collector out) throws Exception { 114 | if (onTimerFn != null) { 115 | onTimerFn.invoke(this, timestamp, timestamp, ctx, out); 116 | } 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljCoProcessFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | import clojure.java.api.Clojure; 4 | import clojure.lang.APersistentMap; 5 | import clojure.lang.IFn; 6 | import clojure.lang.Keyword; 7 | import clojure.lang.Namespace; 8 | import org.apache.flink.api.common.typeinfo.TypeInformation; 9 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 10 | import org.apache.flink.configuration.Configuration; 11 | import org.apache.flink.runtime.state.FunctionInitializationContext; 12 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 13 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 14 | import org.apache.flink.streaming.api.functions.co.CoProcessFunction; 15 | import org.apache.flink.util.Collector; 16 | 17 | public class CljCoProcessFunction extends CoProcessFunction 18 | implements ResultTypeQueryable, CheckpointedFunction { 19 | 20 | private transient Object state; 21 | private transient boolean initialized; 22 | 23 | private final Namespace namespace; 24 | private final TypeInformation returnType; 25 | private final IFn initFn; 26 | private final IFn openFn; 27 | private final IFn closeFn; 28 | private final IFn snapshotStateFn; 29 | private final IFn initializeStateFn; 30 | private final IFn processElement1Fn; 31 | private final IFn processElement2Fn; 32 | private final IFn onTimerFn; 33 | 34 | public CljCoProcessFunction(APersistentMap args) { 35 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 36 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 37 | initFn = (IFn) Keyword.intern("init").invoke(args); 38 | openFn = (IFn) Keyword.intern("open").invoke(args); 39 | closeFn = (IFn) Keyword.intern("close").invoke(args); 40 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 41 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 42 | processElement1Fn = (IFn) Keyword.intern("processElement1").invoke(args); 43 | processElement2Fn = (IFn) Keyword.intern("processElement2").invoke(args); 44 | onTimerFn = (IFn) Keyword.intern("onTimer").invoke(args); 45 | } 46 | 47 | private void init() { 48 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 49 | if (initFn != null) { 50 | this.state = initFn.invoke(this); 51 | } 52 | initialized = true; 53 | } 54 | 55 | public Object state() { 56 | return this.state; 57 | } 58 | 59 | @Override 60 | public void open(Configuration parameters) throws Exception { 61 | if (!initialized) { 62 | init(); 63 | } 64 | if (openFn != null) { 65 | openFn.invoke(this, parameters); 66 | } 67 | } 68 | 69 | @Override 70 | public void close() throws Exception { 71 | if (closeFn != null) { 72 | closeFn.invoke(this); 73 | } 74 | } 75 | 76 | @Override 77 | public TypeInformation getProducedType() { 78 | return returnType; 79 | } 80 | 81 | @Override 82 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 83 | if (snapshotStateFn != null) { 84 | snapshotStateFn.invoke(this, context); 85 | } 86 | } 87 | 88 | @Override 89 | public void initializeState(FunctionInitializationContext context) throws Exception { 90 | if (!initialized) { 91 | init(); 92 | } 93 | if (initializeStateFn != null) { 94 | initializeStateFn.invoke(this, context); 95 | } 96 | } 97 | 98 | @Override 99 | public void processElement1(IN1 value, CoProcessFunction.Context ctx, Collector out) throws Exception { 100 | processElement1Fn.invoke(this, value, ctx, out); 101 | } 102 | 103 | @Override 104 | public void processElement2(IN2 value, CoProcessFunction.Context ctx, Collector out) throws Exception { 105 | processElement2Fn.invoke(this, value, ctx, out); 106 | } 107 | 108 | @Override 109 | public void onTimer(long timestamp, CoProcessFunction.OnTimerContext ctx, Collector out) throws Exception { 110 | if (onTimerFn != null) { 111 | onTimerFn.invoke(this, timestamp, ctx, out); 112 | } 113 | } 114 | 115 | } 116 | -------------------------------------------------------------------------------- /src/main/java/io/kosong/flink/clojure/functions/CljKeyedBroadcastProcessFunction.java: -------------------------------------------------------------------------------- 1 | package io.kosong.flink.clojure.functions; 2 | 3 | import clojure.java.api.Clojure; 4 | import clojure.lang.APersistentMap; 5 | import clojure.lang.IFn; 6 | import clojure.lang.Keyword; 7 | import clojure.lang.Namespace; 8 | import org.apache.flink.api.common.typeinfo.TypeInformation; 9 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 10 | import org.apache.flink.configuration.Configuration; 11 | import org.apache.flink.runtime.state.FunctionInitializationContext; 12 | import org.apache.flink.runtime.state.FunctionSnapshotContext; 13 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; 14 | import org.apache.flink.streaming.api.functions.co.KeyedBroadcastProcessFunction; 15 | import org.apache.flink.util.Collector; 16 | 17 | public class CljKeyedBroadcastProcessFunction extends KeyedBroadcastProcessFunction 18 | implements ResultTypeQueryable, CheckpointedFunction { 19 | 20 | private transient Object state; 21 | private transient boolean initialized; 22 | 23 | private final Namespace namespace; 24 | private final TypeInformation returnType; 25 | private final IFn initFn; 26 | private final IFn openFn; 27 | private final IFn closeFn; 28 | private final IFn snapshotStateFn; 29 | private final IFn initializeStateFn; 30 | private final IFn processElementFn; 31 | private final IFn processBroadcastElementFn; 32 | private final IFn onTimerFn; 33 | 34 | public CljKeyedBroadcastProcessFunction(APersistentMap args) { 35 | namespace = (Namespace) Keyword.intern("ns").invoke(args); 36 | returnType = (TypeInformation) Keyword.intern("returns").invoke(args); 37 | initFn = (IFn) Keyword.intern("init").invoke(args); 38 | openFn = (IFn) Keyword.intern("open").invoke(args); 39 | closeFn = (IFn) Keyword.intern("close").invoke(args); 40 | initializeStateFn = (IFn) Keyword.intern("initializeState").invoke(args); 41 | snapshotStateFn = (IFn) Keyword.intern("snapshotState").invoke(args); 42 | processElementFn = (IFn) Keyword.intern("processElement").invoke(args); 43 | processBroadcastElementFn = (IFn) Keyword.intern("processBroadcastElement").invoke(args); 44 | onTimerFn = (IFn) Keyword.intern("onTimer").invoke(args); 45 | } 46 | 47 | private void init() { 48 | Clojure.var("clojure.core/require").invoke(namespace.getName()); 49 | if (initFn != null) { 50 | this.state = initFn.invoke(this); 51 | } 52 | initialized = true; 53 | } 54 | 55 | public Object state() { 56 | return this.state; 57 | } 58 | 59 | @Override 60 | public void open(Configuration parameters) throws Exception { 61 | if (!initialized) { 62 | init(); 63 | } 64 | if (openFn != null) { 65 | openFn.invoke(this, parameters); 66 | } 67 | } 68 | 69 | @Override 70 | public void close() throws Exception { 71 | if (closeFn != null) { 72 | closeFn.invoke(this); 73 | } 74 | } 75 | 76 | @Override 77 | public TypeInformation getProducedType() { 78 | return returnType; 79 | } 80 | 81 | @Override 82 | public void snapshotState(FunctionSnapshotContext context) throws Exception { 83 | if (snapshotStateFn != null) { 84 | snapshotStateFn.invoke(this, context); 85 | } 86 | } 87 | 88 | @Override 89 | public void initializeState(FunctionInitializationContext context) throws Exception { 90 | if (!initialized) { 91 | init(); 92 | } 93 | if (initializeStateFn != null) { 94 | initializeStateFn.invoke(this, context); 95 | } 96 | } 97 | 98 | @Override 99 | public void processElement(IN1 value, KeyedBroadcastProcessFunction.ReadOnlyContext ctx, Collector out) throws Exception { 100 | processElementFn.invoke(this, value, ctx, out); 101 | } 102 | 103 | @Override 104 | public void processBroadcastElement(IN2 value, KeyedBroadcastProcessFunction.Context ctx, Collector out) throws Exception { 105 | processBroadcastElementFn.invoke(this, value, ctx, out); 106 | } 107 | 108 | @Override 109 | public void onTimer(long timestamp, KeyedBroadcastProcessFunction.OnTimerContext ctx, Collector out) throws Exception { 110 | if (onTimerFn != null) { 111 | onTimerFn.invoke(this, timestamp, ctx, out); 112 | } 113 | } 114 | } 115 | --------------------------------------------------------------------------------