├── .gitignore ├── README.md ├── src └── main │ └── java │ └── com │ └── alibaba │ └── ververica │ └── cep │ └── demo │ ├── condition │ ├── StartCondition.java │ ├── EndCondition.java │ └── MiddleCondition.java │ ├── Constants.java │ ├── dynamic │ ├── DemoPatternProcessFunction.java │ ├── PeriodicPatternProcessorDiscovererFactory.java │ ├── JDBCPeriodicPatternProcessorDiscovererFactory.java │ ├── PeriodicPatternProcessorDiscoverer.java │ ├── DefaultPatternProcessor.java │ └── JDBCPeriodicPatternProcessorDiscoverer.java │ ├── event │ ├── EventDeSerializationSchema.java │ └── Event.java │ └── CepDemo.java └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | .cache 2 | scalastyle-output.xml 3 | .classpath 4 | .idea 5 | .metadata 6 | .settings 7 | .project 8 | .version.properties 9 | filter.properties 10 | logs.zip 11 | target 12 | tmp 13 | *.class 14 | *.iml 15 | *.swp 16 | *.jar 17 | *.zip 18 | *.log 19 | *.pyc 20 | .DS_Store 21 | build-target 22 | out/ 23 | target/ 24 | *.ipr -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ververica-cep-demo 2 | Demo of Flink CEP with dynamic patterns. 3 | 4 | # !!Note!! 5 | The jar of flink-cep that supports dynamic CEP in [public maven](https://mvnrepository.com/artifact/com.alibaba.ververica/flink-cep/1.15-vvr-6.0.2-api) is an API jar that does not contain full implementation. So users can not directly use it in their local machine/cluster. 6 | Instead, users can use it for writing codes or packaging but the final job jar must be submitted to [Flink managed by Aliyun](https://www.alibabacloud.com/product/realtime-compute) | [阿里云实时计算Flink版](https://www.aliyun.com/product/bigdata/sc) as the dynamic CEP is currently a commercial feature. 7 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/ververica/cep/demo/condition/StartCondition.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KTD, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.alibaba.ververica.cep.demo.condition; 20 | 21 | import org.apache.flink.cep.dynamic.condition.AviatorCondition; 22 | 23 | import com.alibaba.ververica.cep.demo.event.Event; 24 | 25 | public class StartCondition extends AviatorCondition { 26 | 27 | public StartCondition(String expression) { 28 | super(expression); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/ververica/cep/demo/condition/EndCondition.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KTD, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.alibaba.ververica.cep.demo.condition; 20 | 21 | import org.apache.flink.cep.pattern.conditions.SimpleCondition; 22 | 23 | import com.alibaba.ververica.cep.demo.event.Event; 24 | 25 | public class EndCondition extends SimpleCondition { 26 | 27 | @Override 28 | public boolean filter(Event value) throws Exception { 29 | return value.getAction() != 1; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/ververica/cep/demo/condition/MiddleCondition.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KTD, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.alibaba.ververica.cep.demo.condition; 20 | 21 | import org.apache.flink.cep.pattern.conditions.SimpleCondition; 22 | 23 | import com.alibaba.ververica.cep.demo.event.Event; 24 | 25 | public class MiddleCondition extends SimpleCondition { 26 | 27 | @Override 28 | public boolean filter(Event value) throws Exception { 29 | return value.getName().contains("middle"); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/ververica/cep/demo/Constants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KTD, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.alibaba.ververica.cep.demo; 20 | 21 | public final class Constants { 22 | // Required configurations constants for connecting to Database 23 | public static final String JDBC_URL_ARG = "jdbcUrl"; 24 | public static final String JDBC_DRIVE = "com.mysql.cj.jdbc.Driver"; 25 | public static final String TABLE_NAME_ARG = "tableName"; 26 | public static final String JDBC_INTERVAL_MILLIS_ARG = "jdbcIntervalMs"; 27 | // Required configurations constants for connecting to Kafka 28 | public static final String KAFKA_BROKERS_ARG = "kafkaBrokers"; 29 | public static final String INPUT_TOPIC_ARG = "inputTopic"; 30 | public static final String INPUT_TOPIC_GROUP_ARG = "inputTopicGroup"; 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/ververica/cep/demo/dynamic/DemoPatternProcessFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KTD, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.alibaba.ververica.cep.demo.dynamic; 20 | 21 | import org.apache.flink.cep.functions.PatternProcessFunction; 22 | import org.apache.flink.util.Collector; 23 | 24 | import java.util.List; 25 | import java.util.Map; 26 | 27 | public class DemoPatternProcessFunction extends PatternProcessFunction { 28 | String id; 29 | int version; 30 | 31 | public DemoPatternProcessFunction(String id, int version) { 32 | this.id = id; 33 | this.version = version; 34 | } 35 | 36 | @Override 37 | public void processMatch( 38 | final Map> match, final Context ctx, final Collector out) { 39 | StringBuilder sb = new StringBuilder(); 40 | sb.append("A match for Pattern of (id, version): (") 41 | .append(id) 42 | .append(", ") 43 | .append(version) 44 | .append(") is found. The event sequence: "); 45 | for (Map.Entry> entry : match.entrySet()) { 46 | sb.append(entry.getKey()).append(": ").append(entry.getValue()); 47 | } 48 | out.collect(sb.toString()); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/ververica/cep/demo/event/EventDeSerializationSchema.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KTD, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.alibaba.ververica.cep.demo.event; 20 | 21 | import org.apache.flink.api.common.serialization.DeserializationSchema; 22 | import org.apache.flink.api.common.typeinfo.TypeInformation; 23 | import org.apache.flink.connector.kafka.source.reader.deserializer.KafkaRecordDeserializationSchema; 24 | import org.apache.flink.util.Collector; 25 | 26 | import org.apache.kafka.clients.consumer.ConsumerRecord; 27 | 28 | /** KafkaRecordDeserializationSchema for Event. */ 29 | public class EventDeSerializationSchema implements KafkaRecordDeserializationSchema { 30 | 31 | private static final long serialVersionUID = 6154188370181669758L; 32 | 33 | @Override 34 | public TypeInformation getProducedType() { 35 | return TypeInformation.of(Event.class); 36 | } 37 | 38 | @Override 39 | public void open(DeserializationSchema.InitializationContext context) throws Exception { 40 | KafkaRecordDeserializationSchema.super.open(context); 41 | } 42 | 43 | @Override 44 | public void deserialize( 45 | ConsumerRecord consumerRecord, Collector collector) { 46 | collector.collect(Event.fromString(new String(consumerRecord.value()))); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/ververica/cep/demo/dynamic/PeriodicPatternProcessorDiscovererFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KTD, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.alibaba.ververica.cep.demo.dynamic; 20 | 21 | import org.apache.flink.cep.dynamic.processor.PatternProcessor; 22 | import org.apache.flink.cep.dynamic.processor.PatternProcessorDiscoverer; 23 | import org.apache.flink.cep.dynamic.processor.PatternProcessorDiscovererFactory; 24 | 25 | import javax.annotation.Nullable; 26 | 27 | import java.util.List; 28 | 29 | /** 30 | * Implementation of the {@link PatternProcessorDiscovererFactory} that creates the {@link 31 | * PeriodicPatternProcessorDiscoverer} instance. 32 | * 33 | * @param Base type of the elements appearing in the pattern. 34 | */ 35 | public abstract class PeriodicPatternProcessorDiscovererFactory 36 | implements PatternProcessorDiscovererFactory { 37 | 38 | @Nullable private final List> initialPatternProcessors; 39 | private final Long intervalMillis; 40 | 41 | public PeriodicPatternProcessorDiscovererFactory( 42 | @Nullable final List> initialPatternProcessors, 43 | @Nullable final Long intervalMillis) { 44 | this.initialPatternProcessors = initialPatternProcessors; 45 | this.intervalMillis = intervalMillis; 46 | } 47 | 48 | @Nullable 49 | public List> getInitialPatternProcessors() { 50 | return initialPatternProcessors; 51 | } 52 | 53 | @Override 54 | public abstract PatternProcessorDiscoverer createPatternProcessorDiscoverer( 55 | ClassLoader userCodeClassLoader) throws Exception; 56 | 57 | public Long getIntervalMillis() { 58 | return intervalMillis; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/ververica/cep/demo/dynamic/JDBCPeriodicPatternProcessorDiscovererFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KTD, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.alibaba.ververica.cep.demo.dynamic; 20 | 21 | import org.apache.flink.cep.dynamic.processor.PatternProcessor; 22 | import org.apache.flink.cep.dynamic.processor.PatternProcessorDiscoverer; 23 | 24 | import javax.annotation.Nullable; 25 | 26 | import java.util.List; 27 | 28 | import static java.util.Objects.requireNonNull; 29 | 30 | /** 31 | * The JDBC implementation of the {@link PeriodicPatternProcessorDiscovererFactory} that creates the 32 | * {@link JDBCPeriodicPatternProcessorDiscoverer} instance. 33 | * 34 | * @param Base type of the elements appearing in the pattern. 35 | */ 36 | public class JDBCPeriodicPatternProcessorDiscovererFactory 37 | extends PeriodicPatternProcessorDiscovererFactory { 38 | 39 | private final String jdbcUrl; 40 | private final String jdbcDriver; 41 | private final String tableName; 42 | 43 | public JDBCPeriodicPatternProcessorDiscovererFactory( 44 | final String jdbcUrl, 45 | final String jdbcDriver, 46 | final String tableName, 47 | @Nullable final List> initialPatternProcessors, 48 | @Nullable final Long intervalMillis) { 49 | super(initialPatternProcessors, intervalMillis); 50 | this.jdbcUrl = requireNonNull(jdbcUrl); 51 | this.jdbcDriver = requireNonNull(jdbcDriver); 52 | this.tableName = requireNonNull(tableName); 53 | } 54 | 55 | @Override 56 | public PatternProcessorDiscoverer createPatternProcessorDiscoverer( 57 | ClassLoader userCodeClassLoader) throws Exception { 58 | return new JDBCPeriodicPatternProcessorDiscoverer<>( 59 | jdbcUrl, 60 | jdbcDriver, 61 | tableName, 62 | userCodeClassLoader, 63 | this.getInitialPatternProcessors(), 64 | getIntervalMillis()); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/ververica/cep/demo/event/Event.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KTD, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.alibaba.ververica.cep.demo.event; 20 | 21 | import java.util.Objects; 22 | 23 | /** Exemplary event for usage in tests of CEP. */ 24 | public class Event { 25 | private final int id; 26 | private final String name; 27 | 28 | private final int productionId; 29 | private final int action; 30 | private final long eventTime; 31 | 32 | public Event(int id, String name, int action, int productionId, long timestamp) { 33 | this.id = id; 34 | this.name = name; 35 | this.action = action; 36 | this.productionId = productionId; 37 | this.eventTime = timestamp; 38 | } 39 | 40 | public static Event fromString(String eventStr) { 41 | String[] split = eventStr.split(","); 42 | return new Event( 43 | Integer.parseInt(split[0]), 44 | split[1], 45 | Integer.parseInt(split[2]), 46 | Integer.parseInt(split[3]), 47 | Long.parseLong(split[4])); 48 | } 49 | 50 | public long getEventTime() { 51 | return eventTime; 52 | } 53 | 54 | public double getAction() { 55 | return action; 56 | } 57 | 58 | public int getId() { 59 | return id; 60 | } 61 | 62 | public int getProductionId() { 63 | return productionId; 64 | } 65 | 66 | public String getName() { 67 | return name; 68 | } 69 | 70 | @Override 71 | public String toString() { 72 | return "Event(" 73 | + id 74 | + ", " 75 | + name 76 | + ", " 77 | + action 78 | + ", " 79 | + productionId 80 | + ", " 81 | + eventTime 82 | + ")"; 83 | } 84 | 85 | @Override 86 | public boolean equals(Object obj) { 87 | if (obj instanceof Event) { 88 | Event other = (Event) obj; 89 | return name.equals(other.name) 90 | && action == other.action 91 | && productionId == other.productionId 92 | && id == other.id; 93 | } else { 94 | return false; 95 | } 96 | } 97 | 98 | @Override 99 | public int hashCode() { 100 | return Objects.hash(name, action, productionId, id); 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/ververica/cep/demo/dynamic/PeriodicPatternProcessorDiscoverer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KTD, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.alibaba.ververica.cep.demo.dynamic; 20 | 21 | import org.apache.flink.cep.dynamic.processor.PatternProcessor; 22 | import org.apache.flink.cep.dynamic.processor.PatternProcessorDiscoverer; 23 | import org.apache.flink.cep.dynamic.processor.PatternProcessorManager; 24 | 25 | import java.io.IOException; 26 | import java.util.List; 27 | import java.util.Timer; 28 | import java.util.TimerTask; 29 | 30 | /** 31 | * Implementation of the {@link PatternProcessorDiscoverer} that periodically discovers the pattern 32 | * processor updates. 33 | * 34 | * @param Base type of the elements appearing in the pattern. 35 | */ 36 | public abstract class PeriodicPatternProcessorDiscoverer 37 | implements PatternProcessorDiscoverer { 38 | 39 | private final Long intervalMillis; 40 | 41 | private final Timer timer; 42 | 43 | /** 44 | * Creates a new {@link PatternProcessorDiscoverer} using the given initial {@link 45 | * PatternProcessor} and the time interval how often to check the pattern processor updates. 46 | * 47 | * @param intervalMillis Time interval in milliseconds how often to check updates. 48 | */ 49 | public PeriodicPatternProcessorDiscoverer(final Long intervalMillis) { 50 | this.intervalMillis = intervalMillis; 51 | this.timer = new Timer(); 52 | } 53 | 54 | /** 55 | * Returns whether there are updated pattern processors. 56 | * 57 | * @return Whether there are updated pattern processors. 58 | */ 59 | public abstract boolean arePatternProcessorsUpdated(); 60 | 61 | /** 62 | * Returns the latest pattern processors. 63 | * 64 | * @return The list of {@link PatternProcessor}. 65 | */ 66 | public abstract List> getLatestPatternProcessors() throws Exception; 67 | 68 | @Override 69 | public void discoverPatternProcessorUpdates( 70 | PatternProcessorManager patternProcessorManager) { 71 | // Periodically discovers the pattern processor updates. 72 | timer.schedule( 73 | new TimerTask() { 74 | @Override 75 | public void run() { 76 | if (arePatternProcessorsUpdated()) { 77 | List> patternProcessors = null; 78 | try { 79 | patternProcessors = getLatestPatternProcessors(); 80 | } catch (Exception e) { 81 | e.printStackTrace(); 82 | } 83 | patternProcessorManager.onPatternProcessorsUpdated(patternProcessors); 84 | } 85 | } 86 | }, 87 | 0, 88 | intervalMillis); 89 | } 90 | 91 | @Override 92 | public void close() throws IOException { 93 | timer.cancel(); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/ververica/cep/demo/dynamic/DefaultPatternProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.alibaba.ververica.cep.demo.dynamic; 20 | 21 | import org.apache.flink.annotation.PublicEvolving; 22 | import org.apache.flink.api.java.functions.KeySelector; 23 | import org.apache.flink.cep.dynamic.impl.json.util.CepJsonUtils; 24 | import org.apache.flink.cep.dynamic.processor.PatternProcessor; 25 | import org.apache.flink.cep.functions.PatternProcessFunction; 26 | import org.apache.flink.cep.pattern.Pattern; 27 | 28 | import javax.annotation.Nullable; 29 | 30 | import static org.apache.flink.util.Preconditions.checkNotNull; 31 | 32 | /** 33 | * Default implementation of the {@link PatternProcessor} that is configurable for {@link Pattern}, 34 | * {@link KeySelector} and {@link PatternProcessFunction}. 35 | * 36 | * @param Type of the elements appearing in the pattern and produced elements based on found 37 | * matches. 38 | */ 39 | @PublicEvolving 40 | public class DefaultPatternProcessor implements PatternProcessor { 41 | 42 | /** The ID of the pattern processor. */ 43 | private final String id; 44 | 45 | /** The version of the pattern processor. */ 46 | private final Integer version; 47 | 48 | /** The pattern of the pattern processor. */ 49 | private final String patternStr; 50 | 51 | private final @Nullable PatternProcessFunction patternProcessFunction; 52 | 53 | public DefaultPatternProcessor( 54 | final String id, 55 | final Integer version, 56 | final String pattern, 57 | final @Nullable PatternProcessFunction patternProcessFunction, 58 | final ClassLoader userCodeClassLoader) { 59 | this.id = checkNotNull(id); 60 | this.version = checkNotNull(version); 61 | this.patternStr = checkNotNull(pattern); 62 | this.patternProcessFunction = patternProcessFunction; 63 | } 64 | 65 | @Override 66 | public String toString() { 67 | return "DefaultPatternProcessor{" 68 | + "id='" 69 | + id 70 | + '\'' 71 | + ", version=" 72 | + version 73 | + ", pattern=" 74 | + patternStr 75 | + ", patternProcessFunction=" 76 | + patternProcessFunction 77 | + '}'; 78 | } 79 | 80 | @Override 81 | public String getId() { 82 | return id; 83 | } 84 | 85 | @Override 86 | public int getVersion() { 87 | return version; 88 | } 89 | 90 | @Override 91 | public Pattern getPattern(ClassLoader classLoader) { 92 | try { 93 | return (Pattern) CepJsonUtils.convertJSONStringToPattern(patternStr, classLoader); 94 | } catch (Exception e) { 95 | throw new RuntimeException(e); 96 | } 97 | } 98 | 99 | /** 100 | * Returns the {@link PatternProcessFunction} to collect all the found matches. 101 | * 102 | * @return The pattern process function of the pattern processor. 103 | */ 104 | @Override 105 | public PatternProcessFunction getPatternProcessFunction() { 106 | return patternProcessFunction; 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/ververica/cep/demo/CepDemo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KTD, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.alibaba.ververica.cep.demo; 20 | 21 | import org.apache.flink.api.common.eventtime.WatermarkStrategy; 22 | import org.apache.flink.api.common.typeinfo.TypeHint; 23 | import org.apache.flink.api.common.typeinfo.TypeInformation; 24 | import org.apache.flink.api.java.functions.KeySelector; 25 | import org.apache.flink.api.java.tuple.Tuple2; 26 | import org.apache.flink.api.java.utils.MultipleParameterTool; 27 | import org.apache.flink.cep.CEP; 28 | import org.apache.flink.cep.TimeBehaviour; 29 | import org.apache.flink.cep.dynamic.impl.json.util.CepJsonUtils; 30 | import org.apache.flink.cep.nfa.aftermatch.AfterMatchSkipStrategy; 31 | import org.apache.flink.cep.pattern.Pattern; 32 | import org.apache.flink.connector.kafka.source.KafkaSource; 33 | import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer; 34 | import org.apache.flink.streaming.api.datastream.DataStreamSource; 35 | import org.apache.flink.streaming.api.datastream.KeyedStream; 36 | import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; 37 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 38 | 39 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; 40 | 41 | import com.alibaba.ververica.cep.demo.condition.EndCondition; 42 | import com.alibaba.ververica.cep.demo.condition.StartCondition; 43 | import com.alibaba.ververica.cep.demo.dynamic.JDBCPeriodicPatternProcessorDiscovererFactory; 44 | import com.alibaba.ververica.cep.demo.event.Event; 45 | import com.alibaba.ververica.cep.demo.event.EventDeSerializationSchema; 46 | 47 | import static com.alibaba.ververica.cep.demo.Constants.INPUT_TOPIC_ARG; 48 | import static com.alibaba.ververica.cep.demo.Constants.INPUT_TOPIC_GROUP_ARG; 49 | import static com.alibaba.ververica.cep.demo.Constants.JDBC_DRIVE; 50 | import static com.alibaba.ververica.cep.demo.Constants.JDBC_INTERVAL_MILLIS_ARG; 51 | import static com.alibaba.ververica.cep.demo.Constants.JDBC_URL_ARG; 52 | import static com.alibaba.ververica.cep.demo.Constants.KAFKA_BROKERS_ARG; 53 | import static com.alibaba.ververica.cep.demo.Constants.TABLE_NAME_ARG; 54 | 55 | public class CepDemo { 56 | 57 | public static void printTestPattern(Pattern pattern) throws JsonProcessingException { 58 | System.out.println(CepJsonUtils.convertPatternToJSONString(pattern)); 59 | } 60 | 61 | public static void checkArg(String argName, MultipleParameterTool params) { 62 | if (!params.has(argName)) { 63 | throw new IllegalArgumentException(argName + " must be set!"); 64 | } 65 | } 66 | 67 | public static void main(String[] args) throws Exception { 68 | // Process args 69 | final MultipleParameterTool params = MultipleParameterTool.fromArgs(args); 70 | checkArg(KAFKA_BROKERS_ARG, params); 71 | checkArg(INPUT_TOPIC_ARG, params); 72 | checkArg(INPUT_TOPIC_GROUP_ARG, params); 73 | checkArg(JDBC_URL_ARG, params); 74 | checkArg(TABLE_NAME_ARG, params); 75 | checkArg(JDBC_INTERVAL_MILLIS_ARG, params); 76 | 77 | // Set up the streaming execution environment 78 | final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 79 | 80 | // Build Kafka source with new Source API based on FLIP-27 81 | KafkaSource kafkaSource = 82 | KafkaSource.builder() 83 | .setBootstrapServers(params.get(KAFKA_BROKERS_ARG)) 84 | .setTopics(params.get(INPUT_TOPIC_ARG)) 85 | .setStartingOffsets(OffsetsInitializer.latest()) 86 | .setGroupId(params.get(INPUT_TOPIC_GROUP_ARG)) 87 | .setDeserializer(new EventDeSerializationSchema()) 88 | .build(); 89 | // DataStream Source 90 | DataStreamSource source = 91 | env.fromSource( 92 | kafkaSource, 93 | WatermarkStrategy.forMonotonousTimestamps() 94 | .withTimestampAssigner((event, ts) -> event.getEventTime()), 95 | "Kafka Source"); 96 | 97 | env.setParallelism(1); 98 | 99 | // keyBy userId and productionId 100 | // Notes, only events with the same key will be processd to see if there is a match 101 | KeyedStream> keyedStream = 102 | source.keyBy( 103 | new KeySelector>() { 104 | 105 | @Override 106 | public Tuple2 getKey(Event value) throws Exception { 107 | return Tuple2.of(value.getId(), value.getProductionId()); 108 | } 109 | }); 110 | 111 | // show how to print test pattern in json format 112 | Pattern pattern = 113 | Pattern.begin("start", AfterMatchSkipStrategy.skipPastLastEvent()) 114 | .where(new StartCondition("action == 0")) 115 | .timesOrMore(3) 116 | .followedBy("end") 117 | .where(new EndCondition()); 118 | printTestPattern(pattern); 119 | 120 | // Dynamic CEP patterns 121 | SingleOutputStreamOperator output = 122 | CEP.dynamicPatterns( 123 | keyedStream, 124 | new JDBCPeriodicPatternProcessorDiscovererFactory<>( 125 | params.get(JDBC_URL_ARG), 126 | JDBC_DRIVE, 127 | params.get(TABLE_NAME_ARG), 128 | null, 129 | Long.parseLong(params.get(JDBC_INTERVAL_MILLIS_ARG))), 130 | TimeBehaviour.ProcessingTime, 131 | TypeInformation.of(new TypeHint() {})); 132 | // Print output stream in taskmanager's stdout 133 | output.print(); 134 | // Compile and submit the job 135 | env.execute("CEPDemo"); 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.alibaba.ververica 8 | cep-demo 9 | 1.0-SNAPSHOT 10 | jar 11 | 12 | 13 | UTF-8 14 | 1.17.2 15 | 1.17-vvr-8.0.8 16 | 1.8 17 | 2.11 18 | ${target.java.version} 19 | ${target.java.version} 20 | 2.17.1 21 | 22 | 23 | 24 | 25 | 26 | 27 | org.apache.flink 28 | flink-java 29 | ${flink.version} 30 | provided 31 | 32 | 33 | org.apache.flink 34 | flink-clients 35 | ${flink.version} 36 | provided 37 | 38 | 39 | org.apache.flink 40 | flink-streaming-java 41 | ${flink.version} 42 | provided 43 | 44 | 45 | com.googlecode.aviator 46 | aviator 47 | 5.3.1 48 | 49 | 50 | com.alibaba.ververica 51 | flink-cep 52 | ${vvr.version} 53 | provided 54 | 55 | 56 | 57 | 58 | 59 | org.slf4j 60 | slf4j-api 61 | 2.0.0 62 | provided 63 | 64 | 65 | org.apache.logging.log4j 66 | log4j-slf4j-impl 67 | ${log4j.version} 68 | runtime 69 | 70 | 71 | org.apache.logging.log4j 72 | log4j-api 73 | ${log4j.version} 74 | runtime 75 | 76 | 77 | org.apache.logging.log4j 78 | log4j-core 79 | ${log4j.version} 80 | runtime 81 | 82 | 83 | mysql 84 | mysql-connector-java 85 | 8.0.28 86 | 87 | 88 | 89 | org.apache.flink 90 | flink-connector-kafka 91 | ${flink.version} 92 | 93 | 94 | 95 | org.apache.kafka 96 | kafka-clients 97 | 3.2.3 98 | 99 | 100 | commons-cli 101 | commons-cli 102 | 1.5.0 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | org.apache.maven.plugins 111 | maven-compiler-plugin 112 | 3.1 113 | 114 | ${target.java.version} 115 | ${target.java.version} 116 | 117 | 118 | 119 | com.diffplug.spotless 120 | spotless-maven-plugin 121 | 2.4.2 122 | 123 | 124 | 125 | 1.7 126 | 127 | 128 | 129 | 130 | 131 | org.apache.flink,org.apache.flink.shaded,,javax,java,scala,\# 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | org.apache.maven.plugins 141 | maven-shade-plugin 142 | 3.1.0 143 | 144 | 145 | package 146 | 147 | 148 | shade 149 | 150 | 151 | 152 | 153 | org.apache.flink:force-shading 154 | com.google.code.findbugs:jsr305 155 | org.slf4j:* 156 | org.apache.logging.log4j:* 157 | 158 | 159 | false 160 | true 161 | jar-with-dependencies 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | -------------------------------------------------------------------------------- /src/main/java/com/alibaba/ververica/cep/demo/dynamic/JDBCPeriodicPatternProcessorDiscoverer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KTD, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.alibaba.ververica.cep.demo.dynamic; 20 | 21 | import org.apache.flink.api.java.tuple.Tuple4; 22 | import org.apache.flink.cep.dynamic.impl.json.deserializer.ConditionSpecStdDeserializer; 23 | import org.apache.flink.cep.dynamic.impl.json.deserializer.NodeSpecStdDeserializer; 24 | import org.apache.flink.cep.dynamic.impl.json.deserializer.TimeStdDeserializer; 25 | import org.apache.flink.cep.dynamic.impl.json.spec.ConditionSpec; 26 | import org.apache.flink.cep.dynamic.impl.json.spec.GraphSpec; 27 | import org.apache.flink.cep.dynamic.impl.json.spec.NodeSpec; 28 | import org.apache.flink.cep.dynamic.processor.PatternProcessor; 29 | import org.apache.flink.cep.functions.PatternProcessFunction; 30 | import org.apache.flink.streaming.api.windowing.time.Time; 31 | import org.apache.flink.util.CollectionUtil; 32 | import org.apache.flink.util.StringUtils; 33 | 34 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; 35 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.SerializationFeature; 36 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.module.SimpleModule; 37 | 38 | import org.slf4j.Logger; 39 | import org.slf4j.LoggerFactory; 40 | 41 | import javax.annotation.Nullable; 42 | 43 | import java.io.IOException; 44 | import java.sql.Connection; 45 | import java.sql.DriverManager; 46 | import java.sql.ResultSet; 47 | import java.sql.SQLException; 48 | import java.sql.Statement; 49 | import java.util.HashMap; 50 | import java.util.List; 51 | import java.util.Map; 52 | import java.util.stream.Collectors; 53 | 54 | import static java.util.Objects.requireNonNull; 55 | 56 | /** 57 | * The JDBC implementation of the {@link PeriodicPatternProcessorDiscoverer} that periodically 58 | * discovers the rule updates from the database by using JDBC. 59 | * 60 | * @param Base type of the elements appearing in the pattern. 61 | */ 62 | public class JDBCPeriodicPatternProcessorDiscoverer 63 | extends PeriodicPatternProcessorDiscoverer { 64 | 65 | private static final Logger LOG = 66 | LoggerFactory.getLogger(JDBCPeriodicPatternProcessorDiscoverer.class); 67 | 68 | private final String tableName; 69 | private final String jdbcUrl; 70 | private final List> initialPatternProcessors; 71 | private final ClassLoader userCodeClassLoader; 72 | private Connection connection; 73 | private Statement statement; 74 | private ResultSet resultSet; 75 | private Map> latestPatternProcessors; 76 | 77 | /** 78 | * Creates a new using the given initial {@link PatternProcessor} and the time interval how 79 | * often to check the pattern processor updates. 80 | * 81 | * @param jdbcUrl The JDBC url of the database. 82 | * @param jdbcDriver The JDBC driver of the database. 83 | * @param initialPatternProcessors The list of the initial {@link PatternProcessor}. 84 | * @param intervalMillis Time interval in milliseconds how often to check updates. 85 | */ 86 | public JDBCPeriodicPatternProcessorDiscoverer( 87 | final String jdbcUrl, 88 | final String jdbcDriver, 89 | final String tableName, 90 | final ClassLoader userCodeClassLoader, 91 | @Nullable final List> initialPatternProcessors, 92 | @Nullable final Long intervalMillis) 93 | throws Exception { 94 | super(intervalMillis); 95 | this.tableName = requireNonNull(tableName); 96 | this.initialPatternProcessors = initialPatternProcessors; 97 | this.userCodeClassLoader = userCodeClassLoader; 98 | this.jdbcUrl = jdbcUrl; 99 | Class.forName(requireNonNull(jdbcDriver)); 100 | this.connection = DriverManager.getConnection(requireNonNull(jdbcUrl)); 101 | this.statement = this.connection.createStatement(); 102 | } 103 | 104 | @Override 105 | public boolean arePatternProcessorsUpdated() { 106 | if (latestPatternProcessors == null 107 | && !CollectionUtil.isNullOrEmpty(initialPatternProcessors)) { 108 | return true; 109 | } 110 | 111 | if (statement == null) { 112 | return false; 113 | } 114 | try { 115 | resultSet = statement.executeQuery("SELECT * FROM " + tableName); 116 | Map> currentPatternProcessors = 117 | new HashMap<>(); 118 | while (resultSet.next()) { 119 | String id = resultSet.getString("id"); 120 | if (currentPatternProcessors.containsKey(id) 121 | && currentPatternProcessors.get(id).f1 >= resultSet.getInt("version")) { 122 | continue; 123 | } 124 | currentPatternProcessors.put( 125 | id, 126 | new Tuple4<>( 127 | requireNonNull(resultSet.getString("id")), 128 | resultSet.getInt("version"), 129 | requireNonNull(resultSet.getString("pattern")), 130 | resultSet.getString("function"))); 131 | } 132 | if (latestPatternProcessors == null 133 | || isPatternProcessorUpdated(currentPatternProcessors)) { 134 | latestPatternProcessors = currentPatternProcessors; 135 | return true; 136 | } else { 137 | return false; 138 | } 139 | } catch (SQLException e) { 140 | LOG.warn( 141 | "Pattern processor discoverer failed to check rule changes, will recreate connection - " 142 | + e.getMessage()); 143 | try { 144 | statement.close(); 145 | connection.close(); 146 | connection = DriverManager.getConnection(requireNonNull(this.jdbcUrl)); 147 | statement = connection.createStatement(); 148 | } catch (SQLException ex) { 149 | throw new RuntimeException("Cannot recreate connection to database."); 150 | } 151 | } 152 | return false; 153 | } 154 | 155 | @SuppressWarnings("unchecked") 156 | @Override 157 | public List> getLatestPatternProcessors() throws Exception { 158 | ObjectMapper objectMapper = 159 | new ObjectMapper() 160 | .registerModule( 161 | new SimpleModule() 162 | .addDeserializer( 163 | ConditionSpec.class, 164 | ConditionSpecStdDeserializer.INSTANCE) 165 | .addDeserializer(Time.class, TimeStdDeserializer.INSTANCE) 166 | .addDeserializer( 167 | NodeSpec.class, NodeSpecStdDeserializer.INSTANCE)); 168 | 169 | return latestPatternProcessors.values().stream() 170 | .map( 171 | patternProcessor -> { 172 | try { 173 | String patternStr = patternProcessor.f2; 174 | GraphSpec graphSpec = 175 | objectMapper.readValue(patternStr, GraphSpec.class); 176 | objectMapper.enable(SerializationFeature.INDENT_OUTPUT); 177 | System.out.println( 178 | objectMapper 179 | .writerWithDefaultPrettyPrinter() 180 | .writeValueAsString(graphSpec)); 181 | PatternProcessFunction patternProcessFunction = null; 182 | String id = patternProcessor.f0; 183 | int version = patternProcessor.f1; 184 | 185 | if (!StringUtils.isNullOrWhitespaceOnly(patternProcessor.f3)) { 186 | patternProcessFunction = 187 | (PatternProcessFunction) 188 | this.userCodeClassLoader 189 | .loadClass(patternProcessor.f3) 190 | .getConstructor(String.class, int.class) 191 | .newInstance(id, version); 192 | } 193 | LOG.warn( 194 | objectMapper 195 | .writerWithDefaultPrettyPrinter() 196 | .writeValueAsString(patternProcessor.f2)); 197 | return new DefaultPatternProcessor<>( 198 | patternProcessor.f0, 199 | patternProcessor.f1, 200 | patternStr, 201 | patternProcessFunction, 202 | this.userCodeClassLoader); 203 | } catch (Exception e) { 204 | 205 | LOG.error( 206 | "Get the latest pattern processors of the discoverer failure. - " 207 | + e.getMessage()); 208 | e.printStackTrace(); 209 | } 210 | return null; 211 | }) 212 | .collect(Collectors.toList()); 213 | } 214 | 215 | @Override 216 | public void close() throws IOException { 217 | super.close(); 218 | try { 219 | if (resultSet != null) { 220 | resultSet.close(); 221 | } 222 | } catch (SQLException e) { 223 | LOG.warn( 224 | "ResultSet of the pattern processor discoverer couldn't be closed - " 225 | + e.getMessage()); 226 | } finally { 227 | resultSet = null; 228 | } 229 | try { 230 | if (statement != null) { 231 | statement.close(); 232 | } 233 | } catch (SQLException e) { 234 | LOG.warn( 235 | "Statement of the pattern processor discoverer couldn't be closed - " 236 | + e.getMessage()); 237 | } finally { 238 | statement = null; 239 | } 240 | } 241 | 242 | private boolean isPatternProcessorUpdated( 243 | Map> currentPatternProcessors) { 244 | return latestPatternProcessors.size() != currentPatternProcessors.size() 245 | || !currentPatternProcessors.equals(latestPatternProcessors); 246 | } 247 | } 248 | --------------------------------------------------------------------------------