├── .gitignore ├── LICENSE ├── README.md ├── flink-cdc-bundle └── pom.xml ├── flink-cdc ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── hiscat │ │ │ └── flink │ │ │ └── cdc │ │ │ ├── function │ │ │ └── MysqlCdcSourceRegister.java │ │ │ ├── model │ │ │ ├── CdcPayload.java │ │ │ └── DbProperties.java │ │ │ └── util │ │ │ ├── ConfigUtil.java │ │ │ ├── DdlGenerator.java │ │ │ ├── HiveDdlGenerator.java │ │ │ ├── JsonUtil.java │ │ │ ├── KafkaUtil.java │ │ │ ├── MysqlDdlPrinter.java │ │ │ └── MysqlSourceOptionsPrinter.java │ └── resources │ │ ├── flink-conf.default.yaml │ │ ├── hive.ddl.template │ │ ├── hive.dml.template │ │ ├── msk2hive.ddl.template │ │ ├── msk2hive.dml.template │ │ ├── msk2mysql.ddl.template │ │ └── msk2mysql.dml.template │ └── test │ └── resources │ └── cdc.sql ├── flink-connector-hive_1.13 ├── pom.xml └── src │ └── main │ └── java │ └── org │ └── apache │ └── flink │ └── table │ └── catalog │ └── hive │ └── client │ ├── HiveShimLoader.java │ ├── HiveShimV239.java │ └── HiveShimV239Amzn0.java ├── flink-connector-hive_1.14 ├── pom.xml └── src │ └── main │ └── java │ └── org │ └── apache │ └── flink │ └── table │ └── catalog │ └── hive │ └── client │ ├── HiveShimLoader.java │ ├── HiveShimV239.java │ └── HiveShimV239Amzn0.java ├── flink-connector-jdbc_1.13 ├── pom.xml └── src │ ├── main │ ├── java │ │ ├── com │ │ │ └── hiscat │ │ │ │ └── flink │ │ │ │ └── connector │ │ │ │ └── jdbc │ │ │ │ └── table │ │ │ │ ├── FieldStringGetter.java │ │ │ │ ├── PartialUpdateJdbcDynamicTableFactory.java │ │ │ │ ├── PartialUpdateJdbcDynamicTableSink.java │ │ │ │ ├── PartialUpdateOptions.java │ │ │ │ ├── PrepareStatementTagExtractor.java │ │ │ │ ├── RedshiftDynamicTableFactory.java │ │ │ │ ├── RedshiftDynamicTableSink.java │ │ │ │ ├── converter │ │ │ │ └── PartialUpdateMySQLRowConverter.java │ │ │ │ └── executor │ │ │ │ ├── InsertOnlyBufferedStringStatementExecutor.java │ │ │ │ ├── PartialUpdateJdbcBatchStatementExecutor.java │ │ │ │ └── PartialUpdateTableBufferReducedStatementExecutorFactory.java │ │ └── org │ │ │ └── apache │ │ │ └── flink │ │ │ └── connector │ │ │ └── jdbc │ │ │ ├── catalog │ │ │ ├── JdbcCatalogUtils.java │ │ │ └── MySqlCatalog.java │ │ │ ├── dialect │ │ │ ├── JdbcDialectTypeMapper.java │ │ │ ├── JdbcDialects.java │ │ │ ├── RedshiftDialect.java │ │ │ └── mysql │ │ │ │ └── MySqlTypeMapper.java │ │ │ └── internal │ │ │ └── converter │ │ │ └── RedshiftRowConverter.java │ └── resources │ │ └── META-INF │ │ └── services │ │ └── org.apache.flink.table.factories.Factory │ └── test │ └── resources │ ├── cdc-2-kafka.sql │ ├── dg-2-kafka.sql │ ├── dws-2-mysql.sql │ ├── kafka-2-msql.sql │ └── partial_update.sql ├── flink-connector-jdbc_1.14 ├── pom.xml └── src │ └── main │ ├── java │ └── org │ │ └── apache │ │ └── flink │ │ └── connector │ │ └── jdbc │ │ ├── catalog │ │ ├── JdbcCatalogUtils.java │ │ └── MySqlCatalog.java │ │ ├── dialect │ │ ├── JdbcDialectTypeMapper.java │ │ └── mysql │ │ │ └── MySqlTypeMapper.java │ │ ├── internal │ │ └── converter │ │ │ ├── AbstractJdbcRowConverter.java │ │ │ └── PartialUpdateMySQLRowConverter.java │ │ ├── table │ │ ├── RouteJdbcDynamicTableFactory.java │ │ ├── RouteJdbcDynamicTableSink.java │ │ ├── TableExtractor.java │ │ └── executor │ │ │ ├── AbstractRouteJdbcBatchStatementExecutor.java │ │ │ ├── RouteJdbcDeleteBatchStatementExecutor.java │ │ │ ├── RouteJdbcUpsertBatchStatementExecutor.java │ │ │ └── RouteTableBufferReducedStatementExecutorFactory.java │ │ └── utils │ │ └── JdbcTypeUtil.java │ └── resources │ └── META-INF │ └── services │ └── org.apache.flink.table.factories.Factory ├── flink-connector-kafka ├── pom.xml └── src │ └── main │ ├── java │ └── org │ │ └── apache │ │ └── flink │ │ └── streaming │ │ └── connectors │ │ └── kafka │ │ └── table │ │ ├── RouteDynamicKafkaRecordSerializationSchema.java │ │ ├── RouteKafkaDynamicSink.java │ │ └── RouteKafkaDynamicTableFactory.java │ └── resources │ └── META-INF │ └── services │ └── org.apache.flink.table.factories.Factory ├── flink-custom-connector ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── hiscat │ │ └── flink │ │ └── custrom │ │ ├── connector │ │ ├── nsq │ │ │ ├── NsqDynamicTableFactory.java │ │ │ ├── NsqDynamicTableSink.java │ │ │ ├── NsqDynamicTableSource.java │ │ │ ├── NsqOptions.java │ │ │ ├── NsqSinkFunction.java │ │ │ ├── NsqSourceFunction.java │ │ │ └── util │ │ │ │ ├── AbstractTimeIndexGenerator.java │ │ │ │ ├── IndexGenerator.java │ │ │ │ ├── IndexGeneratorBase.java │ │ │ │ ├── IndexGeneratorFactory.java │ │ │ │ └── StaticIndexGenerator.java │ │ └── redis │ │ │ ├── ConnectionProvider.java │ │ │ ├── RedisAsyncLookupFunction.java │ │ │ ├── RedisDynamicTableFactory.java │ │ │ ├── RedisLookupFunction.java │ │ │ ├── RedisLookupTableSource.java │ │ │ └── RedisOptions.java │ │ └── format │ │ └── json │ │ ├── JsonOptions.java │ │ ├── JsonRowDataDeserializationSchema.java │ │ ├── JsonRowDataSerializationSchema.java │ │ ├── JsonToRowDataConverters.java │ │ ├── TimeFormats.java │ │ └── ogg │ │ ├── OggJsonDecodingFormat.java │ │ ├── OggJsonDeserializationSchema.java │ │ ├── OggJsonFormatFactory.java │ │ ├── OggJsonOptions.java │ │ └── OggJsonSerializationSchema.java │ └── resources │ └── META-INF │ └── services │ └── org.apache.flink.table.factories.Factory ├── flink-prometheus ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── hiscat │ │ └── flink │ │ └── prometheus │ │ ├── AutoServiceDiscoveryPrometheusReporter.java │ │ ├── AutoServiceDiscoveryPrometheusReporterFactory.java │ │ └── sd │ │ ├── BaseZookeeperServiceDiscovery.java │ │ ├── JobManagerZookeeperServiceDiscovery.java │ │ ├── ServiceDiscovery.java │ │ ├── ServiceDiscoveryFactory.java │ │ ├── ServiceDiscoveryLoader.java │ │ ├── TaskManagerZookeeperServiceDiscovery.java │ │ ├── ZookeeperServiceDiscoveryFactory.java │ │ └── ZookeeperServiceDiscoveryOptions.java │ └── resources │ └── META-INF │ └── services │ ├── com.hiscat.flink.prometheus.sd.ServiceDiscoveryFactory │ └── org.apache.flink.metrics.reporter.MetricReporterFactory ├── flink-sql-submitter ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── hiscat │ └── flink │ ├── SqlSubmitter.java │ ├── cli │ ├── CliStatementSplitter.java │ └── SqlRunnerOptions.java │ └── function │ └── CallContext.java ├── flink-udf ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── hiscat │ └── flink │ ├── Millisecond2LocalDateTimeString.java │ └── TopicGetter.java ├── image └── how-to-run-sql.png └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.log 3 | .idea 4 | target 5 | *.iml 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 hiscat 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # flink-demo 2 | 3 | sql 代码混合执行 4 | 5 | ```sql 6 | 7 | call com.hiscat.flink.cdc.function.MysqlCdcSourceRegister; 8 | 9 | CREATE TABLE ods_binlog_default 10 | ( 11 | `key` STRING, 12 | `value` STRING, 13 | `topic` STRING METADATA FROM 'topic' 14 | ) WITH ( 15 | 'connector' = 'route-kafka', 16 | 'topic' = 'ods_binlog_default', 17 | 'properties.bootstrap.servers' = 'localhost:9092', 18 | 'key.format' = 'raw', 19 | 'key.fields' = 'key', 20 | 'value.format' = 'raw', 21 | 'value.fields-include' = 'EXCEPT_KEY', 22 | 'properties.compression.type' = 'gzip', 23 | 'properties.linger.ms' = '1000' 24 | ); 25 | 26 | INSERT INTO ods_binlog_default 27 | SELECT `key`, `value`, get_topic(`db`, `table`) 28 | FROM cdc; 29 | 30 | ``` -------------------------------------------------------------------------------- /flink-cdc-bundle/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.hiscat.flink.cdc 8 | flink-cdc-bundle 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 8 13 | 8 14 | 2.2.1 15 | 3.3.0 16 | 18.0-13.0 17 | 2.17.172 18 | 19 | 20 | 21 | com.ververica 22 | flink-connector-mysql-cdc 23 | ${flink-connector-mysql-cdc.version} 24 | 25 | 26 | org.slf4j 27 | slf4j-api 28 | 29 | 30 | 31 | 32 | software.amazon.awssdk 33 | secretsmanager 34 | ${aws-sdk-java.version} 35 | 36 | 37 | 38 | 39 | 40 | org.apache.maven.plugins 41 | maven-shade-plugin 42 | ${maven-shade-plugin.version} 43 | 44 | 45 | 46 | 47 | false 48 | 49 | 50 | 51 | package 52 | 53 | shade 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /flink-cdc/src/main/java/com/hiscat/flink/cdc/model/CdcPayload.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.cdc.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | @Data 9 | @AllArgsConstructor 10 | @NoArgsConstructor 11 | @Builder 12 | public class CdcPayload { 13 | private String key; 14 | private String value; 15 | private String db; 16 | private String table; 17 | private long ts; 18 | } 19 | -------------------------------------------------------------------------------- /flink-cdc/src/main/java/com/hiscat/flink/cdc/model/DbProperties.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.cdc.model; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; 8 | 9 | @Data 10 | @Builder 11 | @AllArgsConstructor 12 | @NoArgsConstructor 13 | @JsonIgnoreProperties(ignoreUnknown = true) 14 | public class DbProperties { 15 | private String username; 16 | private String password; 17 | private String host; 18 | private int port; 19 | 20 | } 21 | -------------------------------------------------------------------------------- /flink-cdc/src/main/java/com/hiscat/flink/cdc/util/ConfigUtil.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.cdc.util; 2 | 3 | import com.hiscat.flink.cdc.model.DbProperties; 4 | import org.apache.commons.io.IOUtils; 5 | import org.apache.flink.core.fs.FSDataInputStream; 6 | import org.apache.flink.core.fs.FileSystem; 7 | import org.apache.flink.core.fs.Path; 8 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; 9 | 10 | import java.io.IOException; 11 | import java.net.URI; 12 | import java.nio.charset.StandardCharsets; 13 | 14 | public class ConfigUtil { 15 | public static JsonNode readConfig(String path) throws IOException { 16 | URI uri = URI.create(path); 17 | final JsonNode config; 18 | try (FSDataInputStream fsDataInputStream = FileSystem.get(uri).open(new Path(uri))) { 19 | String yaml = IOUtils.toString(fsDataInputStream, StandardCharsets.UTF_8); 20 | config = JsonUtil.OBJECT_MAPPER.readTree(yaml); 21 | } 22 | return config; 23 | } 24 | 25 | static DbProperties getDbProperties(JsonNode config, String db) throws IOException { 26 | JsonNode node = config.at(String.format("/db/%s", db)); 27 | JsonNode usernameNode = node.get("username"); 28 | if (usernameNode != null && !usernameNode.isNull()) { 29 | String password = node.get("password").asText(); 30 | String host = node.get("host").asText(); 31 | int port = node.get("port").asInt(); 32 | return new DbProperties(usernameNode.asText(), password, host, port); 33 | } 34 | String arn = config.at(String.format("/db/%s/arn", db)).asText(); 35 | return SecretUtil.getSecret(arn, "ap-southeast-1"); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /flink-cdc/src/main/java/com/hiscat/flink/cdc/util/HiveDdlGenerator.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.cdc.util; 2 | 3 | import java.io.IOException; 4 | 5 | public class HiveDdlGenerator extends DdlGenerator { 6 | 7 | public HiveDdlGenerator(String target, String db, String confPath) throws IOException { 8 | super(target, db, confPath); 9 | } 10 | 11 | public static void main(String[] args) throws IOException { 12 | new HiveDdlGenerator(args[0], args[1], args[2]).generate(); 13 | } 14 | 15 | protected String formatColsSql(String table) { 16 | return String.format( 17 | "select group_concat(case DATA_TYPE\n" + 18 | " when 'bit' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'int', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 19 | " when 'tinyint' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'int', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 20 | " when 'smallint' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'int', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 21 | " when 'int' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'int', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 22 | " when 'bigint' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'bigint', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 23 | " when 'float' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'float', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 24 | " when 'double' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'double', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 25 | " when 'decimal' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), COLUMN_TYPE, 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 26 | " when 'char' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'string', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 27 | " when 'varchar' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'string', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 28 | " when 'enum' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'string', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 29 | " when 'longtext' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'string', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 30 | " when 'text' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'string', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 31 | " when 'mediumtext' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'string', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 32 | " when 'json' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'string', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 33 | " when 'set' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'string', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 34 | " when 'binary' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'binary', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 35 | " when 'varbinary' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'binary', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 36 | " when 'blob' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'binary', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 37 | " when 'longblob' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'binary', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 38 | " when 'mediumblob' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'binary', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 39 | " when 'time' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'string', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 40 | " when 'date' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'string', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 41 | " when 'timestamp' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'string', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 42 | " when 'datetime' THEN CONCAT_WS(' ', CONCAT('`', COLUMN_NAME, '`'), 'string', 'COMMENT', CONCAT('\\'',COLUMN_COMMENT,'\\''))\n" + 43 | " ELSE 'other'\n" + 44 | " END separator ',\\n') as cols,\n" + 45 | " GROUP_CONCAT(CONCAT('IF(op <> \\'d\\', after.`', COLUMN_NAME, '`, before.`', COLUMN_NAME, '`)')\n" + 46 | " SEPARATOR ',\\n') as insert_cols\n" + 47 | "from information_schema.COLUMNS\n" + 48 | "where TABLE_SCHEMA = '%s' and TABLE_NAME = '%s'\n" + 49 | "order by ORDINAL_POSITION", 50 | this.db, 51 | table); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /flink-cdc/src/main/java/com/hiscat/flink/cdc/util/JsonUtil.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.cdc.util; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Collections; 5 | import java.util.HashMap; 6 | import java.util.List; 7 | import java.util.Map; 8 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; 9 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; 10 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; 11 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.yaml.YAMLFactory; 12 | 13 | public class JsonUtil { 14 | 15 | static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory()); 16 | 17 | static Map getMap(JsonNode tree, String field) { 18 | JsonNode node = tree.get(field); 19 | if (node.isNull()) { 20 | return Collections.emptyMap(); 21 | } 22 | ArrayNode single = (ArrayNode) node; 23 | Map singleMap = new HashMap<>(single.size()); 24 | for (JsonNode jsonNode : single) { 25 | String property = jsonNode.fieldNames().next(); 26 | singleMap.put(property, jsonNode.get(property).asText()); 27 | } 28 | return singleMap; 29 | } 30 | 31 | static List getList(JsonNode tree, String field) { 32 | JsonNode node = tree.get(field); 33 | if (node.isNull()) { 34 | return Collections.emptyList(); 35 | } 36 | ArrayNode single = (ArrayNode) node; 37 | List result = new ArrayList<>(single.size()); 38 | for (JsonNode jsonNode : single) { 39 | result.add(jsonNode.asText()); 40 | } 41 | return result; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /flink-cdc/src/main/java/com/hiscat/flink/cdc/util/KafkaUtil.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.cdc.util; 2 | 3 | import static java.util.stream.Collectors.toList; 4 | 5 | import java.util.Comparator; 6 | import java.util.HashMap; 7 | import java.util.List; 8 | import java.util.Map; 9 | import java.util.concurrent.ExecutionException; 10 | import org.apache.kafka.clients.CommonClientConfigs; 11 | import org.apache.kafka.clients.admin.Admin; 12 | import org.apache.kafka.clients.admin.TopicDescription; 13 | import org.apache.kafka.clients.admin.TopicListing; 14 | 15 | public class KafkaUtil { 16 | public static void main(String[] args) { 17 | Map props = new HashMap<>(); 18 | props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, args[0]); 19 | try (Admin admin = Admin.create(props)) { 20 | List topics = 21 | admin.listTopics().listings().get().stream().map(TopicListing::name).collect(toList()); 22 | Map map = admin.describeTopics(topics).all().get(); 23 | List result = 24 | map.keySet().stream() 25 | .sorted(Comparator.comparingInt(o -> map.get(o).partitions().size())) 26 | .map(k -> String.format("topic:%s,partitions:%s", k, map.get(k).partitions().size())) 27 | .collect(toList()); 28 | result.forEach(System.out::println); 29 | } catch (ExecutionException | InterruptedException e) { 30 | e.printStackTrace(); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /flink-cdc/src/main/java/com/hiscat/flink/cdc/util/MysqlDdlPrinter.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.cdc.util; 2 | 3 | import com.hiscat.flink.cdc.model.DbProperties; 4 | import org.apache.flink.core.fs.FSDataOutputStream; 5 | import org.apache.flink.core.fs.FileSystem; 6 | import org.apache.flink.core.fs.Path; 7 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; 8 | 9 | import java.io.IOException; 10 | import java.net.URI; 11 | import java.sql.*; 12 | import java.util.List; 13 | 14 | 15 | public class MysqlDdlPrinter { 16 | private final String db; 17 | private final JsonNode config; 18 | 19 | public MysqlDdlPrinter(String db, String confPath) throws IOException { 20 | this.db = db; 21 | config = ConfigUtil.readConfig(confPath); 22 | } 23 | 24 | public static void main(String[] args) throws IOException, SQLException { 25 | new MysqlDdlPrinter(args[0], args[1]).generate(); 26 | } 27 | 28 | private void generate() throws IOException, SQLException { 29 | 30 | try (Connection connection = getConnection(ConfigUtil.getDbProperties(this.config, this.db)); 31 | Statement statement = connection.createStatement(); 32 | Statement pst = connection.createStatement(); 33 | FSDataOutputStream out = getFsDataOutputStream()) { 34 | statement.execute("use " + db); 35 | List tables = JsonUtil.getList(this.config.at(String.format("/db/%s", this.db)), "tables"); 36 | for (String table : tables) { 37 | try (ResultSet rs = pst.executeQuery("show create table " + table)) { 38 | if (rs.next()) { 39 | String ddl = rs.getString(2); 40 | String result = getResult(ddl); 41 | out.write(result.getBytes()); 42 | } 43 | } 44 | } 45 | } 46 | } 47 | 48 | private FSDataOutputStream getFsDataOutputStream() throws IOException { 49 | URI uri = URI.create(config.at(String.format("/db/%s/path", this.db)).asText()); 50 | FileSystem fs = FileSystem.get(uri); 51 | return fs.create(new Path(uri), FileSystem.WriteMode.OVERWRITE); 52 | } 53 | 54 | private Connection getConnection(DbProperties dbProperties) throws SQLException { 55 | return DriverManager.getConnection(makeJdbcUrl(dbProperties), 56 | dbProperties.getUsername(), dbProperties.getPassword()); 57 | } 58 | 59 | private String makeJdbcUrl(DbProperties dbProperties) { 60 | return String.format( 61 | "jdbc:mysql://%s:%s/information_schema", dbProperties.getHost(), dbProperties.getPort()); 62 | } 63 | 64 | private static String getResult(String ddl) { 65 | return appendIfNotExists(appendSemicolon(removeAutoIncrement(appendMetadataCols(ddl)))); 66 | } 67 | 68 | private static String appendMetadataCols(String ddl) { 69 | int pkIndex = ddl.indexOf("PRIMARY KEY"); 70 | return ddl.substring(0, pkIndex) 71 | .concat("`metadata_schema_name` varchar(255) COLLATE utf8mb4_bin NOT NULL,\n" + 72 | "`metadata_table_name` varchar(255) COLLATE utf8mb4_bin NOT NULL,\n" + 73 | "`metadata_timestamp` timestamp(3) NULL DEFAULT NULL,\n" + 74 | "`kafka_topic` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL,\n" + 75 | "`kafka_partition` int DEFAULT NULL,\n" + 76 | "`kafka_offset` bigint DEFAULT NULL,\n" + 77 | "`kafka_timestamp` timestamp(3) NULL DEFAULT NULL,\n" + 78 | "`kafka_timestamp_type` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL,\n") 79 | .concat(ddl.substring(pkIndex)); 80 | } 81 | 82 | private static String removeAutoIncrement(String concat) { 83 | String autoIncrement = "AUTO_INCREMENT"; 84 | int index = concat.indexOf(autoIncrement); 85 | if (index<0) { 86 | return concat; 87 | } 88 | return concat.substring(0, index) 89 | .concat(concat.substring(index + autoIncrement.length())); 90 | } 91 | 92 | private static String appendSemicolon(String result) { 93 | return pkAppendMetadataTableNameAndSchemaName(result).concat(";"); 94 | } 95 | 96 | private static String pkAppendMetadataTableNameAndSchemaName(String result) { 97 | int beginIndex = result.indexOf(")", result.indexOf("PRIMARY KEY (")); 98 | return result.substring(0, beginIndex) 99 | .concat(",`metadata_table_name`,`metadata_schema_name`") 100 | .concat(result.substring(beginIndex)); 101 | } 102 | 103 | private static String appendIfNotExists(String result) { 104 | String createTable = "CREATE TABLE"; 105 | return result.substring(0, createTable.length()) 106 | .concat(" IF NOT EXISTS ") 107 | .concat(result.substring(createTable.length())); 108 | } 109 | 110 | } 111 | -------------------------------------------------------------------------------- /flink-cdc/src/main/java/com/hiscat/flink/cdc/util/MysqlSourceOptionsPrinter.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.cdc.util; 2 | 3 | import com.ververica.cdc.connectors.mysql.source.config.MySqlSourceOptions; 4 | import java.lang.reflect.Field; 5 | import java.lang.reflect.Modifier; 6 | import java.util.Arrays; 7 | import lombok.SneakyThrows; 8 | import org.apache.flink.configuration.ConfigOption; 9 | 10 | public class MysqlSourceOptionsPrinter { 11 | public static void main(String[] args) { 12 | // 13 | Arrays.stream(MySqlSourceOptions.class.getDeclaredFields()) 14 | .filter(f -> Modifier.isStatic(f.getModifiers())) 15 | .map(MysqlSourceOptionsPrinter::apply) 16 | .forEach(System.out::println); 17 | } 18 | 19 | @SneakyThrows 20 | private static Object apply(Field f) { 21 | ConfigOption option = (ConfigOption) f.get(null); 22 | return String.format("SET %s=%s;", option.key(), option.defaultValue()); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /flink-cdc/src/main/resources/flink-conf.default.yaml: -------------------------------------------------------------------------------- 1 | jobmanager.rpc.address: 0.0.0.0 2 | jobmanager.memory.process.size: 4g 3 | taskmanager.memory.process.size: 4g 4 | taskmanager.numberOfTaskSlots: 4 5 | parallelism.default: 4 6 | 7 | env.yarn.conf.dir: /etc/hadoop/conf 8 | env.hadoop.conf.dir: /etc/hadoop/conf 9 | env.pid.dir: /var/run/flink 10 | 11 | fs.allowed-fallback-filesystems: s3 12 | 13 | jobmanager.execution.failover-strategy: region 14 | jobmanager.web.upload.dir: /var/lib/flink/upload 15 | 16 | yarn.properties-file.location: /var/lib/flink/yarn 17 | 18 | yarn.application-attempts: 10 19 | high-availability: zookeeper 20 | high-availability.zookeeper.quorum: ip-10-88-221-41.ap-southeast-1.compute.internal:2181,ip-10-88-221-52.ap-southeast-1.compute.internal:2181,ip-10-88-221-125.ap-southeast-1.compute.internal:2181 21 | high-availability.storageDir: hdfs:///user/flink/recovery 22 | high-availability.zookeeper.path.root: /flink 23 | 24 | classloader.check-leaked-classloader: false 25 | 26 | restart-strategy: fixed-delay 27 | restart-strategy.fixed-delay.attempts: 100 28 | restart-strategy.fixed-delay.delay: 1min 29 | 30 | execution.checkpointing.tolerable-failed-checkpoints: 100 31 | execution.checkpointing.externalized-checkpoint-retention: RETAIN_ON_CANCELLATION 32 | execution.checkpointing.interval: 1min 33 | execution.checkpointing.min-pause: 1min 34 | execution.checkpointing.timeout: 20min 35 | execution.checkpointing.unaligned: true 36 | execution.savepoint.ignore-unclaimed-state: true 37 | 38 | state.backend: hashmap 39 | state.checkpoint-storage: filesystem 40 | state.checkpoints.num-retained: 3 41 | state.backend.incremental: true 42 | heartbeat.timeout: 600000 43 | akka.ask.timeout: 10min 44 | taskmanager.slot.timeout: 10min 45 | client.timeout: 10min 46 | task.cancellation.timers.timeout: 600000 47 | task.cancellation.timeout: 600000 48 | env.java.opts: -Dlog4j2.formatMsgNoLookups=true -verbose:gc -server -XX:+PrintGCDetails -XX:+PrintGCTimeStamps 49 | yarn.containers.vcores: 1 50 | cluster.registration.max-timeout: 300000 51 | cluster.registration.initial-timeout: 300000 52 | cluster.registration.error-delay: 30000 53 | metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter 54 | metrics.reporter.prom.port: 9250-9350 55 | taskmanager.memory.managed.fraction: 0 56 | akka.retry-gate-closed-for: 10000 57 | -------------------------------------------------------------------------------- /flink-cdc/src/main/resources/hive.ddl.template: -------------------------------------------------------------------------------- 1 | 2 | CREATE EXTERNAL TABLE IF NOT EXISTS hive.ods.ods_{db}_{table}_realtime 3 | ( 4 | {cols}, 5 | 6 | `metadata_schema_name` string COMMENT '源库名', 7 | `metadata_table_name` string COMMENT '源表名', 8 | `metadata_timestamp` string COMMENT '源记录时间戳', 9 | `metadata_operation` string COMMENT '源记录操作类型', 10 | `metadata_operation_ts` string COMMENT '源记录操作时间', 11 | `kafka_topic` string COMMENT 'Kafka-主题', 12 | `kafka_partition` int COMMENT 'Kafka-分区', 13 | `kafka_offset` bigint COMMENT 'Kafka-位移', 14 | `kafka_timestamp` timestamp COMMENT 'Kafka-时戳', 15 | `kafka_timestamp_type` string COMMENT 'Kafka-时戳类型' 16 | ) 17 | PARTITIONED BY ( 18 | `dt` string) 19 | STORED AS PARQUET 20 | TBLPROPERTIES ('parquet.compression' = 'SNAPPY'); 21 | -------------------------------------------------------------------------------- /flink-cdc/src/main/resources/hive.dml.template: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MyLanPangzi/flink-demo/369eba5cc59e02f723ee6812402104525718ab3f/flink-cdc/src/main/resources/hive.dml.template -------------------------------------------------------------------------------- /flink-cdc/src/main/resources/msk2hive.ddl.template: -------------------------------------------------------------------------------- 1 | 2 | CREATE TABLE IF NOT EXISTS hive.kafka.ods_{db}_{table}_cdc_realtime 3 | ( 4 | before ROW ( 5 | {cols} 6 | ), 7 | after ROW ( 8 | {cols} 9 | ), 10 | source ROW ( 11 | ts_ms bigint, 12 | db string, 13 | `table` string 14 | ), 15 | ts_ms bigint, 16 | op string, 17 | 18 | kafka_topic string METADATA FROM 'topic', 19 | kafka_partition int METADATA FROM 'partition', 20 | kafka_offset bigint METADATA FROM 'offset', 21 | kafka_timestamp timestamp METADATA FROM 'timestamp', 22 | kafka_timestamp_type string METADATA FROM 'timestamp-type', 23 | -- @formatter:off 24 | proctime AS PROCTIME() 25 | -- @formatter:on 26 | ) WITH ( 27 | 'connector' = 'kafka', 28 | 'topic' = 'ods_{db}_{table}_cdc_realtime', 29 | 'properties.bootstrap.servers' = '', 30 | 'properties.group.id' = 'ods_{db}_{table}_cdc_realtime_2_hive', 31 | 'format' = 'json', 32 | 'scan.topic-partition-discovery.interval' = '60s', 33 | -- 'scan.startup.mode' = 'earliest-offset', 34 | 'json.ignore-parse-errors' = 'true' 35 | ); 36 | -------------------------------------------------------------------------------- /flink-cdc/src/main/resources/msk2hive.dml.template: -------------------------------------------------------------------------------- 1 | 2 | INSERT INTO hive.ods.ods_{db}_{table}_realtime 3 | /*+ OPTIONS( 4 | 'sink.partition-commit.policy.kind'='metastore,success-file' 5 | ) */ 6 | SELECT {insert_cols}, 7 | 8 | source.db AS metadata_schema_name, 9 | source.`table` AS metadata_table_name, 10 | mill_2_local_date_time_string(ts_ms) AS metadata_timestamp, 11 | op AS metadata_operation, 12 | mill_2_local_date_time_string(source.ts_ms) AS metadata_operation_ts, 13 | kafka_topic, 14 | kafka_partition, 15 | kafka_offset, 16 | kafka_timestamp, 17 | kafka_timestamp_type, 18 | DATE_FORMAT(kafka_timestamp, 'yyyyMMdd') AS dt 19 | FROM hive.kafka.ods_{db}_{table}_cdc_realtime 20 | /*+ OPTIONS( 21 | 'properties.group.id' = 'ods_{db}_{table}_cdc_realtime_2_hive' 22 | ) */ 23 | ; 24 | -------------------------------------------------------------------------------- /flink-cdc/src/main/resources/msk2mysql.ddl.template: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS hive.kafka.ods_{db}_{table}_cdc_realtime_debezium_json 2 | ( 3 | 4 | `metadata_schema_name` STRING METADATA FROM 'value.source.database', 5 | `metadata_table_name` STRING METADATA FROM 'value.source.table', 6 | `metadata_timestamp` TIMESTAMP_LTZ(3) METADATA FROM 'value.ingestion-timestamp', 7 | `kafka_topic` STRING METADATA FROM 'topic', 8 | `kafka_partition` INT METADATA FROM 'partition', 9 | `kafka_offset` BIGINT METADATA FROM 'offset', 10 | `kafka_timestamp` TIMESTAMP(3) METADATA FROM 'timestamp', 11 | `kafka_timestamp_type` STRING METADATA FROM 'timestamp-type' 12 | )WITH ( 13 | 'connector' = 'kafka', 14 | 'topic' = 'ods_{db}_{table}_cdc_realtime', 15 | 'properties.bootstrap.servers' = '', 16 | 'value.format' = 'debezium-json', 17 | 'value.debezium-json.ignore-parse-errors' = 'true', 18 | 'value.debezium-json.timestamp-format.standard' = 'ISO-8601', 19 | 'scan.topic-partition-discovery.interval' = '60s' 20 | ) LIKE `{db}`.`{db}`.`{table}` (EXCLUDING ALL); 21 | 22 | -------------------------------------------------------------------------------- /flink-cdc/src/main/resources/msk2mysql.dml.template: -------------------------------------------------------------------------------- 1 | 2 | INSERT INTO mysql.{db}.{table} 3 | /*+ OPTIONS( 4 | 'sink.buffer-flush.max-rows' = '1000', 5 | 'sink.buffer-flush.interval' = '3s' 6 | ) */ 7 | SELECT * 8 | FROM hive.kafka.ods_{db}_{table}_cdc_realtime_debezium_json 9 | /*+ OPTIONS( 10 | 'properties.group.id' = 'ods_{db}_{table}_cdc_realtime_2_mysql' 11 | ) */ 12 | ; 13 | 14 | -------------------------------------------------------------------------------- /flink-cdc/src/test/resources/cdc.sql: -------------------------------------------------------------------------------- 1 | SET hostname=localhost; 2 | SET port=3306; 3 | SET username=root; 4 | SET password=!QAZ2wsx; 5 | SET database-name=cdc; 6 | SET server-id=10000-10100; 7 | SET scan.incremental.snapshot.chunk.size=80960; 8 | SET split-key.even-distribution.factor.upper-bound=2.0; 9 | SET scan.newly-added-table.enabled=true; 10 | SET debezium.bigint.unsigned.handling.mode=long; 11 | SET execution.checkpointing.interval=3s; 12 | 13 | SET table-topic-mapping.cdc.t_withdraws_new_user_active_v3=t_withdraws_new_user_active_v3; 14 | SET table-topic-mapping.cdc.test=test; 15 | 16 | call com.hiscat.flink.cdc.function.MysqlCdcSourceRegister; 17 | 18 | CREATE TABLE ods_binlog_default 19 | ( 20 | `key` STRING, 21 | `value` STRING, 22 | `topic` STRING METADATA FROM 'topic' 23 | ) WITH ( 24 | 'connector' = 'route-kafka', 25 | 'topic' = 'ods_binlog_default', 26 | 'properties.bootstrap.servers' = 'localhost:9092', 27 | 'key.format' = 'raw', 28 | 'key.fields' = 'key', 29 | 'value.format' = 'raw', 30 | 'value.fields-include' = 'EXCEPT_KEY', 31 | 'properties.compression.type' = 'gzip', 32 | 'properties.linger.ms' = '1000' 33 | ); 34 | 35 | INSERT INTO ods_binlog_default 36 | SELECT `key`, `value`, get_topic(`db`, `table`) 37 | FROM cdc; 38 | -------------------------------------------------------------------------------- /flink-connector-hive_1.13/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.hiscat.flink 8 | flink-connector-hive_1.13 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 8 13 | 8 14 | UTF-8 15 | 1.13.1 16 | 3.3.0 17 | 2.3.9 18 | 19 | 20 | 21 | org.apache.flink 22 | flink-table-common 23 | ${flink.version} 24 | provided 25 | 26 | 27 | org.apache.flink 28 | flink-table-api-java 29 | ${flink.version} 30 | provided 31 | 32 | 33 | org.apache.flink 34 | flink-table-planner-blink_2.11 35 | ${flink.version} 36 | provided 37 | 38 | 39 | org.apache.flink 40 | flink-connector-hive_2.11 41 | ${flink.version} 42 | 43 | 44 | org.apache.flink 45 | flink-connector-files 46 | 47 | 48 | org.apache.flink 49 | flink-shaded-force-shading 50 | 51 | 52 | 53 | 54 | com.amazonaws 55 | aws-java-sdk-glue 56 | 1.12.22 57 | provided 58 | 59 | 60 | aws 61 | aws-hive 62 | 2.3.9 63 | system 64 | /Users/bo.xie/Downloads/aws-glue-datacatalog-hive2-client.jar 65 | 66 | 67 | aws 68 | hive 69 | 2.3.9 70 | system 71 | /Users/bo.xie/Downloads/hive-exec-2.3.9-amzn-0.jar 72 | 73 | 74 | org.apache.hadoop 75 | hadoop-client 76 | 2.10.1 77 | provided 78 | 79 | 80 | org.projectlombok 81 | lombok 82 | 1.18.22 83 | provided 84 | true 85 | 86 | 87 | 88 | 89 | 90 | org.apache.maven.plugins 91 | maven-shade-plugin 92 | ${maven-shade-plugin.version} 93 | 94 | 95 | 97 | 98 | false 99 | 100 | 101 | 102 | package 103 | 104 | shade 105 | 106 | 107 | 108 | 109 | 110 | 111 | -------------------------------------------------------------------------------- /flink-connector-hive_1.13/src/main/java/org/apache/flink/table/catalog/hive/client/HiveShimV239.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.table.catalog.hive.client; 2 | 3 | public class HiveShimV239 extends HiveShimV236 { 4 | } 5 | -------------------------------------------------------------------------------- /flink-connector-hive_1.13/src/main/java/org/apache/flink/table/catalog/hive/client/HiveShimV239Amzn0.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.table.catalog.hive.client; 2 | 3 | import com.amazonaws.glue.catalog.metastore.AWSCatalogMetastoreClient; 4 | import org.apache.hadoop.hive.conf.HiveConf; 5 | import org.apache.hadoop.hive.metastore.HiveMetaHookLoader; 6 | import org.apache.hadoop.hive.metastore.IMetaStoreClient; 7 | import org.apache.hadoop.hive.metastore.api.MetaException; 8 | import org.apache.hadoop.hive.ql.metadata.HiveException; 9 | import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; 10 | import org.apache.hadoop.hive.ql.metadata.HiveUtils; 11 | import org.apache.hadoop.util.StringUtils; 12 | 13 | public class HiveShimV239Amzn0 extends HiveShimV236 { 14 | 15 | @lombok.SneakyThrows 16 | @Override 17 | public IMetaStoreClient getHiveMetastoreClient(HiveConf conf) { 18 | HiveMetaHookLoader hookLoader = tbl -> { 19 | try { 20 | if (tbl == null) { 21 | return null; 22 | } else { 23 | HiveStorageHandler storageHandler = HiveUtils.getStorageHandler(conf, 24 | tbl.getParameters().get("storage_handler")); 25 | return storageHandler == null ? null : storageHandler.getMetaHook(); 26 | } 27 | } catch (HiveException e) { 28 | HiveUtils.LOG.error(StringUtils.stringifyException(e)); 29 | throw new MetaException("Failed to load storage handler: " + e.getMessage()); 30 | } 31 | }; 32 | 33 | return new AWSCatalogMetastoreClient(conf, hookLoader); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /flink-connector-hive_1.14/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.hiscat.flink 8 | flink-connector-hive_1.14 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 8 13 | 8 14 | UTF-8 15 | 1.14.2 16 | 3.3.0 17 | 2.3.9 18 | 1.12.22 19 | 2.10.1 20 | 1.18.22 21 | 2.3.9 22 | 23 | 24 | 25 | org.apache.flink 26 | flink-table-common 27 | ${flink.version} 28 | provided 29 | 30 | 31 | org.apache.flink 32 | flink-table-api-java 33 | ${flink.version} 34 | provided 35 | 36 | 37 | org.apache.flink 38 | flink-connector-hive_2.11 39 | ${flink.version} 40 | 41 | 42 | org.apache.flink 43 | flink-connector-files 44 | 45 | 46 | org.apache.flink 47 | flink-shaded-force-shading 48 | 49 | 50 | 51 | 52 | com.amazonaws 53 | aws-java-sdk-glue 54 | ${aws-java-sdk-glue.version} 55 | provided 56 | 57 | 58 | aws 59 | aws-hive 60 | ${glue-hive.version} 61 | system 62 | /Users/bo.xie/Downloads/aws-glue-datacatalog-hive2-client.jar 63 | 64 | 65 | aws 66 | hive 67 | ${glue-hive.version} 68 | system 69 | /Users/bo.xie/Downloads/hive-exec-2.3.9-amzn-0.jar 70 | 71 | 72 | org.apache.hadoop 73 | hadoop-client 74 | ${hadoop-client.version} 75 | provided 76 | 77 | 78 | org.projectlombok 79 | lombok 80 | ${lombok.version} 81 | provided 82 | true 83 | 84 | 85 | 86 | 87 | 88 | org.apache.maven.plugins 89 | maven-shade-plugin 90 | ${maven-shade-plugin.version} 91 | 92 | 93 | 95 | 96 | false 97 | 98 | 99 | 100 | package 101 | 102 | shade 103 | 104 | 105 | 106 | 107 | 108 | 109 | -------------------------------------------------------------------------------- /flink-connector-hive_1.14/src/main/java/org/apache/flink/table/catalog/hive/client/HiveShimV239.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.table.catalog.hive.client; 2 | 3 | import com.amazonaws.ClientConfiguration; 4 | import com.amazonaws.auth.AWSCredentialsProvider; 5 | import com.amazonaws.client.builder.AwsClientBuilder; 6 | import com.amazonaws.glue.catalog.metastore.AWSCatalogMetastoreClient; 7 | import com.amazonaws.glue.catalog.metastore.AWSCredentialsProviderFactory; 8 | import com.amazonaws.glue.catalog.metastore.DefaultAWSCredentialsProviderFactory; 9 | import com.amazonaws.glue.catalog.metastore.GlueClientFactory; 10 | import com.amazonaws.glue.catalog.util.MetastoreClientUtils; 11 | import com.amazonaws.services.glue.AWSGlueClientBuilder; 12 | import com.google.common.base.Strings; 13 | import org.apache.hadoop.hive.conf.HiveConf; 14 | import org.apache.hadoop.hive.metastore.IMetaStoreClient; 15 | import org.apache.hadoop.hive.metastore.Warehouse; 16 | import org.apache.hadoop.security.UserGroupInformation; 17 | import org.apache.hadoop.util.ReflectionUtils; 18 | 19 | import java.io.IOException; 20 | 21 | public class HiveShimV239 extends HiveShimV237 { 22 | 23 | } 24 | -------------------------------------------------------------------------------- /flink-connector-hive_1.14/src/main/java/org/apache/flink/table/catalog/hive/client/HiveShimV239Amzn0.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.table.catalog.hive.client; 2 | 3 | import com.amazonaws.ClientConfiguration; 4 | import com.amazonaws.auth.AWSCredentialsProvider; 5 | import com.amazonaws.client.builder.AwsClientBuilder; 6 | import com.amazonaws.glue.catalog.metastore.AWSCatalogMetastoreClient; 7 | import com.amazonaws.glue.catalog.metastore.AWSCredentialsProviderFactory; 8 | import com.amazonaws.glue.catalog.metastore.DefaultAWSCredentialsProviderFactory; 9 | import com.amazonaws.glue.catalog.metastore.GlueClientFactory; 10 | import com.amazonaws.glue.catalog.util.MetastoreClientUtils; 11 | import com.amazonaws.services.glue.AWSGlueClientBuilder; 12 | import com.google.common.base.Strings; 13 | import org.apache.hadoop.hive.conf.HiveConf; 14 | import org.apache.hadoop.hive.metastore.IMetaStoreClient; 15 | import org.apache.hadoop.hive.metastore.Warehouse; 16 | import org.apache.hadoop.security.UserGroupInformation; 17 | import org.apache.hadoop.util.ReflectionUtils; 18 | 19 | import java.io.IOException; 20 | 21 | public class HiveShimV239Amzn0 extends HiveShimV237 { 22 | 23 | @lombok.SneakyThrows 24 | @Override 25 | public IMetaStoreClient getHiveMetastoreClient(HiveConf conf) { 26 | return new AWSCatalogMetastoreClient.Builder() 27 | .withHiveConf(conf) 28 | .withCatalogId(MetastoreClientUtils.getCatalogId(conf)) 29 | .withWarehouse(new Warehouse(conf)) 30 | .withClientFactory(getGlueClientFactory(conf)) 31 | .build(); 32 | 33 | } 34 | 35 | private static GlueClientFactory getGlueClientFactory(HiveConf conf) { 36 | return () -> AWSGlueClientBuilder.standard() 37 | .withCredentials(getAwsCredentialsProvider(conf)) 38 | .withEndpointConfiguration(getEndpointConfiguration(conf)) 39 | .withClientConfiguration(buildClientConfiguration(conf)).build(); 40 | } 41 | 42 | private static AWSCredentialsProvider getAwsCredentialsProvider(HiveConf conf) { 43 | Class providerFactoryClass = conf 44 | .getClass("aws.catalog.credentials.provider.factory.class", 45 | DefaultAWSCredentialsProviderFactory.class) 46 | .asSubclass(AWSCredentialsProviderFactory.class); 47 | AWSCredentialsProviderFactory provider = ReflectionUtils.newInstance(providerFactoryClass, conf); 48 | return provider.buildAWSCredentialsProvider(conf); 49 | } 50 | 51 | private static AwsClientBuilder.EndpointConfiguration getEndpointConfiguration(HiveConf conf) { 52 | return new AwsClientBuilder.EndpointConfiguration(getEndpoint(conf), getRegion(conf)); 53 | } 54 | 55 | private static String getEndpoint(HiveConf conf) { 56 | return getProperty("aws.glue.endpoint", conf); 57 | } 58 | 59 | private static String getRegion(HiveConf conf) { 60 | return getProperty("aws.region", conf); 61 | } 62 | 63 | private static String getProperty(String propertyName, HiveConf conf) { 64 | return Strings.isNullOrEmpty(System.getProperty(propertyName)) ? conf.get(propertyName) : System.getProperty(propertyName); 65 | } 66 | 67 | private static ClientConfiguration buildClientConfiguration(HiveConf hiveConf) { 68 | return new ClientConfiguration() 69 | .withUserAgent(createUserAgent()) 70 | .withMaxErrorRetry(hiveConf.getInt("aws.glue.max-error-retries", 5)) 71 | .withMaxConnections(hiveConf.getInt("aws.glue.max-connections", 50)) 72 | .withConnectionTimeout(hiveConf.getInt("aws.glue.connection-timeout", 10000)) 73 | .withSocketTimeout(hiveConf.getInt("aws.glue.socket-timeout", 50000)); 74 | } 75 | 76 | private static String createUserAgent() { 77 | try { 78 | String ugi = UserGroupInformation.getCurrentUser().getUserName(); 79 | return "ugi=" + ugi; 80 | } catch (IOException e) { 81 | throw new RuntimeException(e); 82 | } 83 | } 84 | 85 | } 86 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.13/src/main/java/com/hiscat/flink/connector/jdbc/table/FieldStringGetter.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.connector.jdbc.table; 2 | 3 | import java.time.format.DateTimeFormatter; 4 | import javax.annotation.Nullable; 5 | import org.apache.flink.table.data.RowData; 6 | import org.apache.flink.table.data.RowData.FieldGetter; 7 | import org.apache.flink.table.data.TimestampData; 8 | import org.apache.flink.table.types.logical.LogicalType; 9 | 10 | public final class FieldStringGetter implements FieldGetter { 11 | private static final long serialVersionUID = 1L; 12 | 13 | private final LogicalType logicalType; 14 | private final FieldGetter fieldGetter; 15 | 16 | public FieldStringGetter(LogicalType logicalType, FieldGetter fieldGetter) { 17 | this.logicalType = logicalType; 18 | this.fieldGetter = fieldGetter; 19 | } 20 | 21 | @Nullable 22 | @Override 23 | public Object getFieldOrNull(RowData row) { 24 | Object data = fieldGetter.getFieldOrNull(row); 25 | if (data == null) { 26 | return "null"; 27 | } 28 | switch (this.logicalType.getTypeRoot()) { 29 | case CHAR: 30 | case VARCHAR: 31 | return String.format("'%s'", escape(data.toString())); 32 | 33 | case BINARY: 34 | case VARBINARY: 35 | return String.format("'%s'", escape(new String((byte[]) data))); 36 | 37 | case TIMESTAMP_WITHOUT_TIME_ZONE: 38 | case TIMESTAMP_WITH_LOCAL_TIME_ZONE: 39 | TimestampData timestampData = (TimestampData) data; 40 | return String.format( 41 | "'%s'", 42 | timestampData 43 | .toLocalDateTime() 44 | .format(DateTimeFormatter.ofPattern("yyyyMMdd HH:mm:ss.SSS"))); 45 | 46 | case DATE: 47 | case TIMESTAMP_WITH_TIME_ZONE: 48 | case TIME_WITHOUT_TIME_ZONE: 49 | case INTERVAL_YEAR_MONTH: 50 | case ARRAY: 51 | case MULTISET: 52 | case MAP: 53 | case ROW: 54 | case STRUCTURED_TYPE: 55 | case DISTINCT_TYPE: 56 | case RAW: 57 | case NULL: 58 | case SYMBOL: 59 | case UNRESOLVED: 60 | case BOOLEAN: 61 | case DECIMAL: 62 | case FLOAT: 63 | case DOUBLE: 64 | case BIGINT: 65 | case INTEGER: 66 | case SMALLINT: 67 | case TINYINT: 68 | case INTERVAL_DAY_TIME: 69 | return String.valueOf(data); 70 | default: 71 | throw new IllegalArgumentException(); 72 | } 73 | } 74 | 75 | private String escape(String result) { 76 | return result.replace("'", "").replace("\\", ""); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.13/src/main/java/com/hiscat/flink/connector/jdbc/table/PartialUpdateOptions.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.connector.jdbc.table; 2 | 3 | import java.io.Serializable; 4 | import java.sql.SQLException; 5 | import java.util.List; 6 | import lombok.AllArgsConstructor; 7 | import lombok.Builder; 8 | import lombok.Data; 9 | import lombok.NoArgsConstructor; 10 | import org.apache.flink.connector.jdbc.internal.converter.JdbcRowConverter; 11 | import org.apache.flink.connector.jdbc.statement.FieldNamedPreparedStatement; 12 | import org.apache.flink.table.data.RowData; 13 | 14 | @Data 15 | @AllArgsConstructor 16 | @NoArgsConstructor 17 | @Builder 18 | public class PartialUpdateOptions implements Serializable { 19 | private static final long serialVersionUID = 1L; 20 | 21 | private String tag; 22 | private List fields; 23 | private JdbcRowConverter converter; 24 | private transient List buffer; 25 | private transient FieldNamedPreparedStatement preparedStatement; 26 | 27 | public void close() { 28 | try { 29 | preparedStatement.close(); 30 | } catch (SQLException e) { 31 | throw new RuntimeException(e); 32 | } 33 | } 34 | 35 | public void flush() { 36 | buffer.forEach( 37 | e -> { 38 | try { 39 | converter.toExternal(e, preparedStatement); 40 | preparedStatement.addBatch(); 41 | } catch (SQLException ex) { 42 | throw new RuntimeException(ex); 43 | } 44 | }); 45 | try { 46 | preparedStatement.executeBatch(); 47 | } catch (SQLException e) { 48 | throw new RuntimeException(e); 49 | } 50 | buffer.clear(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.13/src/main/java/com/hiscat/flink/connector/jdbc/table/PrepareStatementTagExtractor.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.connector.jdbc.table; 2 | 3 | import java.io.Serializable; 4 | import java.util.function.Function; 5 | import org.apache.flink.table.data.RowData; 6 | 7 | public class PrepareStatementTagExtractor implements Function, Serializable { 8 | 9 | private final int pos; 10 | 11 | public PrepareStatementTagExtractor(int pos) { 12 | this.pos = pos; 13 | } 14 | 15 | @Override 16 | public String apply(RowData rowData) { 17 | return PartialUpdateJdbcDynamicTableSink.PrepareStatementTagMetadata.PREPARE_STATEMENT_TAG 18 | .converter 19 | .read(rowData, pos) 20 | .toString(); 21 | } 22 | 23 | public int getPos() { 24 | return pos; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.13/src/main/java/com/hiscat/flink/connector/jdbc/table/RedshiftDynamicTableSink.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.connector.jdbc.table; 2 | 3 | import com.hiscat.flink.connector.jdbc.table.executor.InsertOnlyBufferedStringStatementExecutor; 4 | import java.util.Arrays; 5 | import java.util.List; 6 | import java.util.Objects; 7 | import java.util.stream.Collectors; 8 | import org.apache.flink.connector.jdbc.JdbcExecutionOptions; 9 | import org.apache.flink.connector.jdbc.dialect.JdbcDialect; 10 | import org.apache.flink.connector.jdbc.internal.GenericJdbcSinkFunction; 11 | import org.apache.flink.connector.jdbc.internal.JdbcBatchingOutputFormat; 12 | import org.apache.flink.connector.jdbc.internal.connection.SimpleJdbcConnectionProvider; 13 | import org.apache.flink.connector.jdbc.internal.options.JdbcDmlOptions; 14 | import org.apache.flink.connector.jdbc.internal.options.JdbcOptions; 15 | import org.apache.flink.table.catalog.ResolvedSchema; 16 | import org.apache.flink.table.connector.ChangelogMode; 17 | import org.apache.flink.table.connector.sink.DynamicTableSink; 18 | import org.apache.flink.table.connector.sink.SinkFunctionProvider; 19 | import org.apache.flink.table.data.RowData; 20 | import org.apache.flink.table.types.DataType; 21 | import org.apache.flink.table.types.logical.LogicalType; 22 | 23 | public class RedshiftDynamicTableSink implements DynamicTableSink { 24 | private final JdbcOptions jdbcOptions; 25 | private final JdbcExecutionOptions executionOptions; 26 | private final JdbcDmlOptions dmlOptions; 27 | private final ResolvedSchema tableSchema; 28 | 29 | public RedshiftDynamicTableSink( 30 | JdbcOptions jdbcOptions, 31 | JdbcExecutionOptions executionOptions, 32 | JdbcDmlOptions dmlOptions, 33 | ResolvedSchema tableSchema) { 34 | this.jdbcOptions = jdbcOptions; 35 | this.executionOptions = executionOptions; 36 | this.dmlOptions = dmlOptions; 37 | this.tableSchema = tableSchema; 38 | } 39 | 40 | @Override 41 | public ChangelogMode getChangelogMode(ChangelogMode requestedMode) { 42 | return ChangelogMode.upsert(); 43 | } 44 | 45 | @Override 46 | public SinkRuntimeProvider getSinkRuntimeProvider(Context context) { 47 | 48 | List fieldStringGetters = getFieldStringGetters(); 49 | String sql = getInsertIntoStatement(); 50 | return SinkFunctionProvider.of( 51 | new GenericJdbcSinkFunction<>( 52 | new JdbcBatchingOutputFormat<>( 53 | new SimpleJdbcConnectionProvider(jdbcOptions), 54 | this.executionOptions, 55 | ctx -> new InsertOnlyBufferedStringStatementExecutor(sql), 56 | rowData -> 57 | fieldStringGetters.stream() 58 | .map(f -> Objects.requireNonNull(f.getFieldOrNull(rowData)).toString()) 59 | .collect(Collectors.joining(",", "(", ")")))), 60 | jdbcOptions.getParallelism()); 61 | } 62 | 63 | private String getInsertIntoStatement() { 64 | JdbcDialect dialect = dmlOptions.getDialect(); 65 | String escapedCols = 66 | Arrays.stream(dmlOptions.getFieldNames()) 67 | .map(dialect::quoteIdentifier) 68 | .collect(Collectors.joining(",")); 69 | return String.format( 70 | "insert into %s (%s) values", 71 | dmlOptions.getTableName(), escapedCols); 72 | } 73 | 74 | private List getFieldStringGetters() { 75 | List columnDataTypes = this.tableSchema.getColumnDataTypes(); 76 | List columnNames = this.tableSchema.getColumnNames(); 77 | return columnNames.stream() 78 | .map( 79 | col -> { 80 | int filedPos = columnNames.indexOf(col); 81 | LogicalType logicalType = columnDataTypes.get(filedPos).getLogicalType(); 82 | return new FieldStringGetter( 83 | logicalType, RowData.createFieldGetter(logicalType, filedPos)); 84 | }) 85 | .collect(Collectors.toList()); 86 | } 87 | 88 | @Override 89 | public DynamicTableSink copy() { 90 | return new RedshiftDynamicTableSink(jdbcOptions, executionOptions, dmlOptions, tableSchema); 91 | } 92 | 93 | @Override 94 | public String asSummaryString() { 95 | return "redshift dynamic table sink"; 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.13/src/main/java/com/hiscat/flink/connector/jdbc/table/executor/InsertOnlyBufferedStringStatementExecutor.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.connector.jdbc.table.executor; 2 | 3 | import java.sql.Connection; 4 | import java.sql.SQLException; 5 | import java.sql.Statement; 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | import lombok.extern.slf4j.Slf4j; 9 | import org.apache.flink.connector.jdbc.internal.executor.JdbcBatchStatementExecutor; 10 | 11 | @Slf4j 12 | public class InsertOnlyBufferedStringStatementExecutor 13 | implements JdbcBatchStatementExecutor { 14 | 15 | private final String sql; 16 | private transient Statement statement; 17 | private final List buffer = new ArrayList<>(); 18 | 19 | public InsertOnlyBufferedStringStatementExecutor(String sql) { 20 | this.sql = sql; 21 | } 22 | 23 | @Override 24 | public void prepareStatements(Connection connection) throws SQLException { 25 | statement = connection.createStatement(); 26 | } 27 | 28 | @Override 29 | public void addToBatch(String record) { 30 | buffer.add(record); 31 | } 32 | 33 | @Override 34 | public void executeBatch() throws SQLException { 35 | if (buffer.isEmpty()) { 36 | return; 37 | } 38 | String sql = this.sql + String.join(",", buffer); 39 | this.statement.execute(sql); 40 | buffer.clear(); 41 | } 42 | 43 | @Override 44 | public void closeStatements() throws SQLException { 45 | statement.close(); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.13/src/main/java/com/hiscat/flink/connector/jdbc/table/executor/PartialUpdateJdbcBatchStatementExecutor.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.connector.jdbc.table.executor; 2 | 3 | import static java.util.stream.Collectors.toMap; 4 | 5 | import com.hiscat.flink.connector.jdbc.table.PartialUpdateOptions; 6 | 7 | import java.sql.Connection; 8 | import java.sql.SQLException; 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | import java.util.Map; 12 | import java.util.NoSuchElementException; 13 | import java.util.function.Function; 14 | import org.apache.flink.connector.jdbc.internal.executor.JdbcBatchStatementExecutor; 15 | import org.apache.flink.connector.jdbc.internal.options.JdbcDmlOptions; 16 | import org.apache.flink.connector.jdbc.statement.FieldNamedPreparedStatement; 17 | import org.apache.flink.table.data.RowData; 18 | 19 | public class PartialUpdateJdbcBatchStatementExecutor 20 | implements JdbcBatchStatementExecutor { 21 | 22 | private final Map options; 23 | 24 | private final Function tagExtractor; 25 | 26 | private final JdbcDmlOptions dmlOptions; 27 | 28 | public PartialUpdateJdbcBatchStatementExecutor( 29 | JdbcDmlOptions dmlOptions, 30 | List partialUpdateOptions, 31 | Function tagExtractor) { 32 | this.tagExtractor = tagExtractor; 33 | this.dmlOptions = dmlOptions; 34 | this.options = 35 | partialUpdateOptions.stream().collect(toMap(PartialUpdateOptions::getTag, o -> o)); 36 | } 37 | 38 | @Override 39 | public void prepareStatements(Connection connection) { 40 | this.options 41 | .values() 42 | .forEach( 43 | o -> { 44 | String[] fields = o.getFields().toArray(new String[0]); 45 | String upsertSql = 46 | dmlOptions 47 | .getDialect() 48 | .getUpsertStatement( 49 | dmlOptions.getTableName(), 50 | fields, 51 | dmlOptions 52 | .getKeyFields() 53 | .orElseThrow(() -> new NoSuchElementException("pk not exists"))) 54 | .orElseThrow(() -> new NoSuchElementException("upsert sql not provided")); 55 | try { 56 | FieldNamedPreparedStatement pst = 57 | FieldNamedPreparedStatement.prepareStatement(connection, upsertSql, fields); 58 | o.setPreparedStatement(pst); 59 | o.setBuffer(new ArrayList<>()); 60 | } catch (SQLException e) { 61 | throw new RuntimeException(e); 62 | } 63 | }); 64 | } 65 | 66 | @Override 67 | public void addToBatch(RowData record) { 68 | options.get(tagExtractor.apply(record)).getBuffer().add(record); 69 | } 70 | 71 | @Override 72 | public void executeBatch() { 73 | options.values().forEach(PartialUpdateOptions::flush); 74 | } 75 | 76 | @Override 77 | public void closeStatements() { 78 | options.values().forEach(PartialUpdateOptions::close); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.13/src/main/java/com/hiscat/flink/connector/jdbc/table/executor/PartialUpdateTableBufferReducedStatementExecutorFactory.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.connector.jdbc.table.executor; 2 | 3 | import static java.util.stream.Collectors.toList; 4 | import static org.apache.flink.table.data.RowData.createFieldGetter; 5 | 6 | import com.hiscat.flink.connector.jdbc.table.PartialUpdateOptions; 7 | import com.hiscat.flink.connector.jdbc.table.PrepareStatementTagExtractor; 8 | 9 | import java.util.Arrays; 10 | import java.util.List; 11 | import java.util.function.Function; 12 | import java.util.stream.IntStream; 13 | import org.apache.flink.api.common.functions.RuntimeContext; 14 | import org.apache.flink.api.common.typeinfo.TypeInformation; 15 | import org.apache.flink.api.common.typeutils.TypeSerializer; 16 | import org.apache.flink.connector.jdbc.internal.JdbcBatchingOutputFormat.StatementExecutorFactory; 17 | import org.apache.flink.connector.jdbc.internal.executor.TableBufferReducedStatementExecutor; 18 | import org.apache.flink.connector.jdbc.internal.options.JdbcDmlOptions; 19 | import org.apache.flink.table.data.GenericRowData; 20 | import org.apache.flink.table.data.RowData; 21 | import org.apache.flink.table.data.RowData.FieldGetter; 22 | import org.apache.flink.table.data.StringData; 23 | import org.apache.flink.table.types.DataType; 24 | import org.apache.flink.table.types.logical.LogicalType; 25 | 26 | public class PartialUpdateTableBufferReducedStatementExecutorFactory 27 | implements StatementExecutorFactory { 28 | 29 | private final TypeInformation rowDataTypeInformation; 30 | private final List columnDataTypes; 31 | private final PrepareStatementTagExtractor tagExtractor; 32 | private final JdbcDmlOptions dmlOptions; 33 | private final List partialUpdateOptions; 34 | 35 | public PartialUpdateTableBufferReducedStatementExecutorFactory( 36 | TypeInformation rowDataTypeInformation, 37 | List columnDataTypes, 38 | PrepareStatementTagExtractor tagExtractor, 39 | JdbcDmlOptions dmlOptions, 40 | List partialUpdateOptions) { 41 | this.rowDataTypeInformation = rowDataTypeInformation; 42 | this.columnDataTypes = columnDataTypes; 43 | this.tagExtractor = tagExtractor; 44 | this.dmlOptions = dmlOptions; 45 | this.partialUpdateOptions = partialUpdateOptions; 46 | } 47 | 48 | @Override 49 | public TableBufferReducedStatementExecutor apply(RuntimeContext ctx) { 50 | 51 | final Function valueTransform = 52 | getValueTransform( 53 | ctx, this.rowDataTypeInformation.createSerializer(ctx.getExecutionConfig())); 54 | List fields = 55 | IntStream.range(0, dmlOptions.getFieldNames().length).boxed().collect(toList()); 56 | int[] pkFields = 57 | Arrays.stream( 58 | dmlOptions.getKeyFields().orElseThrow(() -> new RuntimeException("pk absent!"))) 59 | .mapToInt(Arrays.asList(dmlOptions.getFieldNames())::indexOf) 60 | .toArray(); 61 | LogicalType[] pkTypes = 62 | Arrays.stream(pkFields) 63 | .mapToObj(this.columnDataTypes::get) 64 | .map(DataType::getLogicalType) 65 | .toArray(LogicalType[]::new); 66 | 67 | return new TableBufferReducedStatementExecutor( 68 | new PartialUpdateJdbcBatchStatementExecutor(dmlOptions, partialUpdateOptions, tagExtractor), 69 | new PartialUpdateJdbcBatchStatementExecutor(dmlOptions, partialUpdateOptions, tagExtractor), 70 | createRowKeyExtractor(fields, pkTypes, pkFields, tagExtractor), 71 | valueTransform); 72 | } 73 | 74 | private Function getValueTransform( 75 | RuntimeContext ctx, TypeSerializer typeSerializer) { 76 | return ctx.getExecutionConfig().isObjectReuseEnabled() 77 | ? typeSerializer::copy 78 | : Function.identity(); 79 | } 80 | 81 | private static Function createRowKeyExtractor( 82 | List fields, 83 | LogicalType[] logicalTypes, 84 | int[] pkFields, 85 | PrepareStatementTagExtractor tagExtractor) { 86 | final RowData.FieldGetter[] fieldGetters = new RowData.FieldGetter[fields.size()]; 87 | for (int i = 0; i < pkFields.length; i++) { 88 | fieldGetters[i] = createFieldGetter(logicalTypes[pkFields[i]], pkFields[i]); 89 | } 90 | List pkList = Arrays.stream(pkFields).boxed().collect(toList()); 91 | return row -> getPrimaryKey(fields, pkList, row, fieldGetters, tagExtractor); 92 | } 93 | 94 | private static RowData getPrimaryKey( 95 | List fields, 96 | List pkList, 97 | RowData row, 98 | FieldGetter[] fieldGetters, 99 | PrepareStatementTagExtractor tagExtractor) { 100 | GenericRowData pkRow = new GenericRowData(fields.size()); 101 | for (int i = 0; i < fields.size(); i++) { 102 | if (pkList.contains(i)) { 103 | pkRow.setField(i, fieldGetters[i].getFieldOrNull(row)); 104 | } else { 105 | pkRow.setField(i, null); 106 | } 107 | } 108 | pkRow.setField(tagExtractor.getPos(), StringData.fromString(tagExtractor.apply(row))); 109 | return pkRow; 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.13/src/main/java/org/apache/flink/connector/jdbc/catalog/JdbcCatalogUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.jdbc.catalog; 20 | 21 | import org.apache.flink.connector.jdbc.dialect.JdbcDialect; 22 | import org.apache.flink.connector.jdbc.dialect.JdbcDialects; 23 | import org.apache.flink.connector.jdbc.dialect.MySQLDialect; 24 | import org.apache.flink.connector.jdbc.dialect.PostgresDialect; 25 | 26 | import static org.apache.flink.util.Preconditions.checkArgument; 27 | 28 | /** Utils for {@link JdbcCatalog}. */ 29 | public class JdbcCatalogUtils { 30 | /** 31 | * URL has to be without database, like "jdbc:postgresql://localhost:5432/" or 32 | * "jdbc:postgresql://localhost:5432" rather than "jdbc:postgresql://localhost:5432/db". 33 | */ 34 | public static void validateJdbcUrl(String url) { 35 | String[] parts = url.trim().split("/+"); 36 | 37 | checkArgument(parts.length == 2); 38 | } 39 | 40 | /** Create catalog instance from given information. */ 41 | public static AbstractJdbcCatalog createCatalog( 42 | String catalogName, 43 | String defaultDatabase, 44 | String username, 45 | String pwd, 46 | String baseUrl) { 47 | JdbcDialect dialect = JdbcDialects.get(baseUrl) 48 | .orElseThrow(()->new RuntimeException("no such dialect find " + baseUrl)); 49 | 50 | if (dialect instanceof PostgresDialect) { 51 | return new PostgresCatalog(catalogName, defaultDatabase, username, pwd, baseUrl); 52 | } else if (dialect instanceof MySQLDialect) { 53 | return new MySqlCatalog(catalogName, defaultDatabase, username, pwd, baseUrl); 54 | } else { 55 | throw new UnsupportedOperationException( 56 | String.format("Catalog for '%s' is not supported yet.", dialect)); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.13/src/main/java/org/apache/flink/connector/jdbc/dialect/JdbcDialectTypeMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.jdbc.dialect; 20 | 21 | import org.apache.flink.table.catalog.ObjectPath; 22 | import org.apache.flink.table.types.DataType; 23 | 24 | import java.sql.ResultSetMetaData; 25 | import java.sql.SQLException; 26 | 27 | /** Separate the jdbc meta-information type to flink table type into the interface. */ 28 | public interface JdbcDialectTypeMapper { 29 | 30 | DataType mapping(ObjectPath tablePath, ResultSetMetaData metadata, int colIndex) 31 | throws SQLException; 32 | } 33 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.13/src/main/java/org/apache/flink/connector/jdbc/dialect/JdbcDialects.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.jdbc.dialect; 20 | 21 | import java.util.Arrays; 22 | import java.util.List; 23 | import java.util.Optional; 24 | 25 | /** Default JDBC dialects. */ 26 | public final class JdbcDialects { 27 | 28 | private static final List DIALECTS = 29 | Arrays.asList( 30 | new DerbyDialect(), 31 | new MySQLDialect(), 32 | new PostgresDialect(), 33 | new RedshiftDialect()); 34 | 35 | /** Fetch the JdbcDialect class corresponding to a given database url. */ 36 | public static Optional get(String url) { 37 | for (JdbcDialect dialect : DIALECTS) { 38 | if (dialect.canHandle(url)) { 39 | return Optional.of(dialect); 40 | } 41 | } 42 | return Optional.empty(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.13/src/main/java/org/apache/flink/connector/jdbc/dialect/RedshiftDialect.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.jdbc.dialect; 20 | 21 | import java.util.Arrays; 22 | import java.util.List; 23 | import java.util.Optional; 24 | import org.apache.flink.connector.jdbc.internal.converter.JdbcRowConverter; 25 | import org.apache.flink.connector.jdbc.internal.converter.RedshiftRowConverter; 26 | import org.apache.flink.table.types.logical.LogicalTypeRoot; 27 | import org.apache.flink.table.types.logical.RowType; 28 | 29 | /** JDBC dialect for Redshift. */ 30 | public class RedshiftDialect extends AbstractDialect { 31 | 32 | private static final long serialVersionUID = 1L; 33 | 34 | // Define MAX/MIN precision of TIMESTAMP type according to Redshift docs: 35 | private static final int MAX_TIMESTAMP_PRECISION = 6; 36 | private static final int MIN_TIMESTAMP_PRECISION = 1; 37 | 38 | // Define MAX/MIN precision of DECIMAL type according to Redshift docs: 39 | private static final int MAX_DECIMAL_PRECISION = 38; 40 | private static final int MIN_DECIMAL_PRECISION = 1; 41 | 42 | @Override 43 | public boolean canHandle(String url) { 44 | return url.startsWith("jdbc:redshift:"); 45 | } 46 | 47 | @Override 48 | public JdbcRowConverter getRowConverter(RowType rowType) { 49 | return new RedshiftRowConverter(rowType); 50 | } 51 | 52 | @Override 53 | public String getLimitClause(long limit) { 54 | return "LIMIT " + limit; 55 | } 56 | 57 | @Override 58 | public Optional defaultDriverName() { 59 | return Optional.of("com.amazon.redshift.jdbc.Driver"); 60 | } 61 | 62 | @Override 63 | public Optional getUpsertStatement(String tableName, String[] fieldNames, 64 | String[] uniqueKeyFields) { 65 | return Optional.of(this.getInsertIntoStatement(tableName, fieldNames)); 66 | } 67 | 68 | @Override 69 | public String quoteIdentifier(String identifier) { 70 | return String.format("\"%s\"", identifier); 71 | } 72 | 73 | @Override 74 | public String dialectName() { 75 | return "Redshift"; 76 | } 77 | 78 | @Override 79 | public int maxDecimalPrecision() { 80 | return MAX_DECIMAL_PRECISION; 81 | } 82 | 83 | @Override 84 | public int minDecimalPrecision() { 85 | return MIN_DECIMAL_PRECISION; 86 | } 87 | 88 | @Override 89 | public int maxTimestampPrecision() { 90 | return MAX_TIMESTAMP_PRECISION; 91 | } 92 | 93 | @Override 94 | public int minTimestampPrecision() { 95 | return MIN_TIMESTAMP_PRECISION; 96 | } 97 | 98 | @Override 99 | public List unsupportedTypes() { 100 | // The data types used in PostgresSQL are list at: 101 | 102 | // TODO: We can't convert BINARY data type to 103 | // PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO in 104 | // LegacyTypeInfoDataTypeConverter. 105 | return Arrays.asList( 106 | LogicalTypeRoot.BINARY, 107 | LogicalTypeRoot.TIMESTAMP_WITH_TIME_ZONE, 108 | LogicalTypeRoot.INTERVAL_YEAR_MONTH, 109 | LogicalTypeRoot.INTERVAL_DAY_TIME, 110 | LogicalTypeRoot.MULTISET, 111 | LogicalTypeRoot.MAP, 112 | LogicalTypeRoot.ROW, 113 | LogicalTypeRoot.DISTINCT_TYPE, 114 | LogicalTypeRoot.STRUCTURED_TYPE, 115 | LogicalTypeRoot.NULL, 116 | LogicalTypeRoot.RAW, 117 | LogicalTypeRoot.SYMBOL, 118 | LogicalTypeRoot.UNRESOLVED); 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.13/src/main/java/org/apache/flink/connector/jdbc/internal/converter/RedshiftRowConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.jdbc.internal.converter; 20 | 21 | import org.apache.flink.table.types.logical.RowType; 22 | 23 | /** 24 | * Runtime converter that responsible to convert between JDBC object and Flink internal object for 25 | * Redshift. 26 | */ 27 | public class RedshiftRowConverter extends AbstractJdbcRowConverter { 28 | 29 | private static final long serialVersionUID = 1L; 30 | 31 | @Override 32 | public String converterName() { 33 | return "Redshift"; 34 | } 35 | 36 | public RedshiftRowConverter(RowType rowType) { 37 | super(rowType); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.13/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory: -------------------------------------------------------------------------------- 1 | com.hiscat.flink.connector.jdbc.table.RedshiftDynamicTableFactory 2 | com.hiscat.flink.connector.jdbc.table.PartialUpdateJdbcDynamicTableFactory -------------------------------------------------------------------------------- /flink-connector-jdbc_1.13/src/test/resources/cdc-2-kafka.sql: -------------------------------------------------------------------------------- 1 | SET execution.checkpointing.interval=3s; 2 | -- insert into f1 (id, f1) values (1,1); 3 | -- delete from f1 where 1; 4 | 5 | CREATE TABLE f1 6 | ( 7 | `id` bigint, 8 | `f1` bigint, 9 | 10 | db_name STRING METADATA FROM 'database_name' VIRTUAL, 11 | table_name STRING METADATA FROM 'table_name' VIRTUAL, 12 | operation_ts TIMESTAMP_LTZ(3) METADATA FROM 'op_ts' VIRTUAL, 13 | proctime AS PROCTIME(), 14 | primary key (id) NOT ENFORCED 15 | )WITH ( 16 | 'connector' = 'mysql-cdc', 17 | 'hostname' = 'localhost', 18 | 'port' = '3306', 19 | 'server-id'='5601-5700', 20 | 'username' = 'root', 21 | 'password' = '!QAZ2wsx', 22 | 'database-name' = 'cdc', 23 | 'table-name' = 'f1', 24 | 'split-key.even-distribution.factor.upper-bound' = '2.0', 25 | 'debezium.min.row.count.to.stream.result' = '10240', 26 | 'scan.snapshot.fetch.size' = '10240', 27 | 'scan.incremental.snapshot.chunk.size' = '80960', 28 | 'debezium.bigint.unsigned.handling.mode' = 'long', 29 | 'debezium.decimal.handling.mode' = 'double' 30 | ); 31 | CREATE TABLE cdc_f1 32 | ( 33 | `user_id` BIGINT PRIMARY KEY NOT ENFORCED, 34 | `f1` BIGINT 35 | ) WITH ( 36 | 'connector' = 'upsert-kafka', 37 | 'topic' = 'cdc_f1', 38 | 'properties.bootstrap.servers' = 'localhost:9092', 39 | 'value.format' = 'json', 40 | 'value.fields-include' = 'EXCEPT_KEY', 41 | 'key.format' = 'json' 42 | ); 43 | 44 | INSERT INTO cdc_f1 45 | SELECT id, COUNT(*) 46 | FROM f1 47 | GROUP BY id; -------------------------------------------------------------------------------- /flink-connector-jdbc_1.13/src/test/resources/dg-2-kafka.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE f1 2 | ( 3 | `user_id` BIGINT PRIMARY KEY NOT ENFORCED, 4 | `f1` BIGINT 5 | ) WITH ( 6 | 'connector' = 'upsert-kafka', 7 | 'topic' = 'f1', 8 | 'properties.bootstrap.servers' = 'localhost:9092', 9 | 'value.format' = 'json', 10 | 'value.fields-include' = 'EXCEPT_KEY', 11 | 'key.format' = 'json' 12 | ); 13 | CREATE TABLE f2 14 | ( 15 | `user_id` BIGINT PRIMARY KEY NOT ENFORCED, 16 | `f2` BIGINT 17 | ) WITH ( 18 | 'connector' = 'upsert-kafka', 19 | 'topic' = 'f2', 20 | 'properties.bootstrap.servers' = 'localhost:9092', 21 | 'value.format' = 'json', 22 | 'value.fields-include' = 'EXCEPT_KEY', 23 | 'key.format' = 'json' 24 | ); 25 | 26 | CREATE TABLE dg 27 | ( 28 | f BIGINT 29 | ) WITH ( 30 | 'connector' = 'datagen', 31 | 'number-of-rows' = '10000' 32 | ); 33 | 34 | BEGIN STATEMENT SET ; 35 | 36 | INSERT INTO f1 37 | SELECT 1,COUNT(*) FROM dg 38 | GROUP BY 1; 39 | 40 | INSERT INTO f2 41 | SELECT 1,COUNT(*) FROM dg 42 | GROUP BY 1; 43 | 44 | END; -------------------------------------------------------------------------------- /flink-connector-jdbc_1.13/src/test/resources/dws-2-mysql.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE cdc_f1 2 | ( 3 | `topic` STRING METADATA VIRTUAL, 4 | `user_id` BIGINT PRIMARY KEY NOT ENFORCED, 5 | `f1` BIGINT 6 | ) WITH ( 7 | 'connector' = 'upsert-kafka', 8 | 'topic' = 'cdc_f1', 9 | 'properties.bootstrap.servers' = 'localhost:9092', 10 | 'value.format' = 'json', 11 | 'value.fields-include' = 'EXCEPT_KEY', 12 | 'key.format' = 'json' 13 | ); 14 | CREATE TABLE test 15 | ( 16 | uid bigint primary key not enforced, 17 | f1 bigint, 18 | f2 bigint, 19 | prepareStatementTag STRING METADATA 20 | ) WITH( 21 | 'connector' = 'partial-jdbc', 22 | 'url' = 'jdbc:mysql://localhost:3306/cdc', 23 | 'table-name' = 'test', 24 | 'username' = 'root', 25 | 'password' = '!QAZ2wsx', 26 | 'prepare-statement.cdc_f1.fields' = 'uid,f1', 27 | 'prepare-statement.f2.fields' = 'uid,f2', 28 | 'sink.buffer-flush.interval' = '1s', 29 | 'sink.buffer-flush.max-rows' = '10000' 30 | ); 31 | 32 | INSERT INTO test 33 | SELECT user_id, f1, CAST(NULL AS BIGINT), topic 34 | FROM cdc_f1; -------------------------------------------------------------------------------- /flink-connector-jdbc_1.13/src/test/resources/kafka-2-msql.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE f1 2 | ( 3 | `topic` STRING METADATA VIRTUAL, 4 | `user_id` BIGINT PRIMARY KEY NOT ENFORCED, 5 | `f1` BIGINT 6 | ) WITH ( 7 | 'connector' = 'upsert-kafka', 8 | 'topic' = 'f1', 9 | 'properties.bootstrap.servers' = 'localhost:9092', 10 | 'properties.group.id' = 'testGroup', 11 | 'value.format' = 'json', 12 | 'value.fields-include' = 'EXCEPT_KEY', 13 | 'key.format' = 'json' 14 | ); 15 | CREATE TABLE f2 16 | ( 17 | `topic` STRING METADATA VIRTUAL, 18 | `user_id` BIGINT PRIMARY KEY NOT ENFORCED, 19 | `f2` BIGINT 20 | ) WITH ( 21 | 'connector' = 'upsert-kafka', 22 | 'topic' = 'f2', 23 | 'properties.bootstrap.servers' = 'localhost:9092', 24 | 'properties.group.id' = 'testGroup', 25 | 'value.format' = 'json', 26 | 'value.fields-include' = 'EXCEPT_KEY', 27 | 'key.format' = 'json' 28 | ); 29 | 30 | CREATE TABLE test 31 | ( 32 | uid bigint primary key not enforced, 33 | f1 bigint, 34 | f2 bigint, 35 | prepareStatementTag STRING METADATA 36 | ) WITH( 37 | 'connector' = 'partial-jdbc', 38 | 'url' = 'jdbc:mysql://localhost:3306/cdc', 39 | 'table-name' = 'test', 40 | 'username' = 'root', 41 | 'password' = '!QAZ2wsx', 42 | 'prepare-statement.f1.fields' = 'uid,f1', 43 | 'prepare-statement.f2.fields' = 'uid,f2', 44 | 'sink.buffer-flush.interval' = '10s', 45 | 'sink.buffer-flush.max-rows' = '10000' 46 | ); 47 | INSERT INTO test 48 | SELECT user_id, f1, CAST(NULL AS BIGINT), topic 49 | FROM f1 50 | UNION ALL 51 | SELECT user_id, CAST(NULL AS BIGINT), f2, topic 52 | FROM f2; -------------------------------------------------------------------------------- /flink-connector-jdbc_1.13/src/test/resources/partial_update.sql: -------------------------------------------------------------------------------- 1 | SET execution.checkpointing.interval=1s; 2 | 3 | CREATE TABLE mysql_t 4 | ( 5 | uid bigint primary key not enforced, 6 | f1 bigint, 7 | f2 bigint, 8 | prepareStatementTag STRING METADATA 9 | ) WITH( 10 | 'connector' = 'partial-jdbc', 11 | 'url' = 'jdbc:mysql://localhost:3306/cdc', 12 | 'table-name' = 'test', 13 | 'username' = 'root', 14 | 'password' = '!QAZ2wsx', 15 | 'prepare-statement.1.fields' = 'uid,f1', 16 | 'prepare-statement.2.fields' = 'uid,f2' 17 | -- 'sink.buffer-flush.interval' = '10s', 18 | -- 'sink.buffer-flush.max-rows' = '1000' 19 | 20 | ); 21 | 22 | CREATE TABLE test 23 | ( 24 | uid bigint, 25 | f1 bigint, 26 | f2 bigint, 27 | prepareStatementTag STRING 28 | ) WITH ( 29 | 'connector' = 'kafka', 30 | 'topic' = 'test', 31 | 'properties.bootstrap.servers' = 'localhost:9092', 32 | 'format' = 'json' 33 | ); 34 | 35 | -- SELECT * FROM test; 36 | 37 | INSERT INTO mysql_t 38 | SELECT 1, 1, CAST(null AS BIGINT), '1' 39 | UNION ALL 40 | SELECT 1, CAST(null AS BIGINT), 1, '2'; 41 | 42 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.14/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.hiscat.flink 8 | flink-connector-jdbc_1.14 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 8 13 | 8 14 | UTF-8 15 | 1.14.2 16 | 3.3.0 17 | 2.3.9 18 | 19 | 20 | 21 | org.apache.flink 22 | flink-connector-jdbc_2.11 23 | ${flink.version} 24 | 25 | 26 | com.h2database 27 | h2 28 | 29 | 30 | org.apache.flink 31 | flink-shaded-force-shading 32 | 33 | 34 | 35 | 36 | org.apache.flink 37 | flink-table-common 38 | ${flink.version} 39 | provided 40 | 41 | 42 | org.apache.flink 43 | flink-table-api-java 44 | ${flink.version} 45 | provided 46 | 47 | 48 | org.apache.flink 49 | flink-table-api-java-bridge_2.11 50 | ${flink.version} 51 | provided 52 | 53 | 54 | 55 | 56 | 57 | org.apache.maven.plugins 58 | maven-shade-plugin 59 | ${maven-shade-plugin.version} 60 | 61 | 62 | 64 | 65 | false 66 | 67 | 68 | 69 | package 70 | 71 | shade 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.14/src/main/java/org/apache/flink/connector/jdbc/catalog/JdbcCatalogUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.jdbc.catalog; 20 | 21 | import static org.apache.flink.util.Preconditions.checkArgument; 22 | 23 | import org.apache.flink.connector.jdbc.dialect.JdbcDialect; 24 | import org.apache.flink.connector.jdbc.dialect.JdbcDialects; 25 | import org.apache.flink.connector.jdbc.dialect.MySQLDialect; 26 | import org.apache.flink.connector.jdbc.dialect.PostgresDialect; 27 | 28 | /** Utils for {@link JdbcCatalog}. */ 29 | public class JdbcCatalogUtils { 30 | /** 31 | * URL has to be without database, like "jdbc:postgresql://localhost:5432/" or 32 | * "jdbc:postgresql://localhost:5432" rather than "jdbc:postgresql://localhost:5432/db". 33 | */ 34 | public static void validateJdbcUrl(String url) { 35 | String[] parts = url.trim().split("/+"); 36 | 37 | checkArgument(parts.length == 2); 38 | } 39 | 40 | /** Create catalog instance from given information. */ 41 | public static AbstractJdbcCatalog createCatalog( 42 | String catalogName, 43 | String defaultDatabase, 44 | String username, 45 | String pwd, 46 | String baseUrl) { 47 | JdbcDialect dialect = JdbcDialects.get(baseUrl) 48 | .orElseThrow(()->new RuntimeException("no such dialect find " + baseUrl)); 49 | 50 | if (dialect instanceof PostgresDialect) { 51 | return new PostgresCatalog(catalogName, defaultDatabase, username, pwd, baseUrl); 52 | } else if (dialect instanceof MySQLDialect) { 53 | return new MySqlCatalog(catalogName, defaultDatabase, username, pwd, baseUrl); 54 | } else { 55 | throw new UnsupportedOperationException( 56 | String.format("Catalog for '%s' is not supported yet.", dialect)); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.14/src/main/java/org/apache/flink/connector/jdbc/dialect/JdbcDialectTypeMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.jdbc.dialect; 20 | 21 | import java.sql.ResultSetMetaData; 22 | import java.sql.SQLException; 23 | import org.apache.flink.table.catalog.ObjectPath; 24 | import org.apache.flink.table.types.DataType; 25 | 26 | /** Separate the jdbc meta-information type to flink table type into the interface. */ 27 | public interface JdbcDialectTypeMapper { 28 | 29 | DataType mapping(ObjectPath tablePath, ResultSetMetaData metadata, int colIndex) 30 | throws SQLException; 31 | } 32 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.14/src/main/java/org/apache/flink/connector/jdbc/table/TableExtractor.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.connector.jdbc.table; 2 | 3 | import org.apache.flink.table.data.RowData; 4 | 5 | import java.io.Serializable; 6 | import java.util.function.Function; 7 | 8 | public class TableExtractor implements Function, Serializable { 9 | private static final long serialVersionUID = 1L; 10 | 11 | private final int pos; 12 | 13 | public TableExtractor(int pos) { 14 | this.pos = pos; 15 | } 16 | 17 | public int getPos() { 18 | return pos; 19 | } 20 | 21 | @Override 22 | public String apply(RowData rowData) { 23 | return RouteJdbcDynamicTableSink.TableMetadata.TABLE.converter.read(rowData, pos).toString(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.14/src/main/java/org/apache/flink/connector/jdbc/table/executor/AbstractRouteJdbcBatchStatementExecutor.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.connector.jdbc.table.executor; 2 | 3 | import org.apache.flink.connector.jdbc.internal.converter.JdbcRowConverter; 4 | import org.apache.flink.connector.jdbc.internal.executor.JdbcBatchStatementExecutor; 5 | import org.apache.flink.connector.jdbc.internal.options.JdbcDmlOptions; 6 | import org.apache.flink.connector.jdbc.statement.FieldNamedPreparedStatement; 7 | import org.apache.flink.table.data.RowData; 8 | 9 | import java.sql.Connection; 10 | import java.sql.SQLException; 11 | import java.util.*; 12 | import java.util.function.Function; 13 | 14 | public abstract class AbstractRouteJdbcBatchStatementExecutor 15 | implements JdbcBatchStatementExecutor { 16 | 17 | public static final String TABLE = "table"; 18 | private final Function tableExtractor; 19 | 20 | protected final JdbcDmlOptions dmlOptions; 21 | private final JdbcRowConverter rowConverter; 22 | 23 | protected transient Connection connection; 24 | 25 | private final Map> buffer; 26 | 27 | public AbstractRouteJdbcBatchStatementExecutor( 28 | JdbcDmlOptions dmlOptions, 29 | Function tableExtractor, 30 | JdbcRowConverter rowConverter) { 31 | this.tableExtractor = tableExtractor; 32 | this.dmlOptions = dmlOptions; 33 | this.rowConverter = rowConverter; 34 | this.buffer = new HashMap<>(); 35 | } 36 | 37 | @Override 38 | public void prepareStatements(Connection connection) { 39 | this.connection = connection; 40 | } 41 | 42 | @Override 43 | public void addToBatch(RowData record) { 44 | String table = tableExtractor.apply(record); 45 | buffer.computeIfAbsent(table, k -> new ArrayList<>()).add(record); 46 | } 47 | 48 | @Override 49 | public void executeBatch() { 50 | buffer.forEach((table, buffer) -> { 51 | if (buffer.isEmpty()) { 52 | return; 53 | } 54 | try { 55 | FieldNamedPreparedStatement prepareStatement = getFieldNamedPreparedStatement(table); 56 | buffer.forEach(e -> { 57 | try { 58 | rowConverter.toExternal(e, prepareStatement); 59 | prepareStatement.addBatch(); 60 | } catch (SQLException ex) { 61 | throw new RuntimeException(ex); 62 | } 63 | }); 64 | prepareStatement.executeBatch(); 65 | prepareStatement.close(); 66 | buffer.clear(); 67 | } catch (SQLException e) { 68 | throw new RuntimeException(e); 69 | } 70 | }); 71 | buffer.clear(); 72 | } 73 | 74 | @Override 75 | public void closeStatements() { 76 | buffer.clear(); 77 | } 78 | 79 | protected abstract FieldNamedPreparedStatement getFieldNamedPreparedStatement(String table) throws SQLException; 80 | } 81 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.14/src/main/java/org/apache/flink/connector/jdbc/table/executor/RouteJdbcDeleteBatchStatementExecutor.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.connector.jdbc.table.executor; 2 | 3 | import org.apache.flink.connector.jdbc.internal.converter.JdbcRowConverter; 4 | import org.apache.flink.connector.jdbc.internal.executor.JdbcBatchStatementExecutor; 5 | import org.apache.flink.connector.jdbc.internal.options.JdbcDmlOptions; 6 | import org.apache.flink.connector.jdbc.statement.FieldNamedPreparedStatement; 7 | import org.apache.flink.table.data.RowData; 8 | 9 | import java.sql.SQLException; 10 | import java.util.function.Function; 11 | 12 | public class RouteJdbcDeleteBatchStatementExecutor extends AbstractRouteJdbcBatchStatementExecutor 13 | implements JdbcBatchStatementExecutor { 14 | 15 | 16 | public RouteJdbcDeleteBatchStatementExecutor( 17 | JdbcDmlOptions dmlOptions, 18 | Function tableExtractor, 19 | JdbcRowConverter rowConverter) { 20 | super(dmlOptions, tableExtractor, rowConverter); 21 | } 22 | 23 | @Override 24 | protected FieldNamedPreparedStatement getFieldNamedPreparedStatement(String table) throws SQLException { 25 | String[] uniqueKeyFields = dmlOptions.getKeyFields() 26 | .orElseThrow(() -> new RuntimeException("pk not exists")); 27 | String sql = dmlOptions.getDialect().getDeleteStatement(table, uniqueKeyFields); 28 | return FieldNamedPreparedStatement 29 | .prepareStatement(connection, sql, uniqueKeyFields); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.14/src/main/java/org/apache/flink/connector/jdbc/table/executor/RouteJdbcUpsertBatchStatementExecutor.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.connector.jdbc.table.executor; 2 | 3 | import org.apache.flink.connector.jdbc.internal.converter.JdbcRowConverter; 4 | import org.apache.flink.connector.jdbc.internal.executor.JdbcBatchStatementExecutor; 5 | import org.apache.flink.connector.jdbc.internal.options.JdbcDmlOptions; 6 | import org.apache.flink.connector.jdbc.statement.FieldNamedPreparedStatement; 7 | import org.apache.flink.table.data.RowData; 8 | 9 | import java.sql.SQLException; 10 | import java.util.Arrays; 11 | import java.util.function.Function; 12 | 13 | public class RouteJdbcUpsertBatchStatementExecutor extends AbstractRouteJdbcBatchStatementExecutor 14 | implements JdbcBatchStatementExecutor { 15 | 16 | public RouteJdbcUpsertBatchStatementExecutor( 17 | JdbcDmlOptions dmlOptions, 18 | Function tableExtractor, 19 | JdbcRowConverter rowConverter) { 20 | super(dmlOptions, tableExtractor, rowConverter); 21 | } 22 | 23 | @Override 24 | protected FieldNamedPreparedStatement getFieldNamedPreparedStatement(String table) throws SQLException { 25 | String[] uniqueKeyFields = dmlOptions.getKeyFields() 26 | .orElseThrow(() -> new RuntimeException("pk not exists")); 27 | String[] fieldNames = Arrays.stream(dmlOptions.getFieldNames()) 28 | .filter(f -> !f.equals(TABLE)).toArray(String[]::new); 29 | String sql = dmlOptions.getDialect().getUpsertStatement(table, fieldNames, uniqueKeyFields) 30 | .orElseThrow(() -> new RuntimeException("upsert sql not provided")); 31 | return FieldNamedPreparedStatement 32 | .prepareStatement(connection, sql, fieldNames); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.14/src/main/java/org/apache/flink/connector/jdbc/utils/JdbcTypeUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.jdbc.utils; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | import org.apache.flink.api.common.typeinfo.LocalTimeTypeInfo; 23 | import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; 24 | import org.apache.flink.api.common.typeinfo.SqlTimeTypeInfo; 25 | import org.apache.flink.api.common.typeinfo.TypeInformation; 26 | import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; 27 | import org.apache.flink.table.api.TableSchema; 28 | import org.apache.flink.table.types.DataType; 29 | import org.apache.flink.table.types.inference.TypeTransformations; 30 | import org.apache.flink.table.types.utils.DataTypeUtils; 31 | 32 | import java.sql.Types; 33 | import java.util.Collections; 34 | import java.util.HashMap; 35 | import java.util.Map; 36 | 37 | import static org.apache.flink.api.common.typeinfo.BasicTypeInfo.*; 38 | import static org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO; 39 | 40 | /** Utils for jdbc type. */ 41 | @Internal 42 | public class JdbcTypeUtil { 43 | private static final Map, Integer> TYPE_MAPPING; 44 | private static final Map SQL_TYPE_NAMES; 45 | 46 | static { 47 | HashMap, Integer> m = new HashMap<>(); 48 | m.put(STRING_TYPE_INFO, Types.VARCHAR); 49 | m.put(BOOLEAN_TYPE_INFO, Types.BOOLEAN); 50 | m.put(BYTE_TYPE_INFO, Types.TINYINT); 51 | m.put(SHORT_TYPE_INFO, Types.SMALLINT); 52 | m.put(INT_TYPE_INFO, Types.INTEGER); 53 | m.put(LONG_TYPE_INFO, Types.BIGINT); 54 | m.put(FLOAT_TYPE_INFO, Types.REAL); 55 | m.put(DOUBLE_TYPE_INFO, Types.DOUBLE); 56 | m.put(SqlTimeTypeInfo.DATE, Types.DATE); 57 | m.put(SqlTimeTypeInfo.TIME, Types.TIME); 58 | m.put(SqlTimeTypeInfo.TIMESTAMP, Types.TIMESTAMP); 59 | m.put(LocalTimeTypeInfo.LOCAL_DATE, Types.DATE); 60 | m.put(LocalTimeTypeInfo.LOCAL_TIME, Types.TIME); 61 | m.put(LocalTimeTypeInfo.LOCAL_DATE_TIME, Types.TIMESTAMP); 62 | m.put(INSTANT_TYPE_INFO, Types.TIMESTAMP_WITH_TIMEZONE); 63 | m.put(BIG_DEC_TYPE_INFO, Types.DECIMAL); 64 | m.put(BYTE_PRIMITIVE_ARRAY_TYPE_INFO, Types.BINARY); 65 | TYPE_MAPPING = Collections.unmodifiableMap(m); 66 | 67 | HashMap names = new HashMap<>(); 68 | names.put(Types.VARCHAR, "VARCHAR"); 69 | names.put(Types.BOOLEAN, "BOOLEAN"); 70 | names.put(Types.TINYINT, "TINYINT"); 71 | names.put(Types.SMALLINT, "SMALLINT"); 72 | names.put(Types.INTEGER, "INTEGER"); 73 | names.put(Types.BIGINT, "BIGINT"); 74 | names.put(Types.FLOAT, "FLOAT"); 75 | names.put(Types.DOUBLE, "DOUBLE"); 76 | names.put(Types.CHAR, "CHAR"); 77 | names.put(Types.DATE, "DATE"); 78 | names.put(Types.TIME, "TIME"); 79 | names.put(Types.TIMESTAMP, "TIMESTAMP"); 80 | names.put(Types.DECIMAL, "DECIMAL"); 81 | names.put(Types.BINARY, "BINARY"); 82 | SQL_TYPE_NAMES = Collections.unmodifiableMap(names); 83 | } 84 | 85 | private JdbcTypeUtil() {} 86 | 87 | public static int typeInformationToSqlType(TypeInformation type) { 88 | 89 | if (TYPE_MAPPING.containsKey(type)) { 90 | return TYPE_MAPPING.get(type); 91 | } else if (type instanceof ObjectArrayTypeInfo || type instanceof PrimitiveArrayTypeInfo) { 92 | return Types.ARRAY; 93 | } else { 94 | throw new IllegalArgumentException("Unsupported type: " + type); 95 | } 96 | } 97 | 98 | public static String getTypeName(int type) { 99 | return SQL_TYPE_NAMES.get(type); 100 | } 101 | 102 | public static String getTypeName(TypeInformation type) { 103 | return SQL_TYPE_NAMES.get(typeInformationToSqlType(type)); 104 | } 105 | 106 | /** 107 | * The original table schema may contain generated columns which shouldn't be produced/consumed 108 | * by TableSource/TableSink. And the original TIMESTAMP/DATE/TIME types uses 109 | * LocalDateTime/LocalDate/LocalTime as the conversion classes, however, JDBC connector uses 110 | * Timestamp/Date/Time classes. So that we bridge them to the expected conversion classes. 111 | */ 112 | public static TableSchema normalizeTableSchema(TableSchema schema) { 113 | TableSchema.Builder physicalSchemaBuilder = TableSchema.builder(); 114 | schema.getTableColumns() 115 | .forEach( 116 | c -> { 117 | if (c.isPhysical()) { 118 | final DataType type = 119 | DataTypeUtils.transform( 120 | c.getType(), TypeTransformations.timeToSqlTypes()); 121 | physicalSchemaBuilder.field(c.getName(), type); 122 | } 123 | }); 124 | return physicalSchemaBuilder.build(); 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /flink-connector-jdbc_1.14/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory: -------------------------------------------------------------------------------- 1 | org.apache.flink.connector.jdbc.table.RouteJdbcDynamicTableFactory -------------------------------------------------------------------------------- /flink-connector-kafka/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.hiscat.flink 8 | flink-connector-kafka 9 | 1.0-SNAPSHOT 10 | 11 | 8 12 | 8 13 | 1.14.2 14 | 3.3.0 15 | 1.18.22 16 | 17 | 18 | 19 | org.apache.flink 20 | flink-table-planner_2.11 21 | ${flink.version} 22 | provided 23 | 24 | 25 | org.apache.flink 26 | flink-connector-kafka_2.11 27 | ${flink.version} 28 | 29 | 30 | 31 | org.projectlombok 32 | lombok 33 | ${lombok.version} 34 | provided 35 | 36 | 37 | 38 | 39 | 40 | 41 | org.apache.maven.plugins 42 | maven-shade-plugin 43 | ${maven-shade-plugin.version} 44 | 45 | 46 | 48 | 49 | false 50 | 51 | 52 | 53 | package 54 | 55 | shade 56 | 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/table/RouteDynamicKafkaRecordSerializationSchema.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.streaming.connectors.kafka.table; 2 | 3 | import java.util.Objects; 4 | import org.apache.flink.api.common.serialization.SerializationSchema.InitializationContext; 5 | import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema; 6 | import org.apache.flink.streaming.connectors.kafka.table.RouteKafkaDynamicSink.TopicMetadata; 7 | import org.apache.flink.table.data.RowData; 8 | import org.apache.kafka.clients.producer.ProducerRecord; 9 | 10 | public class RouteDynamicKafkaRecordSerializationSchema 11 | implements KafkaRecordSerializationSchema { 12 | 13 | private final KafkaRecordSerializationSchema rowDataKafkaRecordSerializationSchema; 14 | 15 | private final int topicPos; 16 | 17 | public RouteDynamicKafkaRecordSerializationSchema( 18 | KafkaRecordSerializationSchema rowDataKafkaRecordSerializationSchema, 19 | int topicMetadataPosition) { 20 | this.rowDataKafkaRecordSerializationSchema = rowDataKafkaRecordSerializationSchema; 21 | this.topicPos = topicMetadataPosition; 22 | } 23 | 24 | @Override 25 | public void open(InitializationContext context, KafkaSinkContext sinkContext) throws Exception { 26 | this.rowDataKafkaRecordSerializationSchema.open(context, sinkContext); 27 | } 28 | 29 | @Override 30 | public ProducerRecord serialize( 31 | RowData element, KafkaSinkContext context, Long timestamp) { 32 | ProducerRecord record = 33 | this.rowDataKafkaRecordSerializationSchema.serialize(element, context, timestamp); 34 | Object topic = TopicMetadata.TOPIC.converter.read(element, topicPos); 35 | if (Objects.isNull(topic)) { 36 | return record; 37 | } 38 | return new ProducerRecord<>( 39 | topic.toString(), 40 | record.partition(), 41 | record.timestamp(), 42 | record.key(), 43 | record.value(), 44 | record.headers()); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /flink-connector-kafka/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory: -------------------------------------------------------------------------------- 1 | org.apache.flink.streaming.connectors.kafka.table.RouteKafkaDynamicTableFactory -------------------------------------------------------------------------------- /flink-custom-connector/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.hiscat 8 | flink-demo 9 | 1.0-SNAPSHOT 10 | ../pom.xml 11 | 12 | 13 | flink-custom-connector 14 | 1.0-SNAPSHOT 15 | flink-custom-connector 16 | 17 | 18 | 1.2 19 | 5.1.8.RELEASE 20 | 21 | 22 | 23 | 24 | 25 | com.sproutsocial 26 | nsq-j 27 | ${nsq-j.version} 28 | 29 | 30 | 31 | io.lettuce 32 | lettuce-core 33 | ${lettuce-core.version} 34 | 35 | 36 | org.apache.flink 37 | flink-clients_${scala.binary.version} 38 | 39 | 40 | org.apache.flink 41 | flink-table-planner_${scala.binary.version} 42 | 43 | 44 | 45 | org.apache.flink 46 | flink-json 47 | 48 | 49 | 50 | org.apache.flink 51 | flink-connector-hive_${scala.binary.version} 52 | ${flink.version} 53 | provided 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/connector/nsq/NsqDynamicTableFactory.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.custrom.connector.nsq; 2 | 3 | import com.hiscat.flink.custrom.connector.nsq.util.IndexGenerator; 4 | import com.hiscat.flink.custrom.connector.nsq.util.IndexGeneratorFactory; 5 | import org.apache.flink.api.common.serialization.DeserializationSchema; 6 | import org.apache.flink.api.common.serialization.SerializationSchema; 7 | import org.apache.flink.configuration.ConfigOption; 8 | import org.apache.flink.shaded.guava30.com.google.common.collect.Sets; 9 | import org.apache.flink.table.connector.format.DecodingFormat; 10 | import org.apache.flink.table.connector.format.EncodingFormat; 11 | import org.apache.flink.table.connector.sink.DynamicTableSink; 12 | import org.apache.flink.table.connector.source.DynamicTableSource; 13 | import org.apache.flink.table.data.RowData; 14 | import org.apache.flink.table.factories.*; 15 | import org.apache.flink.table.types.DataType; 16 | 17 | import java.util.Collections; 18 | import java.util.Set; 19 | 20 | public class NsqDynamicTableFactory implements DynamicTableSinkFactory, DynamicTableSourceFactory { 21 | 22 | @Override 23 | public String factoryIdentifier() { 24 | return "nsq"; 25 | } 26 | 27 | @Override 28 | public Set> requiredOptions() { 29 | return Sets.newHashSet( 30 | NsqOptions.HOST, 31 | NsqOptions.TOPIC, 32 | NsqOptions.FORMAT 33 | ); 34 | } 35 | 36 | @Override 37 | public Set> optionalOptions() { 38 | return Collections.singleton(NsqOptions.CHANNEL); 39 | } 40 | 41 | @Override 42 | public DynamicTableSink createDynamicTableSink(final Context context) { 43 | final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context); 44 | 45 | helper.validate(); 46 | 47 | final NsqOptions nsqOptions = NsqOptions.builder() 48 | .host(helper.getOptions().get(NsqOptions.HOST)) 49 | .topic(helper.getOptions().get(NsqOptions.TOPIC)) 50 | .build(); 51 | 52 | final EncodingFormat> encodingFormat = 53 | helper.discoverEncodingFormat(SerializationFormatFactory.class, NsqOptions.FORMAT); 54 | 55 | final DataType producedDataTypes = context.getCatalogTable().getResolvedSchema().toSinkRowDataType(); 56 | 57 | final IndexGenerator indexGenerator = IndexGeneratorFactory 58 | .createIndexGenerator(nsqOptions.getTopic(), context.getCatalogTable().getSchema()); 59 | 60 | return new NsqDynamicTableSink(nsqOptions, encodingFormat, producedDataTypes, indexGenerator); 61 | } 62 | 63 | @Override 64 | public DynamicTableSource createDynamicTableSource(final Context context) { 65 | final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context); 66 | 67 | helper.validate(); 68 | 69 | final String channel = helper.getOptions().getOptional(NsqOptions.CHANNEL) 70 | .orElseThrow(() -> new IllegalArgumentException("miss channel arg")); 71 | 72 | final NsqOptions nsqOptions = NsqOptions.builder() 73 | .host(helper.getOptions().get(NsqOptions.HOST)) 74 | .topic(helper.getOptions().get(NsqOptions.TOPIC)) 75 | .channel(channel) 76 | .build(); 77 | 78 | final DecodingFormat> decodingFormat = 79 | helper.discoverDecodingFormat(DeserializationFormatFactory.class, NsqOptions.FORMAT); 80 | final DataType consumedDataTypes = context.getCatalogTable().getResolvedSchema().toSinkRowDataType(); 81 | 82 | return new NsqDynamicTableSource(nsqOptions, decodingFormat, consumedDataTypes); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/connector/nsq/NsqDynamicTableSink.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.custrom.connector.nsq; 2 | 3 | import com.hiscat.flink.custrom.connector.nsq.util.IndexGenerator; 4 | import org.apache.flink.api.common.serialization.SerializationSchema; 5 | import org.apache.flink.table.connector.ChangelogMode; 6 | import org.apache.flink.table.connector.format.EncodingFormat; 7 | import org.apache.flink.table.connector.sink.DynamicTableSink; 8 | import org.apache.flink.table.connector.sink.SinkFunctionProvider; 9 | import org.apache.flink.table.data.RowData; 10 | import org.apache.flink.table.types.DataType; 11 | 12 | public class NsqDynamicTableSink implements DynamicTableSink { 13 | 14 | private final NsqOptions nsqOptions; 15 | private final EncodingFormat> encodingFormat; 16 | private final DataType producedDataTypes; 17 | private final IndexGenerator indexGenerator; 18 | 19 | public NsqDynamicTableSink(final NsqOptions nsqOptions, 20 | final EncodingFormat> encodingFormat, 21 | final DataType producedDataTypes, final IndexGenerator indexGenerator) { 22 | this.nsqOptions = nsqOptions; 23 | this.encodingFormat = encodingFormat; 24 | this.producedDataTypes = producedDataTypes; 25 | this.indexGenerator = indexGenerator; 26 | } 27 | 28 | @Override 29 | public ChangelogMode getChangelogMode(final ChangelogMode requestedMode) { 30 | return ChangelogMode.upsert(); 31 | } 32 | 33 | @Override 34 | public SinkRuntimeProvider getSinkRuntimeProvider(final Context context) { 35 | final SerializationSchema encoder = encodingFormat.createRuntimeEncoder(context, producedDataTypes); 36 | return SinkFunctionProvider.of(new NsqSinkFunction(nsqOptions, encoder, indexGenerator)); 37 | } 38 | 39 | @Override 40 | public DynamicTableSink copy() { 41 | return new NsqDynamicTableSink(nsqOptions, encodingFormat, producedDataTypes, indexGenerator); 42 | } 43 | 44 | @Override 45 | public String asSummaryString() { 46 | return "nsq dynamic table sink"; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/connector/nsq/NsqDynamicTableSource.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.custrom.connector.nsq; 2 | 3 | import org.apache.flink.api.common.serialization.DeserializationSchema; 4 | import org.apache.flink.table.connector.ChangelogMode; 5 | import org.apache.flink.table.connector.format.DecodingFormat; 6 | import org.apache.flink.table.connector.source.DynamicTableSource; 7 | import org.apache.flink.table.connector.source.ScanTableSource; 8 | import org.apache.flink.table.connector.source.SourceFunctionProvider; 9 | import org.apache.flink.table.data.RowData; 10 | import org.apache.flink.table.types.DataType; 11 | 12 | public class NsqDynamicTableSource implements ScanTableSource { 13 | 14 | private final NsqOptions nsqOptions; 15 | private final DecodingFormat> decodingFormat; 16 | private final DataType consumedDataTypes; 17 | 18 | public NsqDynamicTableSource(final NsqOptions nsqOptions, 19 | final DecodingFormat> decodingFormat, 20 | final DataType consumedDataTypes) { 21 | this.nsqOptions = nsqOptions; 22 | this.consumedDataTypes = consumedDataTypes; 23 | this.decodingFormat = decodingFormat; 24 | } 25 | 26 | 27 | @Override 28 | public DynamicTableSource copy() { 29 | return new NsqDynamicTableSource(nsqOptions, decodingFormat, consumedDataTypes); 30 | } 31 | 32 | @Override 33 | public String asSummaryString() { 34 | return "nsq dynamic table source"; 35 | } 36 | 37 | @Override 38 | public ChangelogMode getChangelogMode() { 39 | return decodingFormat.getChangelogMode(); 40 | } 41 | 42 | @Override 43 | public ScanRuntimeProvider getScanRuntimeProvider(final ScanContext runtimeProviderContext) { 44 | final DeserializationSchema deserializationSchema = decodingFormat.createRuntimeDecoder(runtimeProviderContext, consumedDataTypes); 45 | return SourceFunctionProvider.of(new NsqSourceFunction(nsqOptions, deserializationSchema), false); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/connector/nsq/NsqOptions.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.custrom.connector.nsq; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | import org.apache.flink.configuration.ConfigOption; 8 | import org.apache.flink.configuration.ConfigOptions; 9 | 10 | import java.io.Serializable; 11 | 12 | @Data 13 | @Builder 14 | @NoArgsConstructor 15 | @AllArgsConstructor 16 | public class NsqOptions implements Serializable { 17 | 18 | public static final ConfigOption HOST = ConfigOptions.key("host").stringType().noDefaultValue(); 19 | public static final ConfigOption TOPIC = ConfigOptions.key("topic").stringType().noDefaultValue(); 20 | public static final ConfigOption FORMAT = ConfigOptions.key("format").stringType().noDefaultValue(); 21 | public static final ConfigOption CHANNEL = ConfigOptions.key("channel").stringType().noDefaultValue(); 22 | 23 | private String host; 24 | private String topic; 25 | private String channel; 26 | 27 | } 28 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/connector/nsq/NsqSinkFunction.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.custrom.connector.nsq; 2 | 3 | import com.hiscat.flink.custrom.connector.nsq.util.IndexGenerator; 4 | import com.sproutsocial.nsq.Publisher; 5 | import org.apache.flink.api.common.serialization.SerializationSchema; 6 | import org.apache.flink.configuration.Configuration; 7 | import org.apache.flink.metrics.MetricGroup; 8 | import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; 9 | import org.apache.flink.table.data.RowData; 10 | import org.apache.flink.util.UserCodeClassLoader; 11 | 12 | public class NsqSinkFunction extends RichSinkFunction { 13 | 14 | private final NsqOptions nsqOptions; 15 | private final SerializationSchema encoder; 16 | private final IndexGenerator indexGenerator; 17 | private Publisher publisher; 18 | 19 | public NsqSinkFunction(final NsqOptions nsqOptions, 20 | final SerializationSchema encoder, 21 | final IndexGenerator indexGenerator) { 22 | this.nsqOptions = nsqOptions; 23 | this.encoder = encoder; 24 | this.indexGenerator = indexGenerator; 25 | } 26 | 27 | @Override 28 | public void open(final Configuration parameters) throws Exception { 29 | encoder.open(new SerializationSchema.InitializationContext() { 30 | @Override 31 | public MetricGroup getMetricGroup() { 32 | return getRuntimeContext().getMetricGroup(); 33 | } 34 | 35 | @Override 36 | public UserCodeClassLoader getUserCodeClassLoader() { 37 | return (UserCodeClassLoader) getRuntimeContext().getUserCodeClassLoader(); 38 | } 39 | }); 40 | this.publisher = new Publisher(nsqOptions.getHost()); 41 | } 42 | 43 | @Override 44 | public void close() { 45 | publisher.stop(); 46 | } 47 | 48 | @Override 49 | public void invoke(final RowData value, final Context context) { 50 | switch (value.getRowKind()) { 51 | case INSERT: 52 | case UPDATE_AFTER: 53 | publisher.publishBuffered(indexGenerator.generate(value), encoder.serialize(value)); 54 | break; 55 | default: 56 | break; 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/connector/nsq/NsqSourceFunction.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.custrom.connector.nsq; 2 | 3 | import com.sproutsocial.nsq.MessageDataHandler; 4 | import com.sproutsocial.nsq.Subscriber; 5 | import lombok.extern.slf4j.Slf4j; 6 | import org.apache.flink.api.common.serialization.DeserializationSchema; 7 | import org.apache.flink.configuration.Configuration; 8 | import org.apache.flink.metrics.MetricGroup; 9 | import org.apache.flink.streaming.api.functions.source.RichSourceFunction; 10 | import org.apache.flink.table.data.RowData; 11 | import org.apache.flink.util.UserCodeClassLoader; 12 | 13 | import java.io.IOException; 14 | import java.util.concurrent.Executors; 15 | 16 | @Slf4j 17 | public class NsqSourceFunction extends RichSourceFunction { 18 | 19 | private final NsqOptions nsqOptions; 20 | private final DeserializationSchema deserializationSchema; 21 | private Subscriber subscriber; 22 | private boolean running; 23 | 24 | public NsqSourceFunction(final NsqOptions nsqOptions, final DeserializationSchema deserializationSchema) { 25 | this.nsqOptions = nsqOptions; 26 | this.deserializationSchema = deserializationSchema; 27 | } 28 | 29 | @Override 30 | public void open(final Configuration parameters) throws Exception { 31 | running = true; 32 | subscriber = new Subscriber(nsqOptions.getHost()); 33 | subscriber.getClient().setExecutor(Executors.newSingleThreadExecutor()); 34 | deserializationSchema.open(new DeserializationSchema.InitializationContext() { 35 | @Override 36 | public MetricGroup getMetricGroup() { 37 | return getRuntimeContext().getMetricGroup(); 38 | } 39 | 40 | @Override 41 | public UserCodeClassLoader getUserCodeClassLoader() { 42 | return (UserCodeClassLoader) NsqSourceFunction.class.getClassLoader(); 43 | } 44 | }); 45 | } 46 | 47 | @Override 48 | public void run(final SourceContext ctx) throws InterruptedException { 49 | subscriber.subscribe(nsqOptions.getTopic(), nsqOptions.getChannel(), (MessageDataHandler) data -> { 50 | try { 51 | ctx.collect(deserializationSchema.deserialize(data)); 52 | } catch (IOException e) { 53 | log.error("subscribe error : {}", e.getMessage(), e); 54 | } 55 | }); 56 | while (running) { 57 | Thread.sleep(1000); 58 | } 59 | } 60 | 61 | @Override 62 | public void cancel() { 63 | running = false; 64 | subscriber.stop(); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/connector/nsq/util/AbstractTimeIndexGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.hiscat.flink.custrom.connector.nsq.util; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | 23 | import java.time.format.DateTimeFormatter; 24 | 25 | /** Abstract class for time related {@link IndexGenerator}. */ 26 | @Internal 27 | abstract class AbstractTimeIndexGenerator extends IndexGeneratorBase { 28 | 29 | private final String dateTimeFormat; 30 | protected transient DateTimeFormatter dateTimeFormatter; 31 | 32 | public AbstractTimeIndexGenerator(String index, String dateTimeFormat) { 33 | super(index); 34 | this.dateTimeFormat = dateTimeFormat; 35 | } 36 | 37 | @Override 38 | public void open() { 39 | this.dateTimeFormatter = DateTimeFormatter.ofPattern(dateTimeFormat); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/connector/nsq/util/IndexGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.hiscat.flink.custrom.connector.nsq.util; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | import org.apache.flink.table.data.RowData; 23 | import org.apache.flink.types.Row; 24 | 25 | import java.io.Serializable; 26 | 27 | /** This interface is responsible to generate index name from given {@link Row} record. */ 28 | @Internal 29 | public interface IndexGenerator extends Serializable { 30 | 31 | /** 32 | * Initialize the index generator, this will be called only once before {@link 33 | * #generate(RowData)} is called. 34 | */ 35 | default void open() {} 36 | 37 | /** Generate index name according the the given row. */ 38 | String generate(RowData row); 39 | } 40 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/connector/nsq/util/IndexGeneratorBase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.hiscat.flink.custrom.connector.nsq.util; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | 23 | import java.util.Objects; 24 | 25 | /** Base class for {@link IndexGenerator}. */ 26 | @Internal 27 | public abstract class IndexGeneratorBase implements IndexGenerator { 28 | 29 | private static final long serialVersionUID = 1L; 30 | protected final String index; 31 | 32 | public IndexGeneratorBase(String index) { 33 | this.index = index; 34 | } 35 | 36 | @Override 37 | public boolean equals(Object o) { 38 | if (this == o) { 39 | return true; 40 | } 41 | if (!(o instanceof IndexGeneratorBase)) { 42 | return false; 43 | } 44 | IndexGeneratorBase that = (IndexGeneratorBase) o; 45 | return index.equals(that.index); 46 | } 47 | 48 | @Override 49 | public int hashCode() { 50 | return Objects.hash(index); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/connector/nsq/util/StaticIndexGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.hiscat.flink.custrom.connector.nsq.util; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | import org.apache.flink.table.data.RowData; 23 | 24 | /** A static {@link IndexGenerator} which generate fixed index name. */ 25 | @Internal 26 | final class StaticIndexGenerator extends IndexGeneratorBase { 27 | 28 | public StaticIndexGenerator(String index) { 29 | super(index); 30 | } 31 | 32 | public String generate(RowData row) { 33 | return index; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/connector/redis/ConnectionProvider.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.custrom.connector.redis; 2 | 3 | import io.lettuce.core.RedisClient; 4 | import io.lettuce.core.RedisURI; 5 | import io.lettuce.core.api.StatefulRedisConnection; 6 | 7 | public class ConnectionProvider { 8 | public static StatefulRedisConnection getConnection(RedisOptions options) { 9 | RedisURI redisUri = RedisURI.builder() 10 | .withHost(options.getHost()) 11 | .withPort(options.getPort()) 12 | .build(); 13 | redisUri.setPassword(options.getPassword()); 14 | // if (options.getPassword() != null) { 15 | // redisUri.setPassword(DESUtils.decryptFromBase64(options.getPassword())); 16 | // } 17 | RedisClient redisClient = RedisClient.create(redisUri); 18 | return redisClient.connect(); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/connector/redis/RedisAsyncLookupFunction.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.custrom.connector.redis; 2 | 3 | import io.lettuce.core.api.StatefulRedisConnection; 4 | import io.lettuce.core.api.async.RedisAsyncCommands; 5 | import lombok.extern.slf4j.Slf4j; 6 | import org.apache.flink.shaded.guava30.com.google.common.cache.Cache; 7 | import org.apache.flink.shaded.guava30.com.google.common.cache.CacheBuilder; 8 | import org.apache.flink.table.functions.AsyncTableFunction; 9 | import org.apache.flink.table.functions.FunctionContext; 10 | import org.apache.flink.types.Row; 11 | 12 | import java.util.Collection; 13 | import java.util.Collections; 14 | import java.util.List; 15 | import java.util.concurrent.CompletableFuture; 16 | import java.util.concurrent.TimeUnit; 17 | 18 | @Slf4j 19 | public class RedisAsyncLookupFunction extends AsyncTableFunction { 20 | 21 | 22 | private final RedisOptions options; 23 | private final List columnNames; 24 | private Cache cache; 25 | private RedisAsyncCommands commands; 26 | private StatefulRedisConnection connection; 27 | 28 | public RedisAsyncLookupFunction(final RedisOptions options, final List columnNames) { 29 | this.options = options; 30 | this.columnNames = columnNames; 31 | } 32 | 33 | @Override 34 | public void open(final FunctionContext context) { 35 | connection = ConnectionProvider.getConnection(options); 36 | commands = this.connection.async(); 37 | 38 | this.cache = 39 | options.getCacheSize() == -1 || options.getCacheExpireMs() == -1 40 | ? null 41 | : CacheBuilder.newBuilder() 42 | .expireAfterWrite(options.getCacheExpireMs(), TimeUnit.MILLISECONDS) 43 | .maximumSize(options.getCacheSize()) 44 | .build(); 45 | } 46 | 47 | @SuppressWarnings("unused") 48 | public void eval(CompletableFuture> future, String pk) { 49 | int currentRetry = 0; 50 | if (cache != null) { 51 | Row cacheRow = cache.getIfPresent(pk); 52 | if (cacheRow != null) { 53 | if (cacheRow.getArity() == 0) { 54 | future.complete(Collections.emptyList()); 55 | } else { 56 | future.complete(Collections.singletonList(cacheRow)); 57 | } 58 | return; 59 | } 60 | } 61 | fetchResult(future, currentRetry, pk); 62 | } 63 | 64 | private void fetchResult( 65 | CompletableFuture> resultFuture, int currentRetry, String pk) { 66 | commands.hgetall(String.format("%s%s%s", options.getTableName(), options.getDelimiter(), pk)) 67 | .whenCompleteAsync( 68 | (result, throwable) -> { 69 | if (throwable != null) { 70 | log.error(String.format("Redis asyncLookup error, retry times = %d", currentRetry), throwable); 71 | if (currentRetry >= options.getMaxRetryTimes()) { 72 | resultFuture.completeExceptionally(throwable); 73 | return; 74 | } 75 | try { 76 | Thread.sleep(1000L*currentRetry); 77 | } catch (InterruptedException e1) { 78 | resultFuture.completeExceptionally(e1); 79 | return; 80 | } 81 | fetchResult(resultFuture, currentRetry + 1, pk); 82 | return; 83 | } 84 | final Row row = Row.withNames(); 85 | if (!result.isEmpty()) { 86 | columnNames.forEach(c -> row.setField(c, result.get(c))); 87 | row.setField(options.getPrimaryKey(), pk); 88 | resultFuture.complete(Collections.singletonList(row)); 89 | } else { 90 | resultFuture.complete(Collections.emptyList()); 91 | } 92 | if (cache != null) { 93 | cache.put(pk, row); 94 | } 95 | }); 96 | } 97 | 98 | @Override 99 | public void close() { 100 | connection.close(); 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/connector/redis/RedisDynamicTableFactory.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.custrom.connector.redis; 2 | 3 | import org.apache.flink.configuration.ConfigOption; 4 | import org.apache.flink.shaded.guava30.com.google.common.collect.Sets; 5 | import org.apache.flink.table.catalog.ResolvedSchema; 6 | import org.apache.flink.table.connector.source.DynamicTableSource; 7 | import org.apache.flink.table.factories.DynamicTableSourceFactory; 8 | import org.apache.flink.table.factories.FactoryUtil; 9 | 10 | import java.util.Collections; 11 | import java.util.NoSuchElementException; 12 | import java.util.Set; 13 | 14 | import static com.hiscat.flink.custrom.connector.redis.RedisOptions.*; 15 | 16 | 17 | public class RedisDynamicTableFactory implements DynamicTableSourceFactory { 18 | 19 | @Override 20 | public DynamicTableSource createDynamicTableSource(final Context context) { 21 | final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context); 22 | 23 | helper.validate(); 24 | 25 | final ResolvedSchema schema = context.getCatalogTable().getResolvedSchema(); 26 | final RedisOptions redisOptions = RedisOptions.builder() 27 | .host(helper.getOptions().get(HOST)) 28 | .port(helper.getOptions().get(PORT)) 29 | .password(helper.getOptions().get(PASSWORD)) 30 | .async(helper.getOptions().get(ASYNC)) 31 | .cacheSize(helper.getOptions().get(CACHE_SIZE)) 32 | .cacheExpireMs(helper.getOptions().get(CACHE_EXPIRE)) 33 | .delimiter(helper.getOptions().get(DELIMITER)) 34 | .maxRetryTimes(helper.getOptions().get(MAX_RETRY_TIMES)) 35 | .primaryKey(helper.getOptions().getOptional(PRIMARY_KEY) 36 | .orElse(schema.getPrimaryKey() 37 | .orElseThrow(() -> new NoSuchElementException("missing primary key")).getColumns().get(0))) 38 | .tableName(helper.getOptions().getOptional(TABLE_NAME).orElse(context.getObjectIdentifier().getObjectName())) 39 | .build(); 40 | 41 | return new RedisLookupTableSource(redisOptions, schema.getColumnNames()); 42 | } 43 | 44 | @Override 45 | public String factoryIdentifier() { 46 | return "redis"; 47 | } 48 | 49 | @Override 50 | public Set> requiredOptions() { 51 | return Collections.emptySet(); 52 | } 53 | 54 | @Override 55 | public Set> optionalOptions() { 56 | return Sets.newHashSet( 57 | HOST, 58 | PORT, 59 | PASSWORD, 60 | ASYNC, 61 | CACHE_SIZE, 62 | CACHE_EXPIRE, 63 | DELIMITER, 64 | TABLE_NAME, 65 | PRIMARY_KEY 66 | ); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/connector/redis/RedisLookupFunction.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.custrom.connector.redis; 2 | 3 | import io.lettuce.core.api.StatefulRedisConnection; 4 | import io.lettuce.core.api.sync.RedisCommands; 5 | import lombok.extern.slf4j.Slf4j; 6 | import org.apache.flink.shaded.guava30.com.google.common.cache.Cache; 7 | import org.apache.flink.shaded.guava30.com.google.common.cache.CacheBuilder; 8 | import org.apache.flink.table.functions.FunctionContext; 9 | import org.apache.flink.table.functions.TableFunction; 10 | import org.apache.flink.types.Row; 11 | 12 | import java.util.List; 13 | import java.util.Map; 14 | import java.util.concurrent.TimeUnit; 15 | 16 | @Slf4j 17 | public class RedisLookupFunction extends TableFunction { 18 | 19 | private final RedisOptions options; 20 | private final List columnNames; 21 | private Cache cache; 22 | private RedisCommands commands; 23 | private StatefulRedisConnection connection; 24 | 25 | public RedisLookupFunction(final RedisOptions options, final List columnNames) { 26 | this.options = options; 27 | this.columnNames = columnNames; 28 | } 29 | 30 | @Override 31 | public void open(final FunctionContext context) { 32 | connection = ConnectionProvider.getConnection(options); 33 | commands = this.connection.sync(); 34 | 35 | this.cache = 36 | options.getCacheSize() == -1 || options.getCacheExpireMs() == -1 37 | ? null 38 | : CacheBuilder.newBuilder() 39 | .expireAfterWrite(options.getCacheExpireMs(), TimeUnit.MILLISECONDS) 40 | .maximumSize(options.getCacheSize()) 41 | .build(); 42 | 43 | } 44 | 45 | @SuppressWarnings("unused") 46 | public void eval(String pk) { 47 | final String redisKey = String.format("%s%s%s", options.getTableName(), options.getDelimiter(), pk); 48 | if (cache != null) { 49 | Row cachedRows = cache.getIfPresent(redisKey); 50 | if (cachedRows != null && cachedRows.getArity() > 0) { 51 | collect(cachedRows); 52 | return; 53 | } 54 | } 55 | 56 | 57 | for (int retry = 0; retry <= options.getMaxRetryTimes(); retry++) { 58 | try { 59 | final Map map = commands.hgetall(redisKey); 60 | final Row row = Row.withNames(); 61 | if (!map.isEmpty()) { 62 | columnNames.forEach(c -> row.setField(c, map.get(c))); 63 | row.setField(options.getPrimaryKey(), pk); 64 | collect(row); 65 | } 66 | if (cache != null) { 67 | cache.put(redisKey, row); 68 | } 69 | break; 70 | } catch (Exception e) { 71 | log.error("lookup redis error : {}", e.getMessage(), e); 72 | try { 73 | //noinspection BusyWait 74 | Thread.sleep(1000L*retry); 75 | } catch (InterruptedException e1) { 76 | throw new RuntimeException(e1); 77 | } 78 | } 79 | } 80 | } 81 | 82 | @Override 83 | public void close() { 84 | connection.close(); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/connector/redis/RedisLookupTableSource.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.custrom.connector.redis; 2 | 3 | import org.apache.flink.table.connector.source.AsyncTableFunctionProvider; 4 | import org.apache.flink.table.connector.source.DynamicTableSource; 5 | import org.apache.flink.table.connector.source.LookupTableSource; 6 | import org.apache.flink.table.connector.source.TableFunctionProvider; 7 | 8 | import java.util.List; 9 | 10 | public class RedisLookupTableSource implements LookupTableSource { 11 | 12 | private final RedisOptions options; 13 | private final List columnNames; 14 | 15 | public RedisLookupTableSource(final RedisOptions options, final List columnNames) { 16 | this.options = options; 17 | this.columnNames = columnNames; 18 | } 19 | 20 | @Override 21 | public LookupRuntimeProvider getLookupRuntimeProvider(final LookupContext context) { 22 | if (options.getAsync()) { 23 | return AsyncTableFunctionProvider.of(new RedisAsyncLookupFunction(options, columnNames)); 24 | } 25 | return TableFunctionProvider.of(new RedisLookupFunction(options, columnNames)); 26 | } 27 | 28 | @Override 29 | public DynamicTableSource copy() { 30 | return new RedisLookupTableSource(options, columnNames); 31 | } 32 | 33 | @Override 34 | public String asSummaryString() { 35 | return "redis dynamic table"; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/connector/redis/RedisOptions.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.custrom.connector.redis; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | import org.apache.flink.configuration.ConfigOption; 8 | import org.apache.flink.configuration.ConfigOptions; 9 | 10 | import java.io.Serializable; 11 | 12 | @Data 13 | @AllArgsConstructor 14 | @NoArgsConstructor 15 | @Builder 16 | public class RedisOptions implements Serializable { 17 | 18 | public static final ConfigOption HOST = ConfigOptions.key("host").stringType().defaultValue("localhost"); 19 | public static final ConfigOption PORT = ConfigOptions.key("port").intType().defaultValue(6379); 20 | public static final ConfigOption PASSWORD = ConfigOptions.key("password").stringType().defaultValue(null); 21 | public static final ConfigOption DELIMITER = ConfigOptions.key("delimiter").stringType().defaultValue(":"); 22 | public static final ConfigOption ASYNC = ConfigOptions.key("async").booleanType().defaultValue(false); 23 | public static final ConfigOption CACHE_SIZE = ConfigOptions.key("cache-size").longType().defaultValue(1000L); 24 | public static final ConfigOption CACHE_EXPIRE = ConfigOptions.key("cache-expire-ms").longType().defaultValue(10*1000L); 25 | public static final ConfigOption TABLE_NAME = ConfigOptions.key("table-name").stringType().noDefaultValue(); 26 | public static final ConfigOption PRIMARY_KEY = ConfigOptions.key("primary-key").stringType().noDefaultValue(); 27 | public static final ConfigOption MAX_RETRY_TIMES = ConfigOptions.key("max-retry-times").intType().defaultValue(3); 28 | 29 | private int maxRetryTimes; 30 | private String host; 31 | private Integer port; 32 | private String password; 33 | private String delimiter; 34 | private Boolean async; 35 | private Long cacheSize; 36 | private Long cacheExpireMs; 37 | private String tableName; 38 | private String primaryKey; 39 | 40 | } 41 | 42 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/format/json/JsonRowDataDeserializationSchema.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.custrom.format.json; 2 | 3 | import org.apache.flink.annotation.Internal; 4 | import org.apache.flink.api.common.serialization.DeserializationSchema; 5 | import org.apache.flink.api.common.typeinfo.TypeInformation; 6 | import org.apache.flink.formats.common.TimestampFormat; 7 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.json.JsonReadFeature; 8 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationFeature; 9 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; 10 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; 11 | import org.apache.flink.table.data.RowData; 12 | import org.apache.flink.table.types.logical.DecimalType; 13 | import org.apache.flink.table.types.logical.RowType; 14 | import org.apache.flink.table.types.logical.utils.LogicalTypeChecks; 15 | 16 | import javax.annotation.Nullable; 17 | import java.io.IOException; 18 | import java.util.Objects; 19 | 20 | import static java.lang.String.format; 21 | import static org.apache.flink.util.Preconditions.checkNotNull; 22 | 23 | /** 24 | * Deserialization schema from JSON to Flink Table/SQL internal data structure {@link RowData}. 25 | * 26 | *

Deserializes a byte[] message as a JSON object and reads the specified fields. 27 | * 28 | *

Failures during deserialization are forwarded as wrapped IOExceptions. 29 | */ 30 | @Internal 31 | public class JsonRowDataDeserializationSchema implements DeserializationSchema { 32 | private static final long serialVersionUID = 1L; 33 | 34 | /** 35 | * Flag indicating whether to fail if a field is missing. 36 | */ 37 | private final boolean failOnMissingField; 38 | 39 | /** 40 | * Flag indicating whether to ignore invalid fields/rows (default: throw an exception). 41 | */ 42 | private final boolean ignoreParseErrors; 43 | 44 | /** 45 | * TypeInformation of the produced {@link RowData}. 46 | */ 47 | private final TypeInformation resultTypeInfo; 48 | 49 | /** 50 | * Runtime converter that converts {@link JsonNode}s into objects of Flink SQL internal data 51 | * structures. 52 | */ 53 | private final JsonToRowDataConverters.JsonToRowDataConverter runtimeConverter; 54 | 55 | /** 56 | * Object mapper for parsing the JSON. 57 | */ 58 | private final ObjectMapper objectMapper = new ObjectMapper(); 59 | 60 | /** 61 | * Timestamp format specification which is used to parse timestamp. 62 | */ 63 | private final TimestampFormat timestampFormat; 64 | 65 | public JsonRowDataDeserializationSchema( 66 | RowType rowType, 67 | TypeInformation resultTypeInfo, 68 | boolean failOnMissingField, 69 | boolean ignoreParseErrors, 70 | TimestampFormat timestampFormat) { 71 | if (ignoreParseErrors && failOnMissingField) { 72 | throw new IllegalArgumentException( 73 | "JSON format doesn't support failOnMissingField and ignoreParseErrors are both enabled."); 74 | } 75 | this.resultTypeInfo = checkNotNull(resultTypeInfo); 76 | this.failOnMissingField = failOnMissingField; 77 | this.ignoreParseErrors = ignoreParseErrors; 78 | this.runtimeConverter = 79 | new JsonToRowDataConverters(failOnMissingField, ignoreParseErrors, timestampFormat) 80 | .createConverter(checkNotNull(rowType)); 81 | this.timestampFormat = timestampFormat; 82 | boolean hasDecimalType = 83 | LogicalTypeChecks.hasNested(rowType, t -> t instanceof DecimalType); 84 | if (hasDecimalType) { 85 | objectMapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); 86 | } 87 | objectMapper.configure(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature(), true); 88 | } 89 | 90 | @Override 91 | public RowData deserialize(@Nullable byte[] message) throws IOException { 92 | if (message == null) { 93 | return null; 94 | } 95 | try { 96 | return convertToRowData(deserializeToJsonNode(message)); 97 | } catch (Throwable t) { 98 | if (ignoreParseErrors) { 99 | return null; 100 | } 101 | throw new IOException( 102 | format("Failed to deserialize JSON '%s'.", new String(message)), t); 103 | } 104 | } 105 | 106 | public JsonNode deserializeToJsonNode(byte[] message) throws IOException { 107 | return objectMapper.readTree(message); 108 | } 109 | 110 | public RowData convertToRowData(JsonNode message) { 111 | return (RowData) runtimeConverter.convert(message); 112 | } 113 | 114 | @Override 115 | public boolean isEndOfStream(RowData nextElement) { 116 | return false; 117 | } 118 | 119 | @Override 120 | public TypeInformation getProducedType() { 121 | return resultTypeInfo; 122 | } 123 | 124 | @Override 125 | public boolean equals(Object o) { 126 | if (this == o) { 127 | return true; 128 | } 129 | if (o == null || getClass() != o.getClass()) { 130 | return false; 131 | } 132 | JsonRowDataDeserializationSchema that = (JsonRowDataDeserializationSchema) o; 133 | return failOnMissingField == that.failOnMissingField 134 | && ignoreParseErrors == that.ignoreParseErrors 135 | && resultTypeInfo.equals(that.resultTypeInfo) 136 | && timestampFormat.equals(that.timestampFormat); 137 | } 138 | 139 | @Override 140 | public int hashCode() { 141 | return Objects.hash(failOnMissingField, ignoreParseErrors, resultTypeInfo, timestampFormat); 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/format/json/JsonRowDataSerializationSchema.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.custrom.format.json; 2 | 3 | import org.apache.flink.annotation.Internal; 4 | import org.apache.flink.api.common.serialization.SerializationSchema; 5 | import org.apache.flink.formats.common.TimestampFormat; 6 | import org.apache.flink.formats.json.JsonFormatOptions; 7 | import org.apache.flink.formats.json.JsonRowDataDeserializationSchema; 8 | import org.apache.flink.formats.json.RowDataToJsonConverters; 9 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator; 10 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; 11 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; 12 | import org.apache.flink.table.data.RowData; 13 | import org.apache.flink.table.types.logical.RowType; 14 | 15 | import java.util.Objects; 16 | 17 | /** 18 | * Serialization schema that serializes an object of Flink internal data structure into a JSON 19 | * bytes. 20 | * 21 | *

Serializes the input Flink object into a JSON string and converts it into byte[]. 22 | * 23 | *

Result byte[] messages can be deserialized using {@link 24 | * JsonRowDataDeserializationSchema}. 25 | */ 26 | @Internal 27 | public class JsonRowDataSerializationSchema implements SerializationSchema { 28 | private static final long serialVersionUID = 1L; 29 | 30 | /** RowType to generate the runtime converter. */ 31 | private final RowType rowType; 32 | 33 | /** The converter that converts internal data formats to JsonNode. */ 34 | private final RowDataToJsonConverters.RowDataToJsonConverter runtimeConverter; 35 | 36 | /** Object mapper that is used to create output JSON objects. */ 37 | private final ObjectMapper mapper = new ObjectMapper(); 38 | 39 | /** Reusable object node. */ 40 | private transient ObjectNode node; 41 | 42 | /** Timestamp format specification which is used to parse timestamp. */ 43 | private final TimestampFormat timestampFormat; 44 | 45 | /** The handling mode when serializing null keys for map data. */ 46 | private final JsonFormatOptions.MapNullKeyMode mapNullKeyMode; 47 | 48 | /** The string literal when handling mode for map null key LITERAL. */ 49 | private final String mapNullKeyLiteral; 50 | 51 | /** Flag indicating whether to serialize all decimals as plain numbers. */ 52 | private final boolean encodeDecimalAsPlainNumber; 53 | 54 | public JsonRowDataSerializationSchema( 55 | RowType rowType, 56 | TimestampFormat timestampFormat, 57 | JsonFormatOptions.MapNullKeyMode mapNullKeyMode, 58 | String mapNullKeyLiteral, 59 | boolean encodeDecimalAsPlainNumber) { 60 | this.rowType = rowType; 61 | this.timestampFormat = timestampFormat; 62 | this.mapNullKeyMode = mapNullKeyMode; 63 | this.mapNullKeyLiteral = mapNullKeyLiteral; 64 | this.encodeDecimalAsPlainNumber = encodeDecimalAsPlainNumber; 65 | this.runtimeConverter = 66 | new RowDataToJsonConverters(timestampFormat, mapNullKeyMode, mapNullKeyLiteral) 67 | .createConverter(rowType); 68 | 69 | mapper.configure( 70 | JsonGenerator.Feature.WRITE_BIGDECIMAL_AS_PLAIN, encodeDecimalAsPlainNumber); 71 | } 72 | 73 | @Override 74 | public byte[] serialize(RowData row) { 75 | if (node == null) { 76 | node = mapper.createObjectNode(); 77 | } 78 | 79 | try { 80 | runtimeConverter.convert(mapper, node, row); 81 | return mapper.writeValueAsBytes(node); 82 | } catch (Throwable t) { 83 | throw new RuntimeException("Could not serialize row '" + row + "'. ", t); 84 | } 85 | } 86 | 87 | @Override 88 | public boolean equals(Object o) { 89 | if (this == o) { 90 | return true; 91 | } 92 | if (o == null || getClass() != o.getClass()) { 93 | return false; 94 | } 95 | JsonRowDataSerializationSchema that = (JsonRowDataSerializationSchema) o; 96 | return rowType.equals(that.rowType) 97 | && timestampFormat.equals(that.timestampFormat) 98 | && mapNullKeyMode.equals(that.mapNullKeyMode) 99 | && mapNullKeyLiteral.equals(that.mapNullKeyLiteral) 100 | && encodeDecimalAsPlainNumber == that.encodeDecimalAsPlainNumber; 101 | } 102 | 103 | @Override 104 | public int hashCode() { 105 | return Objects.hash( 106 | rowType, 107 | timestampFormat, 108 | mapNullKeyMode, 109 | mapNullKeyLiteral, 110 | encodeDecimalAsPlainNumber); 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/format/json/TimeFormats.java: -------------------------------------------------------------------------------- 1 | 2 | package com.hiscat.flink.custrom.format.json; 3 | 4 | import java.time.format.DateTimeFormatter; 5 | import java.time.format.DateTimeFormatterBuilder; 6 | import java.time.temporal.ChronoField; 7 | 8 | /** 9 | * Time formats and timestamp formats respecting the RFC3339 specification, ISO-8601 specification 10 | * and SQL specification. 11 | */ 12 | class TimeFormats { 13 | 14 | /** Formatter for RFC 3339-compliant string representation of a time value. */ 15 | static final DateTimeFormatter RFC3339_TIME_FORMAT = 16 | new DateTimeFormatterBuilder() 17 | .appendPattern("HH:mm:ss") 18 | .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) 19 | .appendPattern("'Z'") 20 | .toFormatter(); 21 | 22 | /** 23 | * Formatter for RFC 3339-compliant string representation of a timestamp value (with UTC 24 | * timezone). 25 | */ 26 | static final DateTimeFormatter RFC3339_TIMESTAMP_FORMAT = 27 | new DateTimeFormatterBuilder() 28 | .append(DateTimeFormatter.ISO_LOCAL_DATE) 29 | .appendLiteral('T') 30 | .append(RFC3339_TIME_FORMAT) 31 | .toFormatter(); 32 | 33 | /** Formatter for ISO8601 string representation of a timestamp value (without UTC timezone). */ 34 | static final DateTimeFormatter ISO8601_TIMESTAMP_FORMAT = DateTimeFormatter.ISO_LOCAL_DATE_TIME; 35 | 36 | /** Formatter for ISO8601 string representation of a timestamp value (with UTC timezone). */ 37 | static final DateTimeFormatter ISO8601_TIMESTAMP_WITH_LOCAL_TIMEZONE_FORMAT = 38 | new DateTimeFormatterBuilder() 39 | .append(DateTimeFormatter.ISO_LOCAL_DATE) 40 | .appendLiteral('T') 41 | .append(DateTimeFormatter.ISO_LOCAL_TIME) 42 | .appendPattern("'Z'") 43 | .toFormatter(); 44 | 45 | /** Formatter for SQL string representation of a time value. */ 46 | static final DateTimeFormatter SQL_TIME_FORMAT = 47 | new DateTimeFormatterBuilder() 48 | .appendPattern("HH:mm:ss") 49 | .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) 50 | .toFormatter(); 51 | 52 | /** Formatter for SQL string representation of a timestamp value (without UTC timezone). */ 53 | static final DateTimeFormatter SQL_TIMESTAMP_FORMAT = 54 | new DateTimeFormatterBuilder() 55 | .append(DateTimeFormatter.ISO_LOCAL_DATE) 56 | .appendLiteral(' ') 57 | .append(SQL_TIME_FORMAT) 58 | .toFormatter(); 59 | 60 | /** Formatter for SQL string representation of a timestamp value (with UTC timezone). */ 61 | static final DateTimeFormatter SQL_TIMESTAMP_WITH_LOCAL_TIMEZONE_FORMAT = 62 | new DateTimeFormatterBuilder() 63 | .append(DateTimeFormatter.ISO_LOCAL_DATE) 64 | .appendLiteral(' ') 65 | .append(SQL_TIME_FORMAT) 66 | .appendPattern("'Z'") 67 | .toFormatter(); 68 | 69 | private TimeFormats() {} 70 | } 71 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/format/json/ogg/OggJsonFormatFactory.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.custrom.format.json.ogg; 2 | 3 | import com.hiscat.flink.custrom.format.json.JsonOptions; 4 | import org.apache.flink.api.common.serialization.DeserializationSchema; 5 | import org.apache.flink.api.common.serialization.SerializationSchema; 6 | import org.apache.flink.configuration.ConfigOption; 7 | import org.apache.flink.configuration.ReadableConfig; 8 | import org.apache.flink.formats.common.TimestampFormat; 9 | import org.apache.flink.formats.json.JsonFormatOptions; 10 | import org.apache.flink.table.connector.ChangelogMode; 11 | import org.apache.flink.table.connector.format.DecodingFormat; 12 | import org.apache.flink.table.connector.format.EncodingFormat; 13 | import org.apache.flink.table.connector.sink.DynamicTableSink; 14 | import org.apache.flink.table.data.RowData; 15 | import org.apache.flink.table.factories.DeserializationFormatFactory; 16 | import org.apache.flink.table.factories.DynamicTableFactory; 17 | import org.apache.flink.table.factories.FactoryUtil; 18 | import org.apache.flink.table.factories.SerializationFormatFactory; 19 | import org.apache.flink.table.types.DataType; 20 | import org.apache.flink.table.types.logical.RowType; 21 | import org.apache.flink.types.RowKind; 22 | 23 | import java.util.Collections; 24 | import java.util.HashSet; 25 | import java.util.Set; 26 | 27 | import static com.hiscat.flink.custrom.format.json.JsonOptions.ENCODE_DECIMAL_AS_PLAIN_NUMBER; 28 | import static com.hiscat.flink.custrom.format.json.ogg.OggJsonOptions.*; 29 | 30 | /** 31 | * Format factory for providing configured instances of OGG JSON to RowData {@link 32 | * DeserializationSchema}. 33 | */ 34 | public class OggJsonFormatFactory 35 | implements DeserializationFormatFactory, SerializationFormatFactory { 36 | 37 | public static final String IDENTIFIER = "ogg-json"; 38 | 39 | @Override 40 | public DecodingFormat> createDecodingFormat( 41 | DynamicTableFactory.Context context, ReadableConfig formatOptions) { 42 | FactoryUtil.validateFactoryOptions(this, formatOptions); 43 | validateDecodingFormatOptions(formatOptions); 44 | 45 | final String table = formatOptions.getOptional(TABLE_INCLUDE).orElse(null); 46 | final String type = formatOptions.getOptional(OP_TYPE_INCLUDE).orElse(null); 47 | final boolean ignoreParseErrors = formatOptions.get(IGNORE_PARSE_ERRORS); 48 | final TimestampFormat timestampFormat = JsonOptions.getTimestampFormat(formatOptions); 49 | 50 | return new OggJsonDecodingFormat(table, type, ignoreParseErrors, timestampFormat); 51 | } 52 | 53 | @Override 54 | public EncodingFormat> createEncodingFormat( 55 | DynamicTableFactory.Context context, ReadableConfig formatOptions) { 56 | 57 | FactoryUtil.validateFactoryOptions(this, formatOptions); 58 | validateEncodingFormatOptions(formatOptions); 59 | 60 | TimestampFormat timestampFormat = JsonOptions.getTimestampFormat(formatOptions); 61 | 62 | final JsonFormatOptions.MapNullKeyMode mapNullKeyMode = JsonFormatOptions.MapNullKeyMode.valueOf(formatOptions.get(JSON_MAP_NULL_KEY_LITERAL)); 63 | String mapNullKeyLiteral = formatOptions.get(JSON_MAP_NULL_KEY_LITERAL); 64 | 65 | final boolean encodeDecimalAsPlainNumber = formatOptions.get(ENCODE_DECIMAL_AS_PLAIN_NUMBER); 66 | 67 | return new EncodingFormat>() { 68 | @Override 69 | public ChangelogMode getChangelogMode() { 70 | return ChangelogMode.newBuilder() 71 | .addContainedKind(RowKind.INSERT) 72 | .addContainedKind(RowKind.UPDATE_BEFORE) 73 | .addContainedKind(RowKind.UPDATE_AFTER) 74 | .addContainedKind(RowKind.DELETE) 75 | .build(); 76 | } 77 | 78 | @Override 79 | public SerializationSchema createRuntimeEncoder( 80 | DynamicTableSink.Context context, DataType consumedDataType) { 81 | final RowType rowType = (RowType) consumedDataType.getLogicalType(); 82 | 83 | return new OggJsonSerializationSchema( 84 | rowType, 85 | timestampFormat, 86 | mapNullKeyMode, 87 | mapNullKeyLiteral, 88 | encodeDecimalAsPlainNumber); 89 | } 90 | }; 91 | } 92 | 93 | @Override 94 | public String factoryIdentifier() { 95 | return IDENTIFIER; 96 | } 97 | 98 | @Override 99 | public Set> requiredOptions() { 100 | return Collections.emptySet(); 101 | } 102 | 103 | @Override 104 | public Set> optionalOptions() { 105 | Set> options = new HashSet<>(); 106 | options.add(IGNORE_PARSE_ERRORS); 107 | options.add(TIMESTAMP_FORMAT); 108 | options.add(TABLE_INCLUDE); 109 | options.add(OP_TYPE_INCLUDE); 110 | options.add(JSON_MAP_NULL_KEY_MODE); 111 | options.add(JSON_MAP_NULL_KEY_LITERAL); 112 | options.add(ENCODE_DECIMAL_AS_PLAIN_NUMBER); 113 | return options; 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/format/json/ogg/OggJsonOptions.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.custrom.format.json.ogg; 2 | 3 | import com.hiscat.flink.custrom.format.json.JsonOptions; 4 | import org.apache.flink.configuration.ConfigOption; 5 | import org.apache.flink.configuration.ConfigOptions; 6 | import org.apache.flink.configuration.ReadableConfig; 7 | 8 | /** Option utils for canal-json format. */ 9 | public class OggJsonOptions { 10 | 11 | public static final ConfigOption IGNORE_PARSE_ERRORS = JsonOptions.IGNORE_PARSE_ERRORS; 12 | 13 | public static final ConfigOption TIMESTAMP_FORMAT = JsonOptions.TIMESTAMP_FORMAT; 14 | 15 | public static final ConfigOption JSON_MAP_NULL_KEY_MODE = JsonOptions.MAP_NULL_KEY_MODE; 16 | 17 | public static final ConfigOption JSON_MAP_NULL_KEY_LITERAL = 18 | JsonOptions.MAP_NULL_KEY_LITERAL; 19 | 20 | public static final ConfigOption TABLE_INCLUDE = 21 | ConfigOptions.key("table.include") 22 | .stringType() 23 | .noDefaultValue() 24 | .withDescription( 25 | "An optional regular expression to only read the specific tables changelog rows by regular matching the \"table\" meta field in the Canal record." 26 | + "The pattern string is compatible with Java's Pattern."); 27 | 28 | 29 | public static final ConfigOption OP_TYPE_INCLUDE = 30 | ConfigOptions.key("op-type.include") 31 | .stringType() 32 | .noDefaultValue() 33 | .withDescription("dml operation type values is I(INSERT),U(UPDATE),D(DELETE),T(TRUNCATE)"); 34 | 35 | // -------------------------------------------------------------------------------------------- 36 | // Validation 37 | // -------------------------------------------------------------------------------------------- 38 | 39 | /** Validator for canal decoding format. */ 40 | public static void validateDecodingFormatOptions(ReadableConfig tableOptions) { 41 | JsonOptions.validateDecodingFormatOptions(tableOptions); 42 | } 43 | 44 | /** Validator for canal encoding format. */ 45 | public static void validateEncodingFormatOptions(ReadableConfig tableOptions) { 46 | JsonOptions.validateEncodingFormatOptions(tableOptions); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/java/com/hiscat/flink/custrom/format/json/ogg/OggJsonSerializationSchema.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.custrom.format.json.ogg; 2 | 3 | import com.hiscat.flink.custrom.format.json.JsonRowDataSerializationSchema; 4 | import org.apache.flink.api.common.serialization.SerializationSchema; 5 | import org.apache.flink.formats.common.TimestampFormat; 6 | import org.apache.flink.formats.json.JsonFormatOptions; 7 | import org.apache.flink.table.api.DataTypes; 8 | import org.apache.flink.table.data.*; 9 | import org.apache.flink.table.types.DataType; 10 | import org.apache.flink.table.types.logical.RowType; 11 | import org.apache.flink.types.RowKind; 12 | 13 | import java.util.Objects; 14 | 15 | import static org.apache.flink.table.types.utils.TypeConversions.fromLogicalToDataType; 16 | 17 | /** 18 | * Serialization schema that serializes an object of Flink Table/SQL internal data structure {@link 19 | * RowData} into a Canal JSON bytes. 20 | * 21 | * @see Alibaba Canal 22 | */ 23 | public class OggJsonSerializationSchema implements SerializationSchema { 24 | 25 | private static final long serialVersionUID = 1L; 26 | 27 | private static final StringData OP_INSERT = StringData.fromString("I"); 28 | private static final StringData OP_DELETE = StringData.fromString("D"); 29 | 30 | private transient GenericRowData reuse; 31 | 32 | /** 33 | * The serializer to serialize Canal JSON data. 34 | */ 35 | private final JsonRowDataSerializationSchema jsonSerializer; 36 | 37 | public OggJsonSerializationSchema( 38 | RowType rowType, 39 | TimestampFormat timestampFormat, 40 | JsonFormatOptions.MapNullKeyMode mapNullKeyMode, 41 | String mapNullKeyLiteral, 42 | boolean encodeDecimalAsPlainNumber) { 43 | jsonSerializer = 44 | new JsonRowDataSerializationSchema( 45 | createJsonRowType(fromLogicalToDataType(rowType)), 46 | timestampFormat, 47 | mapNullKeyMode, 48 | mapNullKeyLiteral, 49 | encodeDecimalAsPlainNumber); 50 | } 51 | 52 | @Override 53 | public void open(InitializationContext context) { 54 | reuse = new GenericRowData(2); 55 | } 56 | 57 | @Override 58 | public byte[] serialize(RowData row) { 59 | try { 60 | StringData opType = rowKind2String(row.getRowKind()); 61 | ArrayData arrayData = new GenericArrayData(new RowData[]{row}); 62 | reuse.setField(0, arrayData); 63 | reuse.setField(1, opType); 64 | return jsonSerializer.serialize(reuse); 65 | } catch (Throwable t) { 66 | throw new RuntimeException("Could not serialize row '" + row + "'.", t); 67 | } 68 | } 69 | 70 | private StringData rowKind2String(RowKind rowKind) { 71 | switch (rowKind) { 72 | case INSERT: 73 | case UPDATE_AFTER: 74 | return OP_INSERT; 75 | case UPDATE_BEFORE: 76 | case DELETE: 77 | return OP_DELETE; 78 | default: 79 | throw new UnsupportedOperationException( 80 | "Unsupported operation '" + rowKind + "' for row kind."); 81 | } 82 | } 83 | 84 | @Override 85 | public boolean equals(Object o) { 86 | if (this == o) { 87 | return true; 88 | } 89 | if (o == null || getClass() != o.getClass()) { 90 | return false; 91 | } 92 | OggJsonSerializationSchema that = (OggJsonSerializationSchema) o; 93 | return Objects.equals(jsonSerializer, that.jsonSerializer); 94 | } 95 | 96 | @Override 97 | public int hashCode() { 98 | return Objects.hash(jsonSerializer); 99 | } 100 | 101 | private static RowType createJsonRowType(DataType databaseSchema) { 102 | // Canal JSON contains other information, e.g. "database", "ts" 103 | // but we don't need them 104 | // and we don't need "old" , because can not support UPDATE_BEFORE,UPDATE_AFTER 105 | return (RowType) 106 | DataTypes.ROW( 107 | DataTypes.FIELD("before", databaseSchema), 108 | DataTypes.FIELD("after", databaseSchema), 109 | DataTypes.FIELD("op_type", DataTypes.STRING())) 110 | .getLogicalType(); 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /flink-custom-connector/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory: -------------------------------------------------------------------------------- 1 | com.hiscat.flink.custrom.format.json.ogg.OggJsonFormatFactory -------------------------------------------------------------------------------- /flink-prometheus/src/main/java/com/hiscat/flink/prometheus/AutoServiceDiscoveryPrometheusReporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.hiscat.flink.prometheus; 20 | 21 | import com.hiscat.flink.prometheus.sd.ServiceDiscovery; 22 | import com.hiscat.flink.prometheus.sd.ServiceDiscoveryLoader; 23 | import com.sun.net.httpserver.HttpServer; 24 | import io.prometheus.client.exporter.HTTPServer; 25 | import org.apache.flink.annotation.PublicEvolving; 26 | import org.apache.flink.metrics.Metric; 27 | import org.apache.flink.metrics.MetricConfig; 28 | import org.apache.flink.metrics.prometheus.AbstractPrometheusReporter; 29 | import org.apache.flink.metrics.reporter.InstantiateViaFactory; 30 | import org.apache.flink.metrics.reporter.MetricReporter; 31 | 32 | import java.io.IOException; 33 | import java.lang.reflect.Field; 34 | import java.net.InetSocketAddress; 35 | import java.util.Iterator; 36 | 37 | import static com.hiscat.flink.prometheus.sd.ZookeeperServiceDiscoveryOptions.SD_IDENTIFIER; 38 | 39 | 40 | /** 41 | * {@link MetricReporter} that exports {@link Metric Metrics} via Prometheus. 42 | */ 43 | @PublicEvolving 44 | @InstantiateViaFactory( 45 | factoryClassName = "com.hiscat.flink.prometheus.AtuoServiceDiscoveryPrometheusReporterFactory") 46 | public class AutoServiceDiscoveryPrometheusReporter extends AbstractPrometheusReporter { 47 | 48 | private HTTPServer httpServer; 49 | private ServiceDiscovery sd; 50 | 51 | 52 | @Override 53 | public void open(MetricConfig config) { 54 | super.open(config); 55 | sd = ServiceDiscoveryLoader 56 | .load(config.getString(SD_IDENTIFIER.key(), SD_IDENTIFIER.defaultValue()), this.getClass().getClassLoader()); 57 | sd.register(getAddress(), config); 58 | } 59 | 60 | private InetSocketAddress getAddress() { 61 | try { 62 | Field server = this.httpServer.getClass().getDeclaredField("server"); 63 | server.setAccessible(true); 64 | return ((HttpServer) server.get(this.httpServer)).getAddress(); 65 | } catch (IllegalAccessException | NoSuchFieldException e) { 66 | throw new RuntimeException(e); 67 | } 68 | } 69 | 70 | AutoServiceDiscoveryPrometheusReporter(Iterator ports) { 71 | while (ports.hasNext()) { 72 | int port = ports.next(); 73 | try { 74 | // internally accesses CollectorRegistry.defaultRegistry 75 | httpServer = new HTTPServer(port); 76 | log.info("Started PrometheusReporter HTTP server on port {}.", port); 77 | break; 78 | } catch (IOException ioe) { // assume port conflict 79 | log.debug("Could not start PrometheusReporter HTTP server on port {}.", port, ioe); 80 | } 81 | } 82 | if (httpServer == null) { 83 | throw new RuntimeException( 84 | "Could not start PrometheusReporter HTTP server on any configured port. Ports: " 85 | + ports); 86 | } 87 | } 88 | 89 | @Override 90 | public void close() { 91 | if (httpServer != null) { 92 | httpServer.stop(); 93 | } 94 | 95 | sd.close(); 96 | 97 | super.close(); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /flink-prometheus/src/main/java/com/hiscat/flink/prometheus/AutoServiceDiscoveryPrometheusReporterFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package com.hiscat.flink.prometheus; 19 | 20 | import org.apache.flink.metrics.MetricConfig; 21 | import org.apache.flink.metrics.reporter.InterceptInstantiationViaReflection; 22 | import org.apache.flink.metrics.reporter.MetricReporterFactory; 23 | import org.apache.flink.util.NetUtils; 24 | 25 | import java.util.Iterator; 26 | import java.util.Properties; 27 | 28 | /** {@link MetricReporterFactory} for {@link AutoServiceDiscoveryPrometheusReporter}. */ 29 | @InterceptInstantiationViaReflection( 30 | reporterClassName = "com.hiscat.flink.prometheus.AtuoServiceDiscoveryPrometheusReporter") 31 | public class AutoServiceDiscoveryPrometheusReporterFactory implements MetricReporterFactory { 32 | 33 | static final String ARG_PORT = "port"; 34 | private static final String DEFAULT_PORT = "9249"; 35 | 36 | @Override 37 | public AutoServiceDiscoveryPrometheusReporter createMetricReporter(Properties properties) { 38 | MetricConfig metricConfig = (MetricConfig) properties; 39 | String portsConfig = metricConfig.getString(ARG_PORT, DEFAULT_PORT); 40 | Iterator ports = NetUtils.getPortRangeFromString(portsConfig); 41 | 42 | return new AutoServiceDiscoveryPrometheusReporter(ports); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /flink-prometheus/src/main/java/com/hiscat/flink/prometheus/sd/BaseZookeeperServiceDiscovery.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.prometheus.sd; 2 | 3 | 4 | 5 | import org.apache.flink.shaded.curator5.org.apache.curator.RetryPolicy; 6 | import org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFramework; 7 | import org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFrameworkFactory; 8 | import org.apache.flink.shaded.curator5.org.apache.curator.retry.ExponentialBackoffRetry; 9 | import org.apache.flink.shaded.zookeeper3.org.apache.zookeeper.CreateMode; 10 | import org.apache.flink.shaded.zookeeper3.org.apache.zookeeper.data.Stat; 11 | 12 | import java.net.*; 13 | import java.nio.charset.StandardCharsets; 14 | import java.util.Enumeration; 15 | import java.util.Properties; 16 | 17 | import static com.hiscat.flink.prometheus.sd.ZookeeperServiceDiscoveryOptions.ZK_QUORUM; 18 | 19 | public abstract class BaseZookeeperServiceDiscovery implements ServiceDiscovery { 20 | 21 | private CuratorFramework client; 22 | 23 | @Override 24 | public void register(InetSocketAddress address, Properties properties) { 25 | initClient(properties); 26 | registerNode(address, properties); 27 | } 28 | 29 | private void initClient(Properties properties) { 30 | RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 100); 31 | client = CuratorFrameworkFactory.newClient(properties.getProperty(ZK_QUORUM.key()), retryPolicy); 32 | client.start(); 33 | } 34 | 35 | 36 | private void registerNode(InetSocketAddress address, Properties properties) { 37 | try { 38 | String path = makePath(properties); 39 | Stat stat = client.checkExists().forPath(path); 40 | if (stat != null) { 41 | client.setData().forPath(path, makeServerSetData(address)); 42 | return; 43 | } 44 | 45 | client 46 | .create() 47 | .creatingParentsIfNeeded() 48 | .withMode(getCreateMode()) 49 | .forPath(path, makeServerSetData(address)); 50 | } catch (Exception e) { 51 | throw new RuntimeException(e); 52 | } 53 | } 54 | 55 | protected abstract String makePath(Properties properties); 56 | 57 | protected abstract CreateMode getCreateMode(); 58 | 59 | private byte[] makeServerSetData(InetSocketAddress address) { 60 | String jsonFormat = "{\"serviceEndpoint\":{\"host\":\"%s\",\"port\":%d},\"additionalEndpoints\":{},\"status\":\"ALIVE\"}\n"; 61 | try { 62 | return String.format(jsonFormat, getIpAddress(), 63 | address.getPort()).getBytes(StandardCharsets.UTF_8); 64 | } catch (SocketException | UnknownHostException e) { 65 | throw new RuntimeException(e); 66 | } 67 | } 68 | 69 | private String getIpAddress() throws SocketException, UnknownHostException { 70 | Enumeration enumeration = NetworkInterface.getNetworkInterfaces(); 71 | while (enumeration.hasMoreElements()) { 72 | NetworkInterface network = enumeration.nextElement(); 73 | if (network.isVirtual() || !network.isUp()) { 74 | continue; 75 | } 76 | Enumeration addresses = network.getInetAddresses(); 77 | while (addresses.hasMoreElements()) { 78 | InetAddress address = addresses.nextElement(); 79 | if (address.isLoopbackAddress()) { 80 | continue; 81 | } 82 | if (address.isSiteLocalAddress()) { 83 | return address.getHostAddress(); 84 | } 85 | 86 | } 87 | } 88 | return InetAddress.getLocalHost().getHostAddress(); 89 | } 90 | 91 | @Override 92 | public void close() { 93 | client.close(); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /flink-prometheus/src/main/java/com/hiscat/flink/prometheus/sd/JobManagerZookeeperServiceDiscovery.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.prometheus.sd; 2 | 3 | 4 | 5 | import org.apache.flink.shaded.zookeeper3.org.apache.zookeeper.CreateMode; 6 | 7 | import java.util.Properties; 8 | 9 | import static com.hiscat.flink.prometheus.sd.ZookeeperServiceDiscoveryOptions.JM_PATH; 10 | import static com.hiscat.flink.prometheus.sd.ZookeeperServiceDiscoveryOptions.JOB_NAME; 11 | 12 | 13 | public class JobManagerZookeeperServiceDiscovery extends BaseZookeeperServiceDiscovery{ 14 | @Override 15 | protected String makePath(Properties properties) { 16 | return properties.getProperty(JM_PATH.key()) + "/" + System.getProperty(JOB_NAME.key()); 17 | } 18 | 19 | @Override 20 | protected CreateMode getCreateMode() { 21 | return CreateMode.EPHEMERAL; 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /flink-prometheus/src/main/java/com/hiscat/flink/prometheus/sd/ServiceDiscovery.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.prometheus.sd; 2 | 3 | import java.net.InetSocketAddress; 4 | import java.util.Properties; 5 | 6 | public interface ServiceDiscovery { 7 | void register(InetSocketAddress address, Properties properties); 8 | 9 | void close(); 10 | } 11 | -------------------------------------------------------------------------------- /flink-prometheus/src/main/java/com/hiscat/flink/prometheus/sd/ServiceDiscoveryFactory.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.prometheus.sd; 2 | 3 | public interface ServiceDiscoveryFactory { 4 | String identifier(); 5 | 6 | ServiceDiscovery create(); 7 | } 8 | -------------------------------------------------------------------------------- /flink-prometheus/src/main/java/com/hiscat/flink/prometheus/sd/ServiceDiscoveryLoader.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.prometheus.sd; 2 | 3 | import java.util.LinkedList; 4 | import java.util.List; 5 | import java.util.ServiceConfigurationError; 6 | import java.util.ServiceLoader; 7 | import java.util.stream.Collectors; 8 | 9 | import static java.util.stream.Collectors.toList; 10 | 11 | public class ServiceDiscoveryLoader { 12 | public static ServiceDiscovery load(String identifier, ClassLoader classLoader) { 13 | List factories = discoverFactories(classLoader); 14 | checkNotFoundFactory(factories); 15 | List matches = factories.stream().filter(sd -> sd.identifier().equals(identifier)).collect(toList()); 16 | checkMatchesFactories(identifier, factories, matches); 17 | return matches.get(0).create(); 18 | } 19 | 20 | private static List discoverFactories(ClassLoader classLoader) { 21 | try { 22 | final List result = new LinkedList<>(); 23 | ServiceLoader.load(ServiceDiscoveryFactory.class, classLoader) 24 | .iterator() 25 | .forEachRemaining(result::add); 26 | return result; 27 | } catch (ServiceConfigurationError e) { 28 | throw new RuntimeException( 29 | "Could not load service provider for service discovery factory.", e); 30 | } 31 | } 32 | 33 | private static void checkNotFoundFactory(List factories) { 34 | if (factories.isEmpty()) { 35 | throw new IllegalStateException( 36 | String.format( 37 | "Could not find any service discovery factories that implement '%s' in the classpath.", 38 | ServiceDiscoveryFactory.class.getName())); 39 | } 40 | } 41 | 42 | private static void checkMatchesFactories(String identifier, List factories, List matches) { 43 | checkNotFoundMatchedFactory(identifier, factories, matches); 44 | 45 | checkFindMultipleFactories(identifier, matches); 46 | } 47 | 48 | private static void checkNotFoundMatchedFactory(String identifier, List factories, List matches) { 49 | if (matches.isEmpty()) { 50 | throw new IllegalStateException( 51 | String.format( 52 | "Could not find any service discovery factory that can handle identifier '%s' that implements '%s' in the classpath.\n\n" 53 | + "Available factories are:\n\n" 54 | + "%s", 55 | identifier, 56 | ServiceDiscoveryFactory.class.getName(), 57 | factories.stream() 58 | .map(f -> f.getClass().getName()) 59 | .distinct() 60 | .sorted() 61 | .collect(Collectors.joining("\n")))); 62 | } 63 | } 64 | 65 | private static void checkFindMultipleFactories(String identifier, List matches) { 66 | if (matches.size() > 1) { 67 | throw new IllegalStateException( 68 | String.format( 69 | "Multiple service discovery factories can handle identifier '%s' that implement '%s' found in the classpath.\n\n" 70 | + "Ambiguous factory classes are:\n\n" 71 | + "%s", 72 | identifier, 73 | ServiceDiscoveryFactory.class.getName(), 74 | matches.stream() 75 | .map(f -> f.getClass().getName()) 76 | .sorted() 77 | .collect(Collectors.joining("\n")))); 78 | } 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /flink-prometheus/src/main/java/com/hiscat/flink/prometheus/sd/TaskManagerZookeeperServiceDiscovery.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.prometheus.sd; 2 | 3 | import org.apache.flink.shaded.zookeeper3.org.apache.zookeeper.CreateMode; 4 | 5 | import java.util.Properties; 6 | 7 | import static com.hiscat.flink.prometheus.sd.ZookeeperServiceDiscoveryOptions.*; 8 | 9 | public class TaskManagerZookeeperServiceDiscovery extends BaseZookeeperServiceDiscovery { 10 | 11 | 12 | @Override 13 | protected String makePath(Properties properties) { 14 | return properties.getProperty(TM_PATH.key()) + "/" 15 | + System.getProperty(JOB_NAME.key()) 16 | + "_" 17 | + System.getProperty(CONTAINER_ID.key()); 18 | } 19 | 20 | @Override 21 | protected CreateMode getCreateMode() { 22 | return CreateMode.EPHEMERAL_SEQUENTIAL; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /flink-prometheus/src/main/java/com/hiscat/flink/prometheus/sd/ZookeeperServiceDiscoveryFactory.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.prometheus.sd; 2 | 3 | import static com.hiscat.flink.prometheus.sd.ZookeeperServiceDiscoveryOptions.ROLE; 4 | 5 | public class ZookeeperServiceDiscoveryFactory implements ServiceDiscoveryFactory { 6 | @Override 7 | public String identifier() { 8 | return "zookeeper"; 9 | } 10 | 11 | @Override 12 | public ServiceDiscovery create() { 13 | if (currentNodeIsJobManager()) { 14 | return new JobManagerZookeeperServiceDiscovery(); 15 | } 16 | return new TaskManagerZookeeperServiceDiscovery(); 17 | } 18 | 19 | private static boolean currentNodeIsJobManager() { 20 | return "jm".equals(System.getProperty(ROLE.key())); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /flink-prometheus/src/main/java/com/hiscat/flink/prometheus/sd/ZookeeperServiceDiscoveryOptions.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.prometheus.sd; 2 | 3 | import org.apache.flink.configuration.ConfigOption; 4 | import org.apache.flink.configuration.ConfigOptions; 5 | 6 | public class ZookeeperServiceDiscoveryOptions { 7 | public static final ConfigOption TM_PATH = ConfigOptions.key("tm.path") 8 | .stringType() 9 | .noDefaultValue() 10 | .withDescription("zk tm path"); 11 | public static final ConfigOption JM_PATH = ConfigOptions.key("jm.path") 12 | .stringType() 13 | .noDefaultValue() 14 | .withDescription("zk jm path"); 15 | public static final ConfigOption SD_IDENTIFIER = ConfigOptions.key("sd.identifier") 16 | .stringType() 17 | .defaultValue("none") 18 | .withDescription("service discovery identifier "); 19 | 20 | public static final ConfigOption ZK_QUORUM = ConfigOptions.key("zk.quorum") 21 | .stringType() 22 | .noDefaultValue() 23 | .withDescription("service discovery zk quorum "); 24 | 25 | // ================== cli option ========================== 26 | public static final ConfigOption CONTAINER_ID = ConfigOptions.key("container_id") 27 | .stringType() 28 | .noDefaultValue() 29 | .withDescription("yarn container_id"); 30 | 31 | public static final ConfigOption ROLE = ConfigOptions.key("role") 32 | .stringType() 33 | .noDefaultValue() 34 | .withDescription("env variable role"); 35 | public static final ConfigOption JOB_NAME = ConfigOptions.key("jobName") 36 | .stringType() 37 | .noDefaultValue() 38 | .withDescription("job name "); 39 | } 40 | -------------------------------------------------------------------------------- /flink-prometheus/src/main/resources/META-INF/services/com.hiscat.flink.prometheus.sd.ServiceDiscoveryFactory: -------------------------------------------------------------------------------- 1 | com.hiscat.flink.prometheus.sd.ZookeeperServiceDiscoveryFactory -------------------------------------------------------------------------------- /flink-prometheus/src/main/resources/META-INF/services/org.apache.flink.metrics.reporter.MetricReporterFactory: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | com.hiscat.flink.prometheus.AutoServiceDiscoveryPrometheusReporterFactory 17 | -------------------------------------------------------------------------------- /flink-sql-submitter/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.hiscat.flink 8 | flink-sql-submitter 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 8 13 | 8 14 | 2.11 15 | 1.13.1 16 | 1.3.1 17 | 1.18.20 18 | 2.17.0 19 | 20 | 21 | 22 | org.apache.flink 23 | flink-clients_${scala.binary.version} 24 | provided 25 | ${flink.version} 26 | 27 | 28 | org.apache.flink 29 | flink-table-planner_${scala.binary.version} 30 | provided 31 | ${flink.version} 32 | 33 | 34 | 35 | 36 | org.apache.flink 37 | flink-sql-client_2.11 38 | ${flink.version} 39 | provided 40 | 41 | 42 | org.apache.flink 43 | flink-python_2.11 44 | ${flink.version} 45 | provided 46 | 47 | 48 | org.apache.flink 49 | flink-runtime-web_2.11 50 | ${flink.version} 51 | provided 52 | 53 | 54 | org.apache.flink 55 | flink-yarn_2.11 56 | ${flink.version} 57 | provided 58 | 59 | 60 | 61 | org.projectlombok 62 | lombok 63 | ${lombok.version} 64 | provided 65 | 66 | 67 | commons-cli 68 | commons-cli 69 | ${commons-cli.version} 70 | 71 | 72 | 73 | 74 | org.apache.logging.log4j 75 | log4j-api 76 | ${log4j.version} 77 | provided 78 | 79 | 80 | org.apache.logging.log4j 81 | log4j-core 82 | ${log4j.version} 83 | provided 84 | 85 | 86 | org.apache.logging.log4j 87 | log4j-slf4j-impl 88 | ${log4j.version} 89 | provided 90 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /flink-sql-submitter/src/main/java/com/hiscat/flink/SqlSubmitter.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink; 2 | 3 | import com.hiscat.flink.cli.CliStatementSplitter; 4 | import com.hiscat.flink.cli.SqlRunnerOptions; 5 | import com.hiscat.flink.function.CallContext; 6 | import lombok.extern.slf4j.Slf4j; 7 | import org.apache.commons.cli.ParseException; 8 | import org.apache.commons.io.IOUtils; 9 | import org.apache.flink.api.java.utils.ParameterTool; 10 | import org.apache.flink.core.fs.FileSystem; 11 | import org.apache.flink.core.fs.Path; 12 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 13 | import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; 14 | import org.apache.flink.table.api.bridge.java.internal.StreamTableEnvironmentImpl; 15 | import org.apache.flink.table.delegation.Parser; 16 | import org.apache.flink.table.operations.CatalogSinkModifyOperation; 17 | import org.apache.flink.table.operations.ModifyOperation; 18 | import org.apache.flink.table.operations.Operation; 19 | import org.apache.flink.table.operations.command.SetOperation; 20 | 21 | import java.io.IOException; 22 | import java.net.URI; 23 | import java.nio.charset.StandardCharsets; 24 | import java.util.ArrayList; 25 | import java.util.List; 26 | import java.util.function.Consumer; 27 | 28 | @Slf4j 29 | public class SqlSubmitter { 30 | 31 | public static void main(String[] args) 32 | throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException, 33 | ParseException { 34 | final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 35 | final StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); 36 | 37 | final Parser parser = ((StreamTableEnvironmentImpl) tEnv).getParser(); 38 | List dml = new ArrayList<>(); 39 | 40 | for (String stmt : 41 | CliStatementSplitter.splitContent(getSql(SqlRunnerOptions.parseFromArgs(args)))) { 42 | stmt = stmt.trim(); 43 | if (stmt.endsWith(";")) { 44 | stmt = stmt.substring(0, stmt.length() - 1).trim(); 45 | } 46 | 47 | if (stmt.trim().isEmpty()) { 48 | continue; 49 | } 50 | if (stmt.startsWith("call")) { 51 | call( 52 | stmt, 53 | CallContext.builder().args(ParameterTool.fromArgs(args)).env(env).tEnv(tEnv).build()); 54 | continue; 55 | } 56 | final Operation operation = parser.parse(stmt).get(0); 57 | if (operation instanceof SetOperation) { 58 | // SET 59 | tEnv.getConfig() 60 | .getConfiguration() 61 | .setString( 62 | ((SetOperation) operation) 63 | .getKey() 64 | .orElseThrow(() -> new RuntimeException("set key not exist")), 65 | ((SetOperation) operation) 66 | .getValue() 67 | .orElseThrow(() -> new RuntimeException("set value not exist"))); 68 | } else if (operation instanceof CatalogSinkModifyOperation) { 69 | // INSERT INTO/OVERWRITE 70 | dml.add((ModifyOperation) operation); 71 | } else { 72 | // fallback to default implementation 73 | ((StreamTableEnvironmentImpl) tEnv).executeInternal(operation); 74 | } 75 | } 76 | if (!dml.isEmpty()) { 77 | ((StreamTableEnvironmentImpl) tEnv).executeInternal(dml); 78 | } 79 | } 80 | 81 | private static String getSql(SqlRunnerOptions options) throws IOException { 82 | if (options.getSqlFile().startsWith("classpath:")) { 83 | return IOUtils.toString(options.getSqlFileInputStreamFromClasspath(), StandardCharsets.UTF_8); 84 | } 85 | URI uri = URI.create(options.getSqlFile()); 86 | FileSystem fs = FileSystem.get(uri); 87 | return IOUtils.toString(fs.open(new Path(uri)), StandardCharsets.UTF_8); 88 | } 89 | 90 | private static void call(final String stmt, final CallContext context) 91 | throws ClassNotFoundException, InstantiationException, IllegalAccessException { 92 | //noinspection unchecked 93 | ((Consumer) Class.forName(stmt.split(" ")[1]).newInstance()).accept(context); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /flink-sql-submitter/src/main/java/com/hiscat/flink/cli/CliStatementSplitter.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.cli; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.stream.Collectors; 6 | 7 | /** 8 | * Because we needn't include sql-client dependency. So we need copy this class 9 | * 10 | * Line splitter to determine whether the submitted line is complete. It also offers to split the 11 | * submitted content into multiple statements. 12 | * 13 | *

This is a simple splitter. It just split the line in context-unrelated way, e.g. it fails to 14 | * parse line "';\n'" 15 | */ 16 | public class CliStatementSplitter { 17 | 18 | private static final String MASK = "--.*$"; 19 | private static final String BEGINNING_MASK = "^(\\s)*--.*$"; 20 | 21 | public static boolean isStatementComplete(String statement) { 22 | String[] lines = statement.split("\n"); 23 | // fix input statement is "\n" 24 | if (lines.length == 0) { 25 | return false; 26 | } else { 27 | return isEndOfStatement(lines[lines.length - 1]); 28 | } 29 | } 30 | 31 | public static List splitContent(String content) { 32 | List statements = new ArrayList<>(); 33 | List buffer = new ArrayList<>(); 34 | 35 | for (String line : content.split("\n")) { 36 | if (isEndOfStatement(line)) { 37 | buffer.add(line); 38 | statements.add(normalize(buffer)); 39 | buffer.clear(); 40 | } else { 41 | buffer.add(line); 42 | } 43 | } 44 | if (!buffer.isEmpty()) { 45 | statements.add(normalize(buffer)); 46 | } 47 | return statements; 48 | } 49 | 50 | private static String normalize(List buffer) { 51 | // remove comment lines 52 | return buffer.stream() 53 | .map(statementLine -> statementLine.replaceAll(BEGINNING_MASK, "")) 54 | .collect(Collectors.joining("\n")); 55 | } 56 | 57 | private static boolean isEndOfStatement(String line) { 58 | return line.replaceAll(MASK, "").trim().endsWith(";"); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /flink-sql-submitter/src/main/java/com/hiscat/flink/cli/SqlRunnerOptions.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.cli; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | import org.apache.commons.cli.*; 8 | 9 | import java.io.InputStream; 10 | 11 | @Data 12 | @Builder 13 | @NoArgsConstructor 14 | @AllArgsConstructor 15 | public class SqlRunnerOptions { 16 | 17 | public static final Option OPTION_FILE = 18 | Option.builder("f") 19 | .required(false) 20 | .longOpt("file") 21 | .numberOfArgs(1) 22 | .argName("script file") 23 | .desc( 24 | "Script file that should be executed. In this mode, " 25 | + "the client will not open an interactive terminal.") 26 | .build(); 27 | 28 | private String sqlFile; 29 | 30 | public static SqlRunnerOptions parseFromArgs(String[] args) throws ParseException { 31 | final CommandLine cli = 32 | new DefaultParser().parse(new Options().addOption(OPTION_FILE), args, true); 33 | 34 | return SqlRunnerOptions.builder().sqlFile(cli.getOptionValue(OPTION_FILE.getOpt())).build(); 35 | } 36 | 37 | public InputStream getSqlFileInputStreamFromClasspath() { 38 | return this.getClass() 39 | .getResourceAsStream(String.format("%s", sqlFile.replace("classpath:", ""))); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /flink-sql-submitter/src/main/java/com/hiscat/flink/function/CallContext.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink.function; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | import org.apache.flink.api.java.utils.ParameterTool; 8 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 9 | import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; 10 | 11 | @Data 12 | @AllArgsConstructor 13 | @NoArgsConstructor 14 | @Builder 15 | public class CallContext { 16 | private StreamExecutionEnvironment env; 17 | private StreamTableEnvironment tEnv; 18 | private ParameterTool args; 19 | } 20 | -------------------------------------------------------------------------------- /flink-udf/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.hiscat.flink 8 | flink-udf 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 8 13 | 8 14 | 1.13.1 15 | 31.0.1-jre 16 | 17 | 18 | 19 | org.apache.flink 20 | flink-table-planner-blink_2.11 21 | ${flink.version} 22 | provided 23 | 24 | 25 | com.google.guava 26 | guava 27 | ${guava.version} 28 | provided 29 | 30 | 31 | -------------------------------------------------------------------------------- /flink-udf/src/main/java/com/hiscat/flink/Millisecond2LocalDateTimeString.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink; 2 | 3 | import java.time.Instant; 4 | import java.time.ZoneId; 5 | import org.apache.flink.table.annotation.DataTypeHint; 6 | import org.apache.flink.table.functions.ScalarFunction; 7 | 8 | @SuppressWarnings("unused") 9 | public class Millisecond2LocalDateTimeString extends ScalarFunction { 10 | 11 | public String eval(@DataTypeHint("BIGINT") Long mill ) { 12 | return Instant.ofEpochMilli(mill).atZone(ZoneId.systemDefault()).toLocalDateTime().toString(); 13 | } 14 | 15 | public static void main(String[] args) { 16 | System.out.println(Instant.ofEpochMilli(0).atZone(ZoneId.systemDefault()).toLocalDateTime()); 17 | // 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /flink-udf/src/main/java/com/hiscat/flink/TopicGetter.java: -------------------------------------------------------------------------------- 1 | package com.hiscat.flink; 2 | 3 | import static java.util.stream.Collectors.toList; 4 | 5 | import com.google.common.cache.Cache; 6 | import com.google.common.cache.CacheBuilder; 7 | import java.time.Duration; 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.regex.Pattern; 11 | import org.apache.flink.table.functions.FunctionContext; 12 | import org.apache.flink.table.functions.ScalarFunction; 13 | 14 | public class TopicGetter extends ScalarFunction { 15 | private final List patterns; 16 | private transient Cache cache; 17 | private final Map tableTopicMapping; 18 | private final long cacheSize; 19 | private final Duration cacheExpiredMs; 20 | 21 | @Override 22 | public void open(FunctionContext context) { 23 | this.cache = 24 | CacheBuilder.newBuilder() 25 | .maximumSize(this.cacheSize) 26 | .expireAfterAccess(this.cacheExpiredMs) 27 | .build(); 28 | } 29 | 30 | public TopicGetter( 31 | Map tableTopicMapping, long cacheSize, Duration cacheExpiredMs) { 32 | this.patterns = tableTopicMapping.keySet().stream().map(Pattern::compile).collect(toList()); 33 | this.tableTopicMapping = tableTopicMapping; 34 | this.cacheSize = cacheSize; 35 | this.cacheExpiredMs = cacheExpiredMs; 36 | } 37 | 38 | public String eval(String db, String table) { 39 | String identifier = String.format("%s.%s", db, table); 40 | String topic = cache.getIfPresent(identifier); 41 | if (topic != null) { 42 | if (topic.isEmpty()) { 43 | return null; 44 | } 45 | return topic; 46 | } 47 | topic = 48 | patterns.stream() 49 | .filter(p -> p.matcher(identifier).matches()) 50 | .findFirst() 51 | .map(p -> tableTopicMapping.get(p.pattern())) 52 | .orElse(""); 53 | cache.put(identifier, topic); 54 | if (topic.isEmpty()) { 55 | return null; 56 | } 57 | return topic; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /image/how-to-run-sql.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MyLanPangzi/flink-demo/369eba5cc59e02f723ee6812402104525718ab3f/image/how-to-run-sql.png -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.hiscat 8 | flink-demo 9 | 1.0-SNAPSHOT 10 | flink-demo 11 | pom 12 | 13 | flink-custom-connector 14 | flink-sql-submitter 15 | flink-prometheus 16 | 17 | 18 | 19 | 8 20 | 8 21 | 22 | 1.14.0 23 | 2.11 24 | 2.17.0 25 | 1.18.20 26 | 27 | 28 | 29 | org.projectlombok 30 | lombok 31 | ${lombok.version} 32 | 33 | 34 | 35 | 36 | 37 | org.apache.flink 38 | flink-clients_${scala.binary.version} 39 | ${flink.version} 40 | 41 | 42 | org.apache.flink 43 | flink-table-planner_${scala.binary.version} 44 | ${flink.version} 45 | 46 | 47 | 48 | org.apache.flink 49 | flink-json 50 | ${flink.version} 51 | 52 | 53 | 54 | 55 | org.apache.flink 56 | flink-sql-client_2.11 57 | ${flink.version} 58 | provided 59 | 60 | 61 | org.apache.flink 62 | flink-python_2.11 63 | ${flink.version} 64 | provided 65 | 66 | 67 | org.apache.flink 68 | flink-runtime-web_2.11 69 | ${flink.version} 70 | provided 71 | 72 | 73 | org.apache.flink 74 | flink-yarn_2.11 75 | ${flink.version} 76 | provided 77 | 78 | 79 | 80 | 81 | org.apache.flink 82 | flink-connector-kafka_2.11 83 | ${flink.version} 84 | provided 85 | 86 | 87 | org.apache.flink 88 | flink-connector-jdbc_2.11 89 | ${flink.version} 90 | provided 91 | 92 | 93 | com.ververica 94 | flink-connector-mysql-cdc 95 | ${flink-connector-mysql-cdc.version} 96 | 97 | 98 | org.apache.flink 99 | flink-connector-hive_2.11 100 | ${flink.version} 101 | provided 102 | 103 | 104 | 105 | 106 | org.apache.logging.log4j 107 | log4j-api 108 | ${log4j.version} 109 | 110 | 111 | org.apache.logging.log4j 112 | log4j-core 113 | ${log4j.version} 114 | 115 | 116 | org.apache.logging.log4j 117 | log4j-slf4j-impl 118 | ${log4j.version} 119 | 120 | 121 | 122 | --------------------------------------------------------------------------------