├── .gitignore ├── LICENSE ├── README.md ├── pom.xml └── src └── main ├── java └── org │ └── apache │ └── flink │ └── api │ └── java │ └── io │ └── redis │ ├── RedisInputFormat.java │ ├── RedisLookupFunction.java │ ├── RedisOptions.java │ ├── RedisOutFormat.java │ ├── RedisRetractTableSink.java │ ├── RedisSinkFunction.java │ ├── RedisSourceSinkFactory.java │ └── RedisTableSource.java └── resources └── META-INF └── services └── org.apache.flink.table.factories.TableFactory /.gitignore: -------------------------------------------------------------------------------- 1 | .cache 2 | scalastyle-output.xml 3 | .classpath 4 | .idea/* 5 | .metadata 6 | .settings 7 | .project 8 | .version.properties 9 | filter.properties 10 | logs.zip 11 | target 12 | tmp 13 | *.class 14 | *.iml 15 | *.swp 16 | *.jar 17 | *.log 18 | .DS_Store 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 DinoZhang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # flink-connector-redis 2 | 基于`Blink`的`Flink SQL Redis`连接器，支持`Redis`维表和结果表。 3 | 4 | [![HitCount](http://hits.dwyl.com/DinoZhang/DinoZhang/flink-connector-redis.svg)](http://hits.dwyl.com/DinoZhang/DinoZhang/flink-connector-redis) 5 | 6 | 7 | 8 | ## 安装 9 | ```shell 10 | mvn clean package -Pfast 11 | ``` 12 | 打包完成后，将`flink-jdbc-1.5.1.jar`和`jedis-2.7.2.jar`拷贝至`Fink lib`目录下。 13 | ## 使用 14 | ### 维表 15 | ```sql 16 | create table redis_dim ( 17 | k varchar, 18 | v varchar, 19 | primary key(k) 20 | ) with ( 21 | type = 'redis', 22 | host = '127.0.0.1', 23 | port = '6379', 24 | password = '' 25 | ); 26 | ``` 27 | ### 结果表 28 | 支持String类型，等价于`set key value`。 29 | ```sql 30 | create table redis_sink ( 31 | k varchar, 32 | v varchar, 33 | primary key(k) 34 | ) with ( 35 | type = 'redis', 36 | host = '127.0.0.1', 37 | port = '6379', 38 | password = '', 39 | db = '0' 40 | ); 41 | ``` 42 | ### 示例 43 | ```sql 44 | create table kafka_source ( 45 | messageKey varbinary, 46 | message varbinary, 47 | topic varchar, 48 | `partition` int, 49 | `offset` bigint 50 | ) with ( 51 | type = 'kafka010', 52 | topic = 'test', 53 | bootstrap.servers = 'localhost:9092', 54 | `group.id` = 'sql_group_1' 55 | ); 56 | 57 | create table redis_sink ( 58 | k varchar, 59 | v varchar, 60 | primary key(k) 61 | ) with ( 62 | type = 'redis', 63 | host = '127.0.0.1', 64 | port = '6379', 65 | password = '', 66 | db = '0' 67 | ); 68 | 69 | insert into redis_sink 70 | select cast(`offset` as varchar),cast(CURRENT_TIMESTAMP as varchar) from kafka_source; 71 | 72 | ``` 73 | 74 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | flink-connectors 7 | com.alibaba.blink 8 | 1.5.1 9 | 10 | 4.0.0 11 | 12 | flink-connector-redis 13 | 14 | 2.11 15 | 16 | 17 | 18 | 19 | 20 | com.alibaba.blink 21 | flink-table_${scala.binary.version} 22 | ${project.version} 23 | provided 24 | 26 | true 27 | 28 | 29 | 30 | com.alibaba.blink 31 | flink-streaming-java_${scala.binary.version} 32 | ${project.version} 33 | provided 34 | 35 | 36 | redis.clients 37 | jedis 38 | 2.7.2 39 | jar 40 | compile 41 | 42 | 43 | 44 | com.alibaba.blink 45 | flink-scala_${scala.binary.version} 46 | ${project.version} 47 | test 48 | 49 | 50 | 51 | org.apache.derby 52 | derby 53 | 10.14.2.0 54 | test 55 | 56 | 57 | 58 | commons-codec 59 | commons-codec 60 | 1.10 61 | 62 | 63 | junit 64 | junit 65 | 66 | 67 | 68 | 69 | 70 | 71 | com.alibaba.blink 72 | flink-streaming-scala_${scala.binary.version} 73 | ${project.version} 74 | test 75 | 76 | 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /src/main/java/org/apache/flink/api/java/io/redis/RedisInputFormat.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.api.java.io.redis; 2 | 3 | import org.apache.flink.api.common.io.RichInputFormat; 4 | import org.apache.flink.api.common.io.statistics.BaseStatistics; 5 | import org.apache.flink.api.common.typeinfo.TypeInformation; 6 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 7 | import org.apache.flink.api.java.typeutils.RowTypeInfo; 8 | import org.apache.flink.configuration.Configuration; 9 | import org.apache.flink.core.io.InputSplit; 10 | import org.apache.flink.core.io.InputSplitAssigner; 11 | import org.apache.flink.hive.shaded.org.apache.commons.lang3.StringUtils; 12 | import org.apache.flink.types.Row; 13 | 14 | import org.slf4j.Logger; 15 | import org.slf4j.LoggerFactory; 16 | import redis.clients.jedis.Jedis; 17 | 18 | import java.io.IOException; 19 | 20 | /** 21 | * @version $Id: RedisInputFormat.java, v 0.1 2019年11月29日 11:05 AM Exp $ 22 | */ 23 | public class RedisInputFormat extends RichInputFormat implements ResultTypeQueryable { 24 | 25 | private static final long serialVersionUID = -5714715362124636647L; 26 | private RowTypeInfo rowTypeInfo; 27 | private static Logger log = LoggerFactory.getLogger(RedisOutFormat.class); 28 | private Jedis jedis; 29 | private String host; 30 | private int port; 31 | private String password; 32 | 33 | public RedisInputFormat(final String host, final int port, final String password) { 34 | this.host = host; 35 | this.port = port; 36 | this.password = password; 37 | } 38 | 39 | @Override 40 | public void configure(final Configuration parameters) { 41 | 42 | } 43 | 44 | @Override 45 | public BaseStatistics getStatistics(final BaseStatistics cachedStatistics) throws IOException { 46 | return null; 47 | } 48 | 49 | @Override 50 | public InputSplit[] createInputSplits(final int minNumSplits) throws IOException { 51 | return new InputSplit[0]; 52 | } 53 | 54 | @Override 55 | public InputSplitAssigner getInputSplitAssigner(final InputSplit[] inputSplits) { 56 | return null; 57 | } 58 | 59 | @Override 60 | public void open(final InputSplit split) throws IOException { 61 | } 62 | 63 | @Override 64 | public boolean reachedEnd() throws IOException { 65 | return false; 66 | } 67 | 68 | @Override 69 | public Row nextRecord(final Row reuse) throws IOException { 70 | return null; 71 | } 72 | 73 | @Override 74 | public void close() throws IOException { 75 | jedis.close(); 76 | } 77 | 78 | @Override 79 | public TypeInformation getProducedType() { 80 | return rowTypeInfo; 81 | } 82 | 83 | public Jedis getRedisConnection(){ 84 | return jedis; 85 | } 86 | 87 | @Override 88 | public void openInputFormat() throws IOException { 89 | try { 90 | jedis = new Jedis(host, port); 91 | if (StringUtils.isNotBlank(password)) { 92 | jedis.auth(password); 93 | } 94 | 95 | } catch (Exception se) { 96 | throw new IllegalArgumentException("redis input open() failed." + se.getMessage(), se); 97 | } 98 | } 99 | 100 | @Override 101 | public void closeInputFormat() throws IOException { 102 | if (jedis != null){ 103 | jedis.close(); 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/main/java/org/apache/flink/api/java/io/redis/RedisLookupFunction.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.api.java.io.redis; 2 | 3 | import org.apache.flink.table.api.functions.FunctionContext; 4 | import org.apache.flink.table.api.functions.TableFunction; 5 | import org.apache.flink.table.api.types.DataType; 6 | import org.apache.flink.table.dataformat.BinaryString; 7 | import org.apache.flink.types.Row; 8 | 9 | /** 10 | * @version $Id: RedisLookupFunction.java, v 0.1 2019年11月24日 5:51 PM Exp $ 11 | */ 12 | public class RedisLookupFunction extends TableFunction { 13 | 14 | private RedisInputFormat redisInputFormat; 15 | 16 | public RedisLookupFunction(final RedisInputFormat redisInputFormat) { 17 | this.redisInputFormat = redisInputFormat; 18 | } 19 | 20 | @Override 21 | public DataType getResultType(final Object[] arguments, final Class[] argTypes) { 22 | return super.getResultType(arguments, argTypes); 23 | } 24 | 25 | @Override 26 | public void open(final FunctionContext context) throws Exception { 27 | redisInputFormat.openInputFormat(); 28 | } 29 | 30 | public void eval(Object... key) { 31 | //redisInputFormat. 32 | String result = redisInputFormat.getRedisConnection().get(BinaryString.fromString(key[0]).toString()); 33 | if (result != null){ 34 | Row row = new Row(2); 35 | row.setField(0, BinaryString.fromString(key[0]).toString()); 36 | row.setField(1, result); 37 | collect(row); 38 | } 39 | } 40 | 41 | @Override 42 | public void close() throws Exception { 43 | super.close(); 44 | redisInputFormat.close(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/org/apache/flink/api/java/io/redis/RedisOptions.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.api.java.io.redis; 2 | 3 | import org.apache.flink.configuration.ConfigOption; 4 | 5 | import java.util.Arrays; 6 | import java.util.List; 7 | 8 | import static org.apache.flink.configuration.ConfigOptions.key; 9 | 10 | 11 | 12 | /** 13 | * Options for redis. 14 | */ 15 | public class RedisOptions { 16 | 17 | 18 | 19 | 20 | public static final ConfigOption HOST = key("host".toLowerCase()).noDefaultValue(); 21 | public static final ConfigOption PORT = key("port".toLowerCase()).noDefaultValue(); 22 | public static final ConfigOption PASSWORD = key("password".toLowerCase()).noDefaultValue(); 23 | public static final ConfigOption BATH_SIZE = key("batchsize".toLowerCase()).noDefaultValue(); 24 | public static final ConfigOption DB = key("db".toLowerCase()).noDefaultValue(); 25 | 26 | 27 | public static final List SUPPORTED_KEYS = Arrays.asList(PASSWORD.key(), HOST.key(), PORT.key(), BATH_SIZE.key(), DB.key()); 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/org/apache/flink/api/java/io/redis/RedisOutFormat.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.api.java.io.redis; 2 | 3 | import org.apache.flink.api.common.io.RichOutputFormat; 4 | import org.apache.flink.configuration.Configuration; 5 | import org.apache.flink.hive.shaded.org.apache.commons.lang3.StringUtils; 6 | import org.apache.flink.types.Row; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import redis.clients.jedis.Jedis; 11 | 12 | import java.io.IOException; 13 | import java.util.ArrayList; 14 | import java.util.List; 15 | 16 | /** 17 | * @version $Id: RedisOutFormat.java, v 0.1 2019年11月29日 11:05 AM Exp $ 18 | */ 19 | public class RedisOutFormat extends RichOutputFormat { 20 | 21 | private static final long serialVersionUID = -3819462971300769382L; 22 | 23 | private static Logger log = LoggerFactory.getLogger(RedisOutFormat.class); 24 | private Jedis jedis; 25 | private List cache = new ArrayList(); 26 | private int batchsize; 27 | private String host; 28 | private int port; 29 | private String password; 30 | private int db; 31 | 32 | public RedisOutFormat(final int batchsize, final String host, final int port, final String password, final int db) { 33 | this.batchsize = batchsize; 34 | this.host = host; 35 | this.port = port; 36 | this.password = password; 37 | this.db = db; 38 | } 39 | 40 | @Override 41 | public void configure(final Configuration parameters) { 42 | } 43 | 44 | @Override 45 | public void open(final int taskNumber, final int numTasks) throws IOException { 46 | log.info(String.format("Open Method Called: taskNumber %d numTasks %d", taskNumber, numTasks)); 47 | batchsize = batchsize > 0 ? batchsize : 1; 48 | try { 49 | jedis = new Jedis(host, port); 50 | if (StringUtils.isNotBlank(password)) { 51 | jedis.auth(password); 52 | } 53 | jedis.select(db); 54 | } catch (Exception e) { 55 | throw new IllegalArgumentException("redis open() failed.", e); 56 | } 57 | 58 | } 59 | 60 | @Override 61 | public void writeRecord(final Row record) throws IOException { 62 | String key = record.getField(0).toString(); 63 | String value = record.getField(1).toString(); 64 | //log.info("key:" + key + ",value:" + value); 65 | try { 66 | cache.add(record); 67 | if (cache.size() >= batchsize) { 68 | for (int i = 0; i < cache.size(); i++) { 69 | jedis.set(key, value); 70 | } 71 | cache.clear(); 72 | } 73 | } catch (Exception e) { 74 | throw new IllegalArgumentException("redis write() failed.", e); 75 | } 76 | 77 | } 78 | 79 | @Override 80 | public void close() throws IOException { 81 | try { 82 | jedis.quit(); 83 | jedis.close(); 84 | } catch (Exception e) { 85 | throw new IllegalArgumentException("redis close() failed.", e); 86 | } 87 | 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/org/apache/flink/api/java/io/redis/RedisRetractTableSink.java: -------------------------------------------------------------------------------- 1 | 2 | package org.apache.flink.api.java.io.redis; 3 | 4 | import org.apache.flink.api.common.functions.FlatMapFunction; 5 | import org.apache.flink.api.java.tuple.Tuple2; 6 | import org.apache.flink.streaming.api.datastream.DataStream; 7 | import org.apache.flink.streaming.api.datastream.DataStreamSink; 8 | import org.apache.flink.table.api.types.DataType; 9 | import org.apache.flink.table.api.types.DataTypes; 10 | import org.apache.flink.table.sinks.RetractStreamTableSink; 11 | import org.apache.flink.table.sinks.TableSink; 12 | import org.apache.flink.table.util.TableConnectorUtil; 13 | import org.apache.flink.types.Row; 14 | import org.apache.flink.util.Collector; 15 | import org.apache.flink.util.InstantiationUtil; 16 | 17 | 18 | /** 19 | * @version $Id: RedisRetractTableSink.java, v 0.1 2019年12月04日 10:43 AM Exp $ 20 | */ 21 | public class RedisRetractTableSink implements RetractStreamTableSink { 22 | private final RedisOutFormat outputFormat; 23 | 24 | private String[] fieldNames; 25 | private DataType[] fieldTypes; 26 | 27 | /** 28 | * Setter method for property fieldNames. 29 | * 30 | * @param fieldNames value to be assigned to property fieldNames 31 | */ 32 | public void setFieldNames(final String[] fieldNames) { 33 | this.fieldNames = fieldNames; 34 | } 35 | 36 | /** 37 | * Setter method for property fieldTypes. 38 | * 39 | * @param fieldTypes value to be assigned to property fieldTypes 40 | */ 41 | public void setFieldTypes(final DataType[] fieldTypes) { 42 | this.fieldTypes = fieldTypes; 43 | } 44 | 45 | public RedisRetractTableSink(final RedisOutFormat outputFormat) { 46 | this.outputFormat = outputFormat; 47 | } 48 | 49 | @Override 50 | public DataType getRecordType() { 51 | return null; 52 | } 53 | 54 | @Override 55 | public DataType getOutputType() { 56 | DataType outputType = DataTypes.createRowType(fieldTypes, fieldNames); 57 | 58 | return DataTypes.createTupleType(DataTypes.BOOLEAN, outputType); 59 | } 60 | 61 | @Override 62 | public DataStreamSink emitDataStream(final DataStream> dataStream) { 63 | 64 | return dataStream.flatMap(new FlatMapFunction, Row>() { 65 | @Override 66 | public void flatMap(final Tuple2 value, final Collector out) throws Exception { 67 | if (value.f0) { 68 | out.collect(value.f1); 69 | } 70 | 71 | } 72 | }).addSink(new RedisSinkFunction(outputFormat)).name(TableConnectorUtil.generateRuntimeName(this.getClass(), fieldNames)); 73 | } 74 | 75 | @Override 76 | public String[] getFieldNames() { 77 | return fieldNames; 78 | } 79 | 80 | @Override 81 | public DataType[] getFieldTypes() { 82 | return fieldTypes; 83 | } 84 | 85 | @Override 86 | public TableSink> configure(final String[] fieldNames, final DataType[] fieldTypes) { 87 | RedisRetractTableSink copy = null; 88 | try { 89 | copy = new RedisRetractTableSink(InstantiationUtil.clone(outputFormat)); 90 | copy.setFieldTypes(fieldTypes); 91 | copy.setFieldNames(fieldNames); 92 | } catch (Exception e) { 93 | throw new RuntimeException(e); 94 | } 95 | return copy; 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/main/java/org/apache/flink/api/java/io/redis/RedisSinkFunction.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.api.java.io.redis; 2 | 3 | import org.apache.flink.api.common.functions.RuntimeContext; 4 | import org.apache.flink.configuration.Configuration; 5 | import org.apache.flink.metrics.Counter; 6 | import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; 7 | import org.apache.flink.types.Row; 8 | 9 | /** 10 | * @version $Id: RedisSinkFunction.java, v 0.1 2019年12月04日 11:05 AM Exp $ 11 | */ 12 | public class RedisSinkFunction extends RichSinkFunction { 13 | 14 | private RedisOutFormat redisOutFormat; 15 | private Counter counter; 16 | 17 | public RedisSinkFunction(final RedisOutFormat redisOutFormat) { 18 | this.redisOutFormat = redisOutFormat; 19 | } 20 | 21 | @Override 22 | public void invoke(final Row value, final Context context) throws Exception { 23 | redisOutFormat.writeRecord(value); 24 | counter.inc(); 25 | } 26 | 27 | @Override 28 | public void open(final Configuration parameters) throws Exception { 29 | super.open(parameters); 30 | RuntimeContext ctx = getRuntimeContext(); 31 | redisOutFormat.setRuntimeContext(ctx); 32 | this.counter = getRuntimeContext().getMetricGroup().counter("RedisSink"); 33 | } 34 | 35 | @Override 36 | public void close() throws Exception { 37 | super.close(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/org/apache/flink/api/java/io/redis/RedisSourceSinkFactory.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.api.java.io.redis; 2 | 3 | import org.apache.flink.api.java.typeutils.TupleTypeInfo; 4 | import org.apache.flink.table.api.RichTableSchema; 5 | import org.apache.flink.table.factories.StreamTableSinkFactory; 6 | import org.apache.flink.table.factories.StreamTableSourceFactory; 7 | import org.apache.flink.table.sinks.StreamTableSink; 8 | import org.apache.flink.table.sources.StreamTableSource; 9 | import org.apache.flink.table.util.TableProperties; 10 | import org.apache.flink.types.Row; 11 | 12 | import java.util.HashMap; 13 | import java.util.HashSet; 14 | import java.util.List; 15 | import java.util.Map; 16 | import java.util.Set; 17 | 18 | import static org.apache.flink.api.java.io.redis.RedisOptions.SUPPORTED_KEYS; 19 | import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_PROPERTY_VERSION; 20 | import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_TYPE; 21 | 22 | /** 23 | * @version $Id: RedisSourceSinkFactory.java, v 0.1 2019年11月29日 10:29 AM Exp $ 24 | */ 25 | public class RedisSourceSinkFactory implements StreamTableSourceFactory, StreamTableSinkFactory { 26 | @Override 27 | public StreamTableSink createStreamTableSink(final Map props) { 28 | TableProperties properties = new TableProperties(); 29 | properties.putProperties(props); 30 | RichTableSchema schema = properties.readSchemaFromProperties(null); 31 | 32 | String host = properties.getString(RedisOptions.HOST); 33 | int port = properties.getInteger(RedisOptions.PORT.key(), 6349); 34 | String password = properties.getString(RedisOptions.PASSWORD); 35 | int db = properties.getInteger(RedisOptions.DB.key(), 0); 36 | int batchSize = properties.getInteger(RedisOptions.BATH_SIZE.key(), 1); 37 | 38 | RedisOutFormat redisOutFormat = new RedisOutFormat(batchSize, host, port, password, db); 39 | 40 | RedisRetractTableSink redisRetractTableSink = new RedisRetractTableSink(redisOutFormat); 41 | redisRetractTableSink.setFieldNames(schema.getColumnNames()); 42 | redisRetractTableSink.setFieldTypes(schema.getColumnTypes()); 43 | 44 | return (StreamTableSink) redisRetractTableSink.configure(schema.getColumnNames(), schema.getColumnTypes()); 45 | } 46 | 47 | @Override 48 | public StreamTableSource createStreamTableSource(final Map props) { 49 | TableProperties properties = new TableProperties(); 50 | properties.putProperties(props); 51 | RichTableSchema schema = properties.readSchemaFromProperties(null); 52 | 53 | String host = properties.getString(RedisOptions.HOST); 54 | int port = properties.getInteger(RedisOptions.PORT.key(), 6349); 55 | String password = properties.getString(RedisOptions.PASSWORD); 56 | 57 | Set> uniqueKeys = new HashSet<>(); 58 | Set> normalIndexes = new HashSet<>(); 59 | if (!schema.getPrimaryKeys().isEmpty()) { 60 | uniqueKeys.add(new HashSet<>(schema.getPrimaryKeys())); 61 | } 62 | for (List uniqueKey : schema.getUniqueKeys()) { 63 | uniqueKeys.add(new HashSet<>(uniqueKey)); 64 | } 65 | for (RichTableSchema.Index index : schema.getIndexes()) { 66 | if (index.unique) { 67 | uniqueKeys.add(new HashSet<>(index.keyList)); 68 | } else { 69 | normalIndexes.add(new HashSet<>(index.keyList)); 70 | } 71 | } 72 | 73 | RedisInputFormat redisInputFormat = new RedisInputFormat(host, port, password); 74 | 75 | RedisTableSource redisTableSource = new RedisTableSource(schema, redisInputFormat, uniqueKeys, normalIndexes); 76 | 77 | return redisTableSource; 78 | } 79 | 80 | @Override 81 | public Map requiredContext() { 82 | Map context = new HashMap<>(); 83 | context.put(CONNECTOR_TYPE, "REDIS"); 84 | context.put(CONNECTOR_PROPERTY_VERSION, "1"); 85 | return context; 86 | } 87 | 88 | @Override 89 | public List supportedProperties() { 90 | return SUPPORTED_KEYS; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/org/apache/flink/api/java/io/redis/RedisTableSource.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.api.java.io.redis; 2 | 3 | import org.apache.flink.streaming.api.datastream.DataStream; 4 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 5 | import org.apache.flink.table.api.RichTableSchema; 6 | import org.apache.flink.table.api.TableSchema; 7 | import org.apache.flink.table.api.functions.AsyncTableFunction; 8 | import org.apache.flink.table.api.functions.TableFunction; 9 | import org.apache.flink.table.api.types.DataType; 10 | import org.apache.flink.table.plan.stats.TableStats; 11 | import org.apache.flink.table.sources.LookupConfig; 12 | import org.apache.flink.table.sources.LookupableTableSource; 13 | import org.apache.flink.table.sources.StreamTableSource; 14 | import org.apache.flink.types.Row; 15 | 16 | import java.util.Set; 17 | 18 | /** 19 | * @version $Id: RedisTableSource.java, v 0.1 2019年12月04日 6:14 PM Exp $ 20 | */ 21 | public class RedisTableSource implements LookupableTableSource, StreamTableSource { 22 | 23 | private RichTableSchema tableSchema; // for table type info 24 | 25 | 26 | private RedisInputFormat redisInputFormat; 27 | private Set> uniqueKeySet; 28 | private Set> indexKeySet; 29 | 30 | public RedisTableSource(final RichTableSchema tableSchema, final RedisInputFormat redisInputFormat, final Set> uniqueKeySet, final Set> indexKeySet) { 31 | this.tableSchema = tableSchema; 32 | this.redisInputFormat = redisInputFormat; 33 | this.uniqueKeySet = uniqueKeySet; 34 | this.indexKeySet = indexKeySet; 35 | } 36 | 37 | @Override 38 | public TableFunction getLookupFunction(final int[] lookupKeys) { 39 | return new RedisLookupFunction(redisInputFormat); 40 | } 41 | 42 | @Override 43 | public AsyncTableFunction getAsyncLookupFunction(final int[] lookupKeys) { 44 | return null; 45 | } 46 | 47 | @Override 48 | public LookupConfig getLookupConfig() { 49 | return null; 50 | } 51 | 52 | @Override 53 | public DataStream getDataStream(final StreamExecutionEnvironment execEnv) { 54 | return null; 55 | } 56 | 57 | @Override 58 | public DataType getReturnType() { 59 | return tableSchema.getResultRowType(); 60 | } 61 | 62 | @Override 63 | public TableSchema getTableSchema() { 64 | TableSchema.Builder builder = TableSchema.builder(); 65 | 66 | // uniqueKeySet.toArray(new String[uniqueKeySet.size()]); 67 | 68 | // HashSet[] uniqueKeySettemp = uniqueKeySet.toArray(new HashSet[uniqueKeySet.size()]); 69 | 70 | // indexKeySet.forEach(indexKey ->builder.uniqueIndex(indexKeySet.toArray(new String[indexKeySet.size()]))); 71 | // builder.uniqueKey("pv"); 72 | for (int i = 0; i < tableSchema.getColumnNames().length; i++) { 73 | 74 | builder.column(tableSchema.getColumnNames()[i], tableSchema.getColumnTypes()[i], tableSchema.getNullables()[i]); 75 | 76 | } 77 | 78 | /// ..foreach(c => builder.field(c.name, c.internalType, c.isNullable)) 79 | 80 | if (uniqueKeySet != null) { 81 | uniqueKeySet.forEach(uniqueKey -> builder.uniqueIndex(uniqueKey.toArray((new String[0])))); 82 | indexKeySet.forEach(indexKey -> builder.normalIndex(indexKey.toArray((new String[0])))); 83 | } 84 | return new TableSchema(builder.build().getColumns(), builder.build().getPrimaryKeys(), builder.build().getUniqueKeys(), builder.build().getUniqueIndexes(), null, null); 85 | } 86 | 87 | @Override 88 | public String explainSource() { 89 | return "Redis"; 90 | } 91 | 92 | @Override 93 | public TableStats getTableStats() { 94 | return null; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/services/org.apache.flink.table.factories.TableFactory: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | org.apache.flink.api.java.io.redis.RedisSourceSinkFactory 17 | --------------------------------------------------------------------------------