├── .gitignore ├── README.md ├── data └── flink_es_table ├── pom.xml └── src ├── main ├── java │ └── org │ │ └── apache │ │ └── flink │ │ ├── connector │ │ └── elasticsearch │ │ │ ├── ElasticsearchTableOptions.java │ │ │ └── table │ │ │ ├── ElasticsearchSourceFactory.java │ │ │ └── ElasticsearchTableSource.java │ │ └── hadoopcompatibility │ │ └── OutputRowFunction.java └── resources │ └── META-INF │ └── services │ └── org.apache.flink.table.factories.TableFactory └── test ├── java └── org │ └── apache │ └── flink │ └── connector │ └── elasticsearch │ └── table │ └── ElasticsearchTableSourceTest.java └── resources └── META-INF └── services └── org.apache.flink.table.factories.TableFactory /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### Java template 3 | # Compiled class file 4 | *.class 5 | 6 | # Log file 7 | *.log 8 | 9 | # BlueJ files 10 | *.ctxt 11 | 12 | # Mobile Tools for Java (J2ME) 13 | .mtj.tmp/ 14 | 15 | # Package Files # 16 | *.jar 17 | *.war 18 | *.nar 19 | *.ear 20 | *.zip 21 | *.tar.gz 22 | *.rar 23 | 24 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 25 | hs_err_pid* 26 | 27 | .idea 28 | target 29 | *.iml 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # flink-connector-elasticsearch-source 2 | 3 | ## Why 4 | 5 | [Elasticsearch | Apache Flink](https://ci.apache.org/projects/flink/flink-docs-release-1.13/docs/connectors/datastream/elasticsearch/) / [Elasticsearch | Apache Flink](https://ci.apache.org/projects/flink/flink-docs-release-1.13/docs/connectors/table/elasticsearch/) / [flink-connector-elasticsearch7](https://github.com/apache/flink/tree/release-1.13.1/flink-connectors/flink-connector-elasticsearch7) doesn't support Elasticsearch Source and Table 6 | 7 | ## How 8 | 9 | * [Hadoop Compatibility | Apache Flink](https://ci.apache.org/projects/flink/flink-docs-release-1.13/docs/dev/dataset/hadoop_compatibility/) 10 | 11 | * [elasticsearch-hadoop](https://github.com/elastic/elasticsearch-hadoop) / [ Elasticsearch for Apache Hadoop](https://www.elastic.co/guide/en/elasticsearch/hadoop/7.x/mapreduce.html) 12 | 13 | ## Install 14 | 15 | ``` 16 | mvn package -Dmaven.test.skip=true 17 | cp target/target/flink-connector-elasticsearch-hadoop-1.0.jar /opt/flink/lib/ 18 | ``` 19 | 20 | ## Use 21 | 22 | ```sql 23 | CREATE TABLE flink_es_table( 24 | _metadata ROW<_index STRING,_type STRING,_id STRING> 25 | ) WITH ( 26 | 'connector.type'='elasticsearch', 27 | 'es.resource'='flink_es_table/_doc', 28 | 'es.nodes'='127.0.0.1:9200', 29 | 'es.port'='9200', 30 | 'es.query'='?q=*', 31 | 'es.nodes.client.only'='false', 32 | 'es.nodes.discovery'='false', 33 | 'es.nodes.wan.only'='true' 34 | ); 35 | 36 | SELECT _index,_type,_id FROM flink_es_table; 37 | ``` 38 | 39 | ## Detail 40 | 41 | [Elasticsearch for Apache Hadoop](https://www.elastic.co/guide/en/elasticsearch/hadoop/7.x/mapreduce.html) 42 | 43 | [Configuration](https://www.elastic.co/guide/en/elasticsearch/hadoop/7.x/configuration.html) 44 | 45 | ## Test 46 | 47 | ### init 48 | 49 | `docker run -d --name es7 -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.10.2-amd64` 50 | 51 | `curl -XPOST --header "Content-Type: application/json" "http://127.0.0.1:9200/_bulk" --data-binary @data/flink_es_table` 52 | 53 | ```json 54 | {"took":640,"errors":false,"items":[{"index":{"_index":"flink_es_table","_type":"_doc","_id":"es_id_1","_version":1,"result":"created","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":0,"_primary_term":1,"status":201}},{"create":{"_index":"flink_es_table","_type":"_doc","_id":"es_id_2","_version":1,"result":"created","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":1,"_primary_term":1,"status":201}}]} 55 | ``` 56 | 57 | ### run 58 | 59 | `org.apache.flink.connector.elasticsearch.table.ElasticsearchTableSourceTest` 60 | 61 | output 62 | 63 | ```sql 64 | CREATE TABLE flink_es_table(_metadata ROW<_index STRING,_type STRING,_id STRING>,int_key INT,int_array ARRAY,int_object MAP,int_nested ARRAY>,string_key STRING,string_array ARRAY,string_object MAP,string_nested ARRAY>,double_key DOUBLE,double_array ARRAY,double_object MAP,double_nested ARRAY>,time_key TIMESTAMP,time_array ARRAY,time_object MAP,time_nested ARRAY>,bool_key BOOLEAN,bool_array ARRAY,bool_object MAP,bool_nested ARRAY>) WITH ( 'connector.type'='elasticsearch', 'es.resource'='flink_es_table/_doc', 'es.nodes'='k8s.cuidp.top:9201', 'es.port'='9201', 'es.query'='?q=*', 'es.nodes.client.only'='false', 'es.nodes.discovery'='false', 'es.nodes.wan.only'='true') 65 | 66 | SELECT _index,_type,_id,int_key,int_array,int_object,int_nested,int_key,int_array,int_object,int_nested,string_key,string_array,string_object,string_nested,time_key,time_array,time_object,time_nested,bool_key,bool_array,bool_object,bool_nested FROM flink_es_table 67 | 68 | input results: 69 | +I[flink_es_table, _doc, es_id_1, 10, [11, 12], {key_2=14, key_1=13}, [+I[15, 16], +I[17, 18]], 10, [11, 12], {key_2=14, key_1=13}, [+I[15, 16], +I[17, 18]], str0, [str1, str2], {key_2=str4, key_1=str3}, [+I[str5, str6], +I[str7, str8]], 2021-01-10 00:00:00.0, [2021-01-11T00:00, 2021-01-12T00:00], {key_2=2021-01-14 00:00:00.0, key_1=2021-01-13 00:00:00.0}, [+I[2021-01-15 00:00:00.0, 2021-01-16 00:00:00.0], +I[2021-01-17 00:00:00.0, 2021-01-18 00:00:00.0]], true, [true, false], {key_2=false, key_1=true}, [+I[true, false], +I[false, true]]] 70 | +I[flink_es_table, _doc, es_id_2, 20, [21, 22], {key_2=24, key_1=23}, [+I[25, 26], +I[27, 28]], 20, [21, 22], {key_2=24, key_1=23}, [+I[25, 26], +I[27, 28]], str0, [str1, str2], {key_2=str4, key_1=str3}, [+I[str5, str6], +I[str7, str8]], 2021-01-20 00:00:00.0, [2021-01-21T00:00, 2021-01-22T00:00], {key_2=2021-01-24 00:00:00.0, key_1=2021-01-23 00:00:00.0}, [+I[2021-01-25 00:00:00.0, 2021-01-26 00:00:00.0], +I[2021-01-27 00:00:00.0, 2021-01-28 00:00:00.0]], true, [true, false], {key_2=false, key_1=true}, [+I[true, false], +I[false, true]]] 71 | 72 | ``` 73 | 74 | 75 | ## Read And Write(just for example) 76 | 77 | [Remote clusters](https://www.elastic.co/guide/en/elasticsearch/reference/7.x/modules-remote-clusters.html) and [Reindex API](https://www.elastic.co/guide/en/elasticsearch/reference/7.x/docs-reindex.html#reindex-from-remote) 78 | 79 | ` ./bin/sql-client.sh` 80 | 81 | ### create elasticsearch source table 82 | 83 | ```sql 84 | CREATE TABLE flink_es_table( 85 | _metadata ROW<_index STRING,_type STRING,_id STRING>, 86 | int_key INT,int_array ARRAY,int_object MAP,int_nested ARRAY>, 87 | string_key STRING,string_array ARRAY,string_object MAP,string_nested ARRAY>, 88 | double_key DOUBLE,double_array ARRAY,double_object MAP,double_nested ARRAY>, 89 | time_key TIMESTAMP,time_array ARRAY,time_object MAP,time_nested ARRAY>, 90 | bool_key BOOLEAN,bool_array ARRAY,bool_object MAP,bool_nested ARRAY> 91 | ) WITH ( 92 | 'connector.type'='elasticsearch', 93 | 'es.resource'='flink_es_table/_doc', 94 | 'es.nodes'='127.0.0.1:9200', 95 | 'es.port'='9200', 96 | 'es.query'='?q=*', 97 | 'es.nodes.client.only'='false', 98 | 'es.nodes.discovery'='false', 99 | 'es.nodes.wan.only'='true' 100 | ); 101 | ``` 102 | 103 | 104 | 105 | ### create elasticsearch sink table 106 | 107 | * orginal 108 | 109 | es doc source with id 110 | 111 | ```sql 112 | CREATE TABLE flink_es_table_copy_sink( 113 | id STRING, 114 | int_key INT,int_array ARRAY,int_object MAP,int_nested ARRAY>, 115 | PRIMARY KEY (`id`) NOT ENFORCED) WITH ( 116 | 'connector'='elasticsearch-7', 117 | 'index'='flink_es_table_copy', 118 | 'hosts'='127.0.0.1:9200' 119 | ); 120 | ``` 121 | 122 | 123 | 124 | * custom [flink-connector-elasticsearch-sink](https://github.com/cclient/flink-connector-elasticsearch-sink) 125 | 126 | es doc source without id 127 | 128 | ```sql 129 | CREATE TABLE flink_es_table_copy_sink( 130 | id STRING, 131 | int_key INT,int_array ARRAY,int_object MAP,int_nested ARRAY>, 132 | PRIMARY KEY (`id`) NOT ENFORCED) WITH ( 133 | 'connector'='elasticsearch-7-ignore', 134 | 'index'='flink_es_table_copy', 135 | 'hosts'='127.0.0.1:9200', 136 | 'ignore-fields'='id' 137 | ); 138 | ``` 139 | 140 | ### source->sink 141 | 142 | ```sql 143 | insert into flink_es_table_copy_sink SELECT _id,int_key,int_array,int_object,int_nested FROM flink_es_table; 144 | ``` 145 | 146 | 147 | 148 | #### search copy 149 | 150 | ```sql 151 | CREATE TABLE flink_es_table_copy_source( 152 | _metadata ROW<_index STRING,_type STRING,_id STRING>, 153 | int_key INT,int_array ARRAY,int_object MAP,int_nested ARRAY> 154 | ) WITH ( 155 | 'connector.type'='elasticsearch', 156 | 'es.resource'='flink_es_table_copy/_doc', 157 | 'es.nodes'='127.0.0.1:9200', 158 | 'es.port'='9200', 159 | 'es.query'='?q=*', 160 | 'es.nodes.client.only'='false', 161 | 'es.nodes.discovery'='false', 162 | 'es.nodes.wan.only'='true' 163 | ); 164 | 165 | select * from flink_es_table_copy_source; 166 | ``` 167 | 168 | #### result 169 | 170 | ```shell 171 | SQL Query Result (Table) 172 | Table program finished. Page: Last of 1 Updated: 11:25:32.615 173 | 174 | _metadata int_key 175 | +I[flink_es_table_copy, _doc,~ 10 176 | +I[flink_es_table_copy, _doc,~ 20 177 | Q Quit + Inc Refresh G Goto Page N Next Page O Open Row 178 | R Refresh - Dec Refresh L Last Page P Prev Page 179 | 180 | --- 181 | _metadata (ROW<`_index` STRING, `_type` STRING, `_id` STRING>): 182 | +I[flink_es_table_copy, _doc, es_id_1] 183 | int_key (INT): 184 | 10 185 | int_array (ARRAY): 186 | [11, 12] 187 | int_object (MAP): 188 | {key_2=14, key_1=13} 189 | int_nested (ARRAY>): 190 | [+I[15, 16], +I[17, 18]] 191 | ``` 192 | 193 | ## More 194 | 195 | OutputRowFunction`+`DeserializationSchema` should load other Hadoop InputFormats as Table 196 | 197 | -------------------------------------------------------------------------------- /data/flink_es_table: -------------------------------------------------------------------------------- 1 | { "index" : { "_index" : "flink_es_table", "_id" : "es_id_1" } } 2 | {"bool_key":true,"bool_array":[true,false],"bool_object":{"key_1":true,"key_2":false},"bool_nested":[{"key_3":true,"key_4":false},{"key_3":false,"key_4":true}],"int_key":10,"int_array":[11,12],"int_object":{"key_1":13,"key_2":14},"int_nested":[{"key_3":15,"key_4":16},{"key_3":17,"key_4":18}],"double_key":10,"double_array":[11,12],"double_object":{"key_1":13,"key_2":14},"double_nested":[{"key_3":15,"key_4":16},{"key_3":17,"key_4":18}],"string_key":"str0","string_array":["str1","str2"],"string_object":{"key_1":"str3","key_2":"str4"},"string_nested":[{"key_3":"str5","key_4":"str6"},{"key_3":"str7","key_4":"str8"}],"time_key":"2021-01-10T00:00:00","time_array":["2021-01-11T00:00:00","2021-01-12T00:00:00"],"time_object":{"key_1":"2021-01-13T00:00:00","key_2":"2021-01-14T00:00:00"},"time_nested":[{"key_3":"2021-01-15T00:00:00","key_4":"2021-01-16T00:00:00"},{"key_3":"2021-01-17T00:00:00","key_4":"2021-01-18T00:00:00"}]} 3 | { "create" : { "_index" : "flink_es_table", "_id" : "es_id_2" } } 4 | {"bool_key":true,"bool_array":[true,false],"bool_object":{"key_1":true,"key_2":false},"bool_nested":[{"key_3":true,"key_4":false},{"key_3":false,"key_4":true}],"int_key":20,"int_array":[21,22],"int_object":{"key_1":23,"key_2":24},"int_nested":[{"key_3":25,"key_4":26},{"key_3":27,"key_4":28}],"double_key":20,"double_array":[21,22],"double_object":{"key_1":23,"key_2":24},"double_nested":[{"key_3":25,"key_4":26},{"key_3":27,"key_4":28}],"string_key":"str0","string_array":["str1","str2"],"string_object":{"key_1":"str3","key_2":"str4"},"string_nested":[{"key_3":"str5","key_4":"str6"},{"key_3":"str7","key_4":"str8"}],"time_key":"2021-01-20T00:00:00","time_array":["2021-01-21T00:00:00","2021-01-22T00:00:00"],"time_object":{"key_1":"2021-01-23T00:00:00","key_2":"2021-01-24T00:00:00"},"time_nested":[{"key_3":"2021-01-25T00:00:00","key_4":"2021-01-26T00:00:00"},{"key_3":"2021-01-27T00:00:00","key_4":"2021-01-28T00:00:00"}]} -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 20 | 22 | 23 | 4.0.0 24 | com.github.cclient 25 | 1.0 26 | jar 27 | 28 | flink-connector-elasticsearch-hadoop 29 | Flink : Connectors : Elasticsearch 30 | 31 | 32 | 1.8 33 | 1.13.1 34 | 35 | 2.8.3 36 | 7.5.1 37 | 2.11 38 | ${java.version} 39 | ${java.version} 40 | UTF-8 41 | 1.20 42 | 43 | 44 | 45 | 46 | 47 | org.apache.flink 48 | flink-java 49 | ${flink.version} 50 | provided 51 | 52 | 53 | 54 | org.apache.flink 55 | flink-json 56 | ${flink.version} 57 | provided 58 | 59 | 60 | 61 | org.elasticsearch 62 | elasticsearch-hadoop-mr 63 | ${elasticsearch-hadoop.version} 64 | 65 | 66 | org.apache.flink 67 | flink-hadoop-compatibility_${scala.binary.version} 68 | ${flink.version} 69 | 70 | 71 | org.apache.hadoop 72 | hadoop-common 73 | 74 | ${hadoop.version} 75 | 76 | 77 | commons-cli 78 | commons-cli 79 | 80 | 81 | jdk.tools 82 | jdk.tools 83 | 84 | 85 | log4j 86 | log4j 87 | 88 | 89 | org.slf4j 90 | slf4j-log4j12 91 | 92 | 93 | 94 | 95 | org.apache.hadoop 96 | hadoop-hdfs-client 97 | ${hadoop.version} 98 | 99 | 100 | com.squareup.okhttp 101 | okhttp 102 | 103 | 104 | commons-cli 105 | commons-cli 106 | 107 | 108 | jdk.tools 109 | jdk.tools 110 | 111 | 112 | log4j 113 | log4j 114 | 115 | 116 | org.slf4j 117 | slf4j-log4j12 118 | 119 | 120 | 121 | 122 | 123 | org.apache.hadoop 124 | hadoop-client 125 | provided 126 | ${hadoop.version} 127 | 128 | 129 | 130 | org.apache.commons 131 | commons-compress 132 | ${commons-compress.version} 133 | 134 | 135 | 136 | 137 | org.apache.flink 138 | flink-table-api-java-bridge_${scala.binary.version} 139 | ${flink.version} 140 | provided 141 | true 142 | 143 | 144 | org.apache.flink 145 | flink-table-planner_${scala.binary.version} 146 | ${flink.version} 147 | provided 148 | 149 | 150 | 151 | org.apache.flink 152 | flink-table-runtime-blink_${scala.binary.version} 153 | ${flink.version} 154 | provided 155 | 156 | 157 | 158 | 159 | org.apache.flink 160 | flink-test-utils_${scala.binary.version} 161 | ${flink.version} 162 | test 163 | 164 | 165 | org.apache.flink 166 | flink-table-common 167 | ${flink.version} 168 | test-jar 169 | test 170 | 171 | 172 | org.apache.flink 173 | flink-streaming-java_${scala.binary.version} 174 | ${flink.version} 175 | test-jar 176 | test 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | org.apache.maven.plugins 185 | maven-compiler-plugin 186 | 3.1 187 | 188 | ${java.version} 189 | ${java.version} 190 | 191 | 192 | 193 | 194 | 195 | org.apache.maven.plugins 196 | maven-shade-plugin 197 | 3.1.1 198 | 199 | 200 | 201 | package 202 | 203 | shade 204 | 205 | 206 | 207 | 208 | org.apache.flink:force-shading 209 | com.google.code.findbugs:jsr305 210 | org.slf4j:* 211 | org.apache.logging.log4j:* 212 | 213 | 214 | 215 | 216 | 218 | *:* 219 | 220 | META-INF/*.SF 221 | META-INF/*.DSA 222 | META-INF/*.RSA 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | -------------------------------------------------------------------------------- /src/main/java/org/apache/flink/connector/elasticsearch/ElasticsearchTableOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.elasticsearch; 20 | 21 | import org.apache.flink.configuration.ConfigOption; 22 | import org.apache.flink.configuration.ConfigOptions; 23 | import org.elasticsearch.hadoop.cfg.ConfigurationOptions; 24 | import org.elasticsearch.hadoop.serialization.field.DateIndexFormatter; 25 | import org.elasticsearch.hadoop.serialization.field.DefaultIndexExtractor; 26 | import org.elasticsearch.hadoop.serialization.field.DefaultParamsExtractor; 27 | 28 | /** 29 | * @author cclient 30 | */ 31 | public class ElasticsearchTableOptions { 32 | public static final ConfigOption ES_NODES = ConfigOptions.key(ConfigurationOptions.ES_NODES).stringType().defaultValue(ConfigurationOptions.ES_NODES_DEFAULT).withDescription(""); 33 | public static final ConfigOption ES_NODES_DISCOVERY = ConfigOptions.key(ConfigurationOptions.ES_NODES_DISCOVERY).stringType().defaultValue(ConfigurationOptions.ES_NODES_DISCOVERY_DEFAULT).withDescription(""); 34 | public static final ConfigOption ES_PORT = ConfigOptions.key(ConfigurationOptions.ES_PORT).stringType().defaultValue(ConfigurationOptions.ES_PORT_DEFAULT).withDescription(""); 35 | public static final ConfigOption ES_NODES_PATH_PREFIX = ConfigOptions.key(ConfigurationOptions.ES_NODES_PATH_PREFIX).stringType().defaultValue(ConfigurationOptions.ES_NODES_PATH_PREFIX_DEFAULT).withDescription(""); 36 | public static final ConfigOption ES_NODES_CLIENT_ONLY = ConfigOptions.key(ConfigurationOptions.ES_NODES_CLIENT_ONLY).stringType().defaultValue(ConfigurationOptions.ES_NODES_CLIENT_ONLY_DEFAULT).withDescription(""); 37 | public static final ConfigOption ES_NODES_DATA_ONLY = ConfigOptions.key(ConfigurationOptions.ES_NODES_DATA_ONLY).stringType().defaultValue(ConfigurationOptions.ES_NODES_DATA_ONLY_DEFAULT).withDescription(""); 38 | public static final ConfigOption ES_NODES_INGEST_ONLY = ConfigOptions.key(ConfigurationOptions.ES_NODES_INGEST_ONLY).stringType().defaultValue(ConfigurationOptions.ES_NODES_INGEST_ONLY_DEFAULT).withDescription(""); 39 | public static final ConfigOption ES_NODES_WAN_ONLY = ConfigOptions.key(ConfigurationOptions.ES_NODES_WAN_ONLY).stringType().defaultValue(ConfigurationOptions.ES_NODES_WAN_ONLY_DEFAULT).withDescription(""); 40 | public static final ConfigOption ES_BATCH_SIZE_BYTES = ConfigOptions.key(ConfigurationOptions.ES_BATCH_SIZE_BYTES).stringType().defaultValue(ConfigurationOptions.ES_BATCH_SIZE_BYTES_DEFAULT).withDescription(""); 41 | public static final ConfigOption ES_BATCH_SIZE_ENTRIES = ConfigOptions.key(ConfigurationOptions.ES_BATCH_SIZE_ENTRIES).stringType().defaultValue(ConfigurationOptions.ES_BATCH_SIZE_ENTRIES_DEFAULT).withDescription(""); 42 | public static final ConfigOption ES_BATCH_FLUSH_MANUAL = ConfigOptions.key(ConfigurationOptions.ES_BATCH_FLUSH_MANUAL).stringType().defaultValue(ConfigurationOptions.ES_BATCH_FLUSH_MANUAL_DEFAULT).withDescription(""); 43 | public static final ConfigOption ES_BATCH_WRITE_REFRESH = ConfigOptions.key(ConfigurationOptions.ES_BATCH_WRITE_REFRESH).stringType().defaultValue(ConfigurationOptions.ES_BATCH_WRITE_REFRESH_DEFAULT).withDescription(""); 44 | public static final ConfigOption ES_BATCH_WRITE_RETRY_COUNT = ConfigOptions.key(ConfigurationOptions.ES_BATCH_WRITE_RETRY_COUNT).stringType().defaultValue(ConfigurationOptions.ES_BATCH_WRITE_RETRY_COUNT_DEFAULT).withDescription(""); 45 | public static final ConfigOption ES_BATCH_WRITE_RETRY_LIMIT = ConfigOptions.key(ConfigurationOptions.ES_BATCH_WRITE_RETRY_LIMIT).stringType().defaultValue(ConfigurationOptions.ES_BATCH_WRITE_RETRY_LIMIT_DEFAULT).withDescription(""); 46 | public static final ConfigOption ES_BATCH_WRITE_RETRY_WAIT = ConfigOptions.key(ConfigurationOptions.ES_BATCH_WRITE_RETRY_WAIT).stringType().defaultValue(ConfigurationOptions.ES_BATCH_WRITE_RETRY_WAIT_DEFAULT).withDescription(""); 47 | public static final ConfigOption ES_BATCH_WRITE_RETRY_POLICY = ConfigOptions.key(ConfigurationOptions.ES_BATCH_WRITE_RETRY_POLICY).stringType().defaultValue(ConfigurationOptions.ES_BATCH_WRITE_RETRY_POLICY_DEFAULT).withDescription(""); 48 | public static final ConfigOption ES_HTTP_TIMEOUT = ConfigOptions.key(ConfigurationOptions.ES_HTTP_TIMEOUT).stringType().defaultValue(ConfigurationOptions.ES_HTTP_TIMEOUT_DEFAULT).withDescription(""); 49 | public static final ConfigOption ES_HTTP_RETRIES = ConfigOptions.key(ConfigurationOptions.ES_HTTP_RETRIES).stringType().defaultValue(ConfigurationOptions.ES_HTTP_RETRIES_DEFAULT).withDescription(""); 50 | public static final ConfigOption ES_SCROLL_KEEPALIVE = ConfigOptions.key(ConfigurationOptions.ES_SCROLL_KEEPALIVE).stringType().defaultValue(ConfigurationOptions.ES_SCROLL_KEEPALIVE_DEFAULT).withDescription(""); 51 | public static final ConfigOption ES_SCROLL_SIZE = ConfigOptions.key(ConfigurationOptions.ES_SCROLL_SIZE).stringType().defaultValue(ConfigurationOptions.ES_SCROLL_SIZE_DEFAULT).withDescription(""); 52 | public static final ConfigOption ES_SCROLL_LIMIT = ConfigOptions.key(ConfigurationOptions.ES_SCROLL_LIMIT).stringType().defaultValue(ConfigurationOptions.ES_SCROLL_LIMIT_DEFAULT).withDescription(""); 53 | public static final ConfigOption ES_HEART_BEAT_LEAD = ConfigOptions.key(ConfigurationOptions.ES_HEART_BEAT_LEAD).stringType().defaultValue(ConfigurationOptions.ES_HEART_BEAT_LEAD_DEFAULT).withDescription(""); 54 | public static final ConfigOption ES_INPUT_JSON = ConfigOptions.key(ConfigurationOptions.ES_INPUT_JSON).stringType().defaultValue(ConfigurationOptions.ES_INPUT_JSON_DEFAULT).withDescription(""); 55 | public static final ConfigOption ES_INDEX_AUTO_CREATE = ConfigOptions.key(ConfigurationOptions.ES_INDEX_AUTO_CREATE).stringType().defaultValue(ConfigurationOptions.ES_INDEX_AUTO_CREATE_DEFAULT).withDescription(""); 56 | public static final ConfigOption ES_INDEX_READ_MISSING_AS_EMPTY = ConfigOptions.key(ConfigurationOptions.ES_INDEX_READ_MISSING_AS_EMPTY).stringType().defaultValue(ConfigurationOptions.ES_INDEX_READ_MISSING_AS_EMPTY_DEFAULT).withDescription(""); 57 | public static final ConfigOption ES_INDEX_READ_ALLOW_RED_STATUS = ConfigOptions.key(ConfigurationOptions.ES_INDEX_READ_ALLOW_RED_STATUS).stringType().defaultValue(ConfigurationOptions.ES_INDEX_READ_ALLOW_RED_STATUS_DEFAULT).withDescription(""); 58 | public static final ConfigOption ES_READ_SHARD_PREFERENCE = ConfigOptions.key(ConfigurationOptions.ES_READ_SHARD_PREFERENCE).stringType().defaultValue(ConfigurationOptions.ES_READ_SHARD_PREFERENCE_DEFAULT).withDescription(""); 59 | public static final ConfigOption ES_MAPPING_CONSTANT_AUTO_QUOTE = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_CONSTANT_AUTO_QUOTE).stringType().defaultValue(ConfigurationOptions.ES_MAPPING_CONSTANT_AUTO_QUOTE_DEFAULT).withDescription(""); 60 | public static final ConfigOption ES_MAPPING_DATE_RICH_OBJECT = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_DATE_RICH_OBJECT).stringType().defaultValue(ConfigurationOptions.ES_MAPPING_DATE_RICH_OBJECT_DEFAULT).withDescription(""); 61 | public static final ConfigOption ES_MAPPING_INCLUDE = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_INCLUDE).stringType().defaultValue(ConfigurationOptions.ES_MAPPING_INCLUDE_DEFAULT).withDescription(""); 62 | public static final ConfigOption ES_MAPPING_EXCLUDE = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_EXCLUDE).stringType().defaultValue(ConfigurationOptions.ES_MAPPING_EXCLUDE_DEFAULT).withDescription(""); 63 | public static final ConfigOption ES_INGEST_PIPELINE = ConfigOptions.key(ConfigurationOptions.ES_INGEST_PIPELINE).stringType().defaultValue(ConfigurationOptions.ES_INGEST_PIPELINE_DEFAULT).withDescription(""); 64 | public static final ConfigOption ES_SPARK_DATAFRAME_WRITE_NULL_VALUES = ConfigOptions.key(ConfigurationOptions.ES_SPARK_DATAFRAME_WRITE_NULL_VALUES).stringType().defaultValue(ConfigurationOptions.ES_SPARK_DATAFRAME_WRITE_NULL_VALUES_DEFAULT).withDescription(""); 65 | public static final ConfigOption ES_READ_FIELD_EMPTY_AS_NULL = ConfigOptions.key(ConfigurationOptions.ES_READ_FIELD_EMPTY_AS_NULL).stringType().defaultValue(ConfigurationOptions.ES_READ_FIELD_EMPTY_AS_NULL_DEFAULT).withDescription(""); 66 | public static final ConfigOption ES_READ_FIELD_VALIDATE_PRESENCE = ConfigOptions.key(ConfigurationOptions.ES_READ_FIELD_VALIDATE_PRESENCE).stringType().defaultValue(ConfigurationOptions.ES_READ_FIELD_VALIDATE_PRESENCE_DEFAULT).withDescription(""); 67 | public static final ConfigOption ES_READ_METADATA = ConfigOptions.key(ConfigurationOptions.ES_READ_METADATA).stringType().defaultValue(ConfigurationOptions.ES_READ_METADATA_DEFAULT).withDescription(""); 68 | public static final ConfigOption ES_READ_METADATA_FIELD = ConfigOptions.key(ConfigurationOptions.ES_READ_METADATA_FIELD).stringType().defaultValue(ConfigurationOptions.ES_READ_METADATA_FIELD_DEFAULT).withDescription(""); 69 | public static final ConfigOption ES_READ_METADATA_VERSION = ConfigOptions.key(ConfigurationOptions.ES_READ_METADATA_VERSION).stringType().defaultValue(ConfigurationOptions.ES_READ_METADATA_VERSION_DEFAULT).withDescription(""); 70 | public static final ConfigOption ES_READ_UNMAPPED_FIELDS_IGNORE = ConfigOptions.key(ConfigurationOptions.ES_READ_UNMAPPED_FIELDS_IGNORE).stringType().defaultValue(ConfigurationOptions.ES_READ_UNMAPPED_FIELDS_IGNORE_DEFAULT).withDescription(""); 71 | public static final ConfigOption ES_WRITE_OPERATION = ConfigOptions.key(ConfigurationOptions.ES_WRITE_OPERATION).stringType().defaultValue(ConfigurationOptions.ES_WRITE_OPERATION_DEFAULT).withDescription(""); 72 | public static final ConfigOption ES_UPDATE_RETRY_ON_CONFLICT = ConfigOptions.key(ConfigurationOptions.ES_UPDATE_RETRY_ON_CONFLICT).stringType().defaultValue(ConfigurationOptions.ES_UPDATE_RETRY_ON_CONFLICT_DEFAULT).withDescription(""); 73 | public static final ConfigOption ES_OUTPUT_JSON = ConfigOptions.key(ConfigurationOptions.ES_OUTPUT_JSON).stringType().defaultValue(ConfigurationOptions.ES_OUTPUT_JSON_DEFAULT).withDescription(""); 74 | public static final ConfigOption ES_NET_TRANSPORT_POOLING_EXPIRATION_TIMEOUT = ConfigOptions.key(ConfigurationOptions.ES_NET_TRANSPORT_POOLING_EXPIRATION_TIMEOUT).stringType().defaultValue(ConfigurationOptions.ES_NET_TRANSPORT_POOLING_EXPIRATION_TIMEOUT_DEFAULT).withDescription(""); 75 | public static final ConfigOption ES_NET_USE_SSL = ConfigOptions.key(ConfigurationOptions.ES_NET_USE_SSL).stringType().defaultValue(ConfigurationOptions.ES_NET_USE_SSL_DEFAULT).withDescription(""); 76 | public static final ConfigOption ES_NET_SSL_PROTOCOL = ConfigOptions.key(ConfigurationOptions.ES_NET_SSL_PROTOCOL).stringType().defaultValue(ConfigurationOptions.ES_NET_SSL_PROTOCOL_DEFAULT).withDescription(""); 77 | public static final ConfigOption ES_NET_SSL_KEYSTORE_TYPE = ConfigOptions.key(ConfigurationOptions.ES_NET_SSL_KEYSTORE_TYPE).stringType().defaultValue(ConfigurationOptions.ES_NET_SSL_KEYSTORE_TYPE_DEFAULT).withDescription(""); 78 | public static final ConfigOption ES_NET_SSL_CERT_ALLOW_SELF_SIGNED = ConfigOptions.key(ConfigurationOptions.ES_NET_SSL_CERT_ALLOW_SELF_SIGNED).stringType().defaultValue(ConfigurationOptions.ES_NET_SSL_CERT_ALLOW_SELF_SIGNED_DEFAULT).withDescription(""); 79 | public static final ConfigOption ES_NET_SPNEGO_AUTH_MUTUAL = ConfigOptions.key(ConfigurationOptions.ES_NET_SPNEGO_AUTH_MUTUAL).stringType().defaultValue(ConfigurationOptions.ES_NET_SPNEGO_AUTH_MUTUAL_DEFAULT).withDescription(""); 80 | public static final ConfigOption ES_NET_PROXY_HTTP_USE_SYSTEM_PROPS = ConfigOptions.key(ConfigurationOptions.ES_NET_PROXY_HTTP_USE_SYSTEM_PROPS).stringType().defaultValue(ConfigurationOptions.ES_NET_PROXY_HTTP_USE_SYSTEM_PROPS_DEFAULT).withDescription(""); 81 | public static final ConfigOption ES_NET_PROXY_HTTPS_USE_SYSTEM_PROPS = ConfigOptions.key(ConfigurationOptions.ES_NET_PROXY_HTTPS_USE_SYSTEM_PROPS).stringType().defaultValue(ConfigurationOptions.ES_NET_PROXY_HTTPS_USE_SYSTEM_PROPS_DEFAULT).withDescription(""); 82 | public static final ConfigOption ES_NET_PROXY_SOCKS_USE_SYSTEM_PROPS = ConfigOptions.key(ConfigurationOptions.ES_NET_PROXY_SOCKS_USE_SYSTEM_PROPS).stringType().defaultValue(ConfigurationOptions.ES_NET_PROXY_SOCKS_USE_SYSTEM_PROPS_DEFAULT).withDescription(""); 83 | public static final ConfigOption ES_MAPPING_DEFAULT_INDEX_EXTRACTOR_CLASS = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_DEFAULT_INDEX_EXTRACTOR_CLASS).stringType().defaultValue(DefaultIndexExtractor.class.getName()).withDescription(""); 84 | public static final ConfigOption ES_MAPPING_DEFAULT_INDEX_FORMATTER_CLASS = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_DEFAULT_INDEX_FORMATTER_CLASS).stringType().defaultValue(DateIndexFormatter.class.getName()).withDescription(""); 85 | public static final ConfigOption ES_MAPPING_PARAMS_DEFAULT_EXTRACTOR_CLASS = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_PARAMS_DEFAULT_EXTRACTOR_CLASS).stringType().defaultValue(DefaultParamsExtractor.class.getName()).withDescription(""); 86 | public static final ConfigOption ES_RESOURCE = ConfigOptions.key(ConfigurationOptions.ES_RESOURCE).stringType().noDefaultValue().withDescription(""); 87 | public static final ConfigOption ES_RESOURCE_READ = ConfigOptions.key(ConfigurationOptions.ES_RESOURCE_READ).stringType().noDefaultValue().withDescription(""); 88 | public static final ConfigOption ES_RESOURCE_WRITE = ConfigOptions.key(ConfigurationOptions.ES_RESOURCE_WRITE).stringType().noDefaultValue().withDescription(""); 89 | public static final ConfigOption ES_QUERY = ConfigOptions.key(ConfigurationOptions.ES_QUERY).stringType().noDefaultValue().withDescription(""); 90 | public static final ConfigOption ES_NODES_RESOLVE_HOST_NAME = ConfigOptions.key(ConfigurationOptions.ES_NODES_RESOLVE_HOST_NAME).stringType().noDefaultValue().withDescription(""); 91 | public static final ConfigOption ES_KEYSTORE_LOCATION = ConfigOptions.key(ConfigurationOptions.ES_KEYSTORE_LOCATION).stringType().noDefaultValue().withDescription(""); 92 | public static final ConfigOption ES_BATCH_WRITE_RETRY_POLICY_NONE = ConfigOptions.key(ConfigurationOptions.ES_BATCH_WRITE_RETRY_POLICY_NONE).stringType().noDefaultValue().withDescription(""); 93 | public static final ConfigOption ES_BATCH_WRITE_RETRY_POLICY_SIMPLE = ConfigOptions.key(ConfigurationOptions.ES_BATCH_WRITE_RETRY_POLICY_SIMPLE).stringType().noDefaultValue().withDescription(""); 94 | public static final ConfigOption ES_SERIALIZATION_WRITER_VALUE_CLASS = ConfigOptions.key(ConfigurationOptions.ES_SERIALIZATION_WRITER_VALUE_CLASS).stringType().noDefaultValue().withDescription(""); 95 | public static final ConfigOption ES_SERIALIZATION_WRITER_BYTES_CLASS = ConfigOptions.key(ConfigurationOptions.ES_SERIALIZATION_WRITER_BYTES_CLASS).stringType().noDefaultValue().withDescription(""); 96 | public static final ConfigOption ES_SERIALIZATION_READER_VALUE_CLASS = ConfigOptions.key(ConfigurationOptions.ES_SERIALIZATION_READER_VALUE_CLASS).stringType().noDefaultValue().withDescription(""); 97 | public static final ConfigOption ES_MAX_DOCS_PER_PARTITION = ConfigOptions.key(ConfigurationOptions.ES_MAX_DOCS_PER_PARTITION).stringType().noDefaultValue().withDescription(""); 98 | public static final ConfigOption ES_MAPPING_METADATA_EXTRACTOR_CLASS = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_METADATA_EXTRACTOR_CLASS).stringType().noDefaultValue().withDescription(""); 99 | public static final ConfigOption ES_MAPPING_ID = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_ID).stringType().noDefaultValue().withDescription(""); 100 | public static final ConfigOption ES_MAPPING_ID_EXTRACTOR_CLASS = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_ID_EXTRACTOR_CLASS).stringType().noDefaultValue().withDescription(""); 101 | public static final ConfigOption ES_MAPPING_PARENT = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_PARENT).stringType().noDefaultValue().withDescription(""); 102 | public static final ConfigOption ES_MAPPING_PARENT_EXTRACTOR_CLASS = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_PARENT_EXTRACTOR_CLASS).stringType().noDefaultValue().withDescription(""); 103 | public static final ConfigOption ES_MAPPING_JOIN = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_JOIN).stringType().noDefaultValue().withDescription(""); 104 | public static final ConfigOption ES_MAPPING_JOIN_EXTRACTOR_CLASS = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_JOIN_EXTRACTOR_CLASS).stringType().noDefaultValue().withDescription(""); 105 | public static final ConfigOption ES_MAPPING_VERSION = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_VERSION).stringType().noDefaultValue().withDescription(""); 106 | public static final ConfigOption ES_MAPPING_VERSION_EXTRACTOR_CLASS = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_VERSION_EXTRACTOR_CLASS).stringType().noDefaultValue().withDescription(""); 107 | public static final ConfigOption ES_MAPPING_ROUTING = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_ROUTING).stringType().noDefaultValue().withDescription(""); 108 | public static final ConfigOption ES_MAPPING_ROUTING_EXTRACTOR_CLASS = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_ROUTING_EXTRACTOR_CLASS).stringType().noDefaultValue().withDescription(""); 109 | public static final ConfigOption ES_MAPPING_TTL = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_TTL).stringType().noDefaultValue().withDescription(""); 110 | public static final ConfigOption ES_MAPPING_TTL_EXTRACTOR_CLASS = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_TTL_EXTRACTOR_CLASS).stringType().noDefaultValue().withDescription(""); 111 | public static final ConfigOption ES_MAPPING_TIMESTAMP = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_TIMESTAMP).stringType().noDefaultValue().withDescription(""); 112 | public static final ConfigOption ES_MAPPING_TIMESTAMP_EXTRACTOR_CLASS = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_TIMESTAMP_EXTRACTOR_CLASS).stringType().noDefaultValue().withDescription(""); 113 | public static final ConfigOption ES_MAPPING_INDEX_EXTRACTOR_CLASS = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_INDEX_EXTRACTOR_CLASS).stringType().noDefaultValue().withDescription(""); 114 | public static final ConfigOption ES_MAPPING_INDEX_FORMATTER_CLASS = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_INDEX_FORMATTER_CLASS).stringType().noDefaultValue().withDescription(""); 115 | public static final ConfigOption ES_MAPPING_PARAMS_EXTRACTOR_CLASS = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_PARAMS_EXTRACTOR_CLASS).stringType().noDefaultValue().withDescription(""); 116 | public static final ConfigOption ES_MAPPING_VERSION_TYPE = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_VERSION_TYPE).stringType().noDefaultValue().withDescription(""); 117 | public static final ConfigOption ES_MAPPING_VERSION_TYPE_INTERNAL = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_VERSION_TYPE_INTERNAL).stringType().noDefaultValue().withDescription(""); 118 | public static final ConfigOption ES_MAPPING_VERSION_TYPE_EXTERNAL = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_VERSION_TYPE_EXTERNAL).stringType().noDefaultValue().withDescription(""); 119 | public static final ConfigOption ES_MAPPING_VERSION_TYPE_EXTERNAL_GT = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_VERSION_TYPE_EXTERNAL_GT).stringType().noDefaultValue().withDescription(""); 120 | public static final ConfigOption ES_MAPPING_VERSION_TYPE_EXTERNAL_GTE = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_VERSION_TYPE_EXTERNAL_GTE).stringType().noDefaultValue().withDescription(""); 121 | public static final ConfigOption ES_MAPPING_VERSION_TYPE_FORCE = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_VERSION_TYPE_FORCE).stringType().noDefaultValue().withDescription(""); 122 | public static final ConfigOption ES_READ_FIELD_EMPTY_AS_NULL_LEGACY = ConfigOptions.key(ConfigurationOptions.ES_READ_FIELD_EMPTY_AS_NULL_LEGACY).stringType().noDefaultValue().withDescription(""); 123 | public static final ConfigOption ES_READ_FIELD_VALIDATE_PRESENCE_LEGACY = ConfigOptions.key(ConfigurationOptions.ES_READ_FIELD_VALIDATE_PRESENCE_LEGACY).stringType().noDefaultValue().withDescription(""); 124 | public static final ConfigOption ES_READ_FIELD_INCLUDE = ConfigOptions.key(ConfigurationOptions.ES_READ_FIELD_INCLUDE).stringType().noDefaultValue().withDescription(""); 125 | public static final ConfigOption ES_READ_FIELD_EXCLUDE = ConfigOptions.key(ConfigurationOptions.ES_READ_FIELD_EXCLUDE).stringType().noDefaultValue().withDescription(""); 126 | public static final ConfigOption ES_READ_FIELD_AS_ARRAY_INCLUDE = ConfigOptions.key(ConfigurationOptions.ES_READ_FIELD_AS_ARRAY_INCLUDE).stringType().noDefaultValue().withDescription(""); 127 | public static final ConfigOption ES_READ_FIELD_AS_ARRAY_EXCLUDE = ConfigOptions.key(ConfigurationOptions.ES_READ_FIELD_AS_ARRAY_EXCLUDE).stringType().noDefaultValue().withDescription(""); 128 | public static final ConfigOption ES_READ_SOURCE_FILTER = ConfigOptions.key(ConfigurationOptions.ES_READ_SOURCE_FILTER).stringType().noDefaultValue().withDescription(""); 129 | public static final ConfigOption ES_OPERATION_INDEX = ConfigOptions.key(ConfigurationOptions.ES_OPERATION_INDEX).stringType().noDefaultValue().withDescription(""); 130 | public static final ConfigOption ES_OPERATION_CREATE = ConfigOptions.key(ConfigurationOptions.ES_OPERATION_CREATE).stringType().noDefaultValue().withDescription(""); 131 | public static final ConfigOption ES_OPERATION_UPDATE = ConfigOptions.key(ConfigurationOptions.ES_OPERATION_UPDATE).stringType().noDefaultValue().withDescription(""); 132 | public static final ConfigOption ES_OPERATION_UPSERT = ConfigOptions.key(ConfigurationOptions.ES_OPERATION_UPSERT).stringType().noDefaultValue().withDescription(""); 133 | public static final ConfigOption ES_OPERATION_DELETE = ConfigOptions.key(ConfigurationOptions.ES_OPERATION_DELETE).stringType().noDefaultValue().withDescription(""); 134 | public static final ConfigOption ES_UPDATE_SCRIPT_FILE = ConfigOptions.key(ConfigurationOptions.ES_UPDATE_SCRIPT_FILE).stringType().noDefaultValue().withDescription(""); 135 | public static final ConfigOption ES_UPDATE_SCRIPT_INLINE = ConfigOptions.key(ConfigurationOptions.ES_UPDATE_SCRIPT_INLINE).stringType().noDefaultValue().withDescription(""); 136 | public static final ConfigOption ES_UPDATE_SCRIPT_STORED = ConfigOptions.key(ConfigurationOptions.ES_UPDATE_SCRIPT_STORED).stringType().noDefaultValue().withDescription(""); 137 | public static final ConfigOption ES_UPDATE_SCRIPT_LEGACY = ConfigOptions.key(ConfigurationOptions.ES_UPDATE_SCRIPT_LEGACY).stringType().noDefaultValue().withDescription(""); 138 | public static final ConfigOption ES_UPDATE_SCRIPT_LANG = ConfigOptions.key(ConfigurationOptions.ES_UPDATE_SCRIPT_LANG).stringType().noDefaultValue().withDescription(""); 139 | public static final ConfigOption ES_UPDATE_SCRIPT_PARAMS = ConfigOptions.key(ConfigurationOptions.ES_UPDATE_SCRIPT_PARAMS).stringType().noDefaultValue().withDescription(""); 140 | public static final ConfigOption ES_UPDATE_SCRIPT_PARAMS_JSON = ConfigOptions.key(ConfigurationOptions.ES_UPDATE_SCRIPT_PARAMS_JSON).stringType().noDefaultValue().withDescription(""); 141 | public static final ConfigOption ES_NET_SSL_KEYSTORE_LOCATION = ConfigOptions.key(ConfigurationOptions.ES_NET_SSL_KEYSTORE_LOCATION).stringType().noDefaultValue().withDescription(""); 142 | public static final ConfigOption ES_NET_SSL_KEYSTORE_PASS = ConfigOptions.key(ConfigurationOptions.ES_NET_SSL_KEYSTORE_PASS).stringType().noDefaultValue().withDescription(""); 143 | public static final ConfigOption ES_NET_SSL_TRUST_STORE_LOCATION = ConfigOptions.key(ConfigurationOptions.ES_NET_SSL_TRUST_STORE_LOCATION).stringType().noDefaultValue().withDescription(""); 144 | public static final ConfigOption ES_NET_SSL_TRUST_STORE_PASS = ConfigOptions.key(ConfigurationOptions.ES_NET_SSL_TRUST_STORE_PASS).stringType().noDefaultValue().withDescription(""); 145 | public static final ConfigOption ES_NET_HTTP_HEADER_PREFIX = ConfigOptions.key(ConfigurationOptions.ES_NET_HTTP_HEADER_PREFIX).stringType().noDefaultValue().withDescription(""); 146 | public static final ConfigOption ES_NET_HTTP_AUTH_USER = ConfigOptions.key(ConfigurationOptions.ES_NET_HTTP_AUTH_USER).stringType().noDefaultValue().withDescription(""); 147 | public static final ConfigOption ES_NET_HTTP_AUTH_PASS = ConfigOptions.key(ConfigurationOptions.ES_NET_HTTP_AUTH_PASS).stringType().noDefaultValue().withDescription(""); 148 | public static final ConfigOption ES_NET_SPNEGO_AUTH_ELASTICSEARCH_PRINCIPAL = ConfigOptions.key(ConfigurationOptions.ES_NET_SPNEGO_AUTH_ELASTICSEARCH_PRINCIPAL).stringType().noDefaultValue().withDescription(""); 149 | public static final ConfigOption ES_NET_PROXY_HTTP_HOST = ConfigOptions.key(ConfigurationOptions.ES_NET_PROXY_HTTP_HOST).stringType().noDefaultValue().withDescription(""); 150 | public static final ConfigOption ES_NET_PROXY_HTTP_PORT = ConfigOptions.key(ConfigurationOptions.ES_NET_PROXY_HTTP_PORT).stringType().noDefaultValue().withDescription(""); 151 | public static final ConfigOption ES_NET_PROXY_HTTP_USER = ConfigOptions.key(ConfigurationOptions.ES_NET_PROXY_HTTP_USER).stringType().noDefaultValue().withDescription(""); 152 | public static final ConfigOption ES_NET_PROXY_HTTP_PASS = ConfigOptions.key(ConfigurationOptions.ES_NET_PROXY_HTTP_PASS).stringType().noDefaultValue().withDescription(""); 153 | public static final ConfigOption ES_NET_PROXY_HTTPS_HOST = ConfigOptions.key(ConfigurationOptions.ES_NET_PROXY_HTTPS_HOST).stringType().noDefaultValue().withDescription(""); 154 | public static final ConfigOption ES_NET_PROXY_HTTPS_PORT = ConfigOptions.key(ConfigurationOptions.ES_NET_PROXY_HTTPS_PORT).stringType().noDefaultValue().withDescription(""); 155 | public static final ConfigOption ES_NET_PROXY_HTTPS_USER = ConfigOptions.key(ConfigurationOptions.ES_NET_PROXY_HTTPS_USER).stringType().noDefaultValue().withDescription(""); 156 | public static final ConfigOption ES_NET_PROXY_HTTPS_PASS = ConfigOptions.key(ConfigurationOptions.ES_NET_PROXY_HTTPS_PASS).stringType().noDefaultValue().withDescription(""); 157 | public static final ConfigOption ES_NET_PROXY_SOCKS_HOST = ConfigOptions.key(ConfigurationOptions.ES_NET_PROXY_SOCKS_HOST).stringType().noDefaultValue().withDescription(""); 158 | public static final ConfigOption ES_NET_PROXY_SOCKS_PORT = ConfigOptions.key(ConfigurationOptions.ES_NET_PROXY_SOCKS_PORT).stringType().noDefaultValue().withDescription(""); 159 | public static final ConfigOption ES_NET_PROXY_SOCKS_USER = ConfigOptions.key(ConfigurationOptions.ES_NET_PROXY_SOCKS_USER).stringType().noDefaultValue().withDescription(""); 160 | public static final ConfigOption ES_NET_PROXY_SOCKS_PASS = ConfigOptions.key(ConfigurationOptions.ES_NET_PROXY_SOCKS_PASS).stringType().noDefaultValue().withDescription(""); 161 | public static final ConfigOption ES_SECURITY_AUTHENTICATION = ConfigOptions.key(ConfigurationOptions.ES_SECURITY_AUTHENTICATION).stringType().noDefaultValue().withDescription(""); 162 | public static final ConfigOption ES_SECURITY_USER_PROVIDER_CLASS = ConfigOptions.key(ConfigurationOptions.ES_SECURITY_USER_PROVIDER_CLASS).stringType().noDefaultValue().withDescription(""); 163 | public static final ConfigOption ES_MAPPING_DEFAULT_EXTRACTOR_CLASS = ConfigOptions.key(ConfigurationOptions.ES_MAPPING_DEFAULT_EXTRACTOR_CLASS).stringType().noDefaultValue().withDescription(""); 164 | 165 | public static ConfigOption[] ES_CONFIG_OPTIONS = new ConfigOption[]{ 166 | ES_NODES, 167 | ES_NODES_DISCOVERY, 168 | ES_PORT, 169 | ES_NODES_PATH_PREFIX, 170 | ES_NODES_CLIENT_ONLY, 171 | ES_NODES_DATA_ONLY, 172 | ES_NODES_INGEST_ONLY, 173 | ES_NODES_WAN_ONLY, 174 | ES_BATCH_SIZE_BYTES, 175 | ES_BATCH_SIZE_ENTRIES, 176 | ES_BATCH_FLUSH_MANUAL, 177 | ES_BATCH_WRITE_REFRESH, 178 | ES_BATCH_WRITE_RETRY_COUNT, 179 | ES_BATCH_WRITE_RETRY_LIMIT, 180 | ES_BATCH_WRITE_RETRY_WAIT, 181 | ES_BATCH_WRITE_RETRY_POLICY, 182 | ES_HTTP_TIMEOUT, 183 | ES_HTTP_RETRIES, 184 | ES_SCROLL_KEEPALIVE, 185 | ES_SCROLL_SIZE, 186 | ES_SCROLL_LIMIT, 187 | ES_HEART_BEAT_LEAD, 188 | ES_INPUT_JSON, 189 | ES_INDEX_AUTO_CREATE, 190 | ES_INDEX_READ_MISSING_AS_EMPTY, 191 | ES_INDEX_READ_ALLOW_RED_STATUS, 192 | ES_READ_SHARD_PREFERENCE, 193 | ES_MAPPING_CONSTANT_AUTO_QUOTE, 194 | ES_MAPPING_DATE_RICH_OBJECT, 195 | ES_MAPPING_INCLUDE, 196 | ES_MAPPING_EXCLUDE, 197 | ES_INGEST_PIPELINE, 198 | ES_SPARK_DATAFRAME_WRITE_NULL_VALUES, 199 | ES_READ_FIELD_EMPTY_AS_NULL, 200 | ES_READ_FIELD_VALIDATE_PRESENCE, 201 | ES_READ_METADATA, 202 | ES_READ_METADATA_FIELD, 203 | ES_READ_METADATA_VERSION, 204 | ES_READ_UNMAPPED_FIELDS_IGNORE, 205 | ES_WRITE_OPERATION, 206 | ES_UPDATE_RETRY_ON_CONFLICT, 207 | ES_OUTPUT_JSON, 208 | ES_NET_TRANSPORT_POOLING_EXPIRATION_TIMEOUT, 209 | ES_NET_USE_SSL, 210 | ES_NET_SSL_PROTOCOL, 211 | ES_NET_SSL_KEYSTORE_TYPE, 212 | ES_NET_SSL_CERT_ALLOW_SELF_SIGNED, 213 | ES_NET_SPNEGO_AUTH_MUTUAL, 214 | ES_NET_PROXY_HTTP_USE_SYSTEM_PROPS, 215 | ES_NET_PROXY_HTTPS_USE_SYSTEM_PROPS, 216 | ES_NET_PROXY_SOCKS_USE_SYSTEM_PROPS, 217 | ES_MAPPING_DEFAULT_INDEX_EXTRACTOR_CLASS, 218 | ES_MAPPING_DEFAULT_INDEX_FORMATTER_CLASS, 219 | ES_MAPPING_PARAMS_DEFAULT_EXTRACTOR_CLASS, 220 | ES_RESOURCE, 221 | ES_RESOURCE_READ, 222 | ES_RESOURCE_WRITE, 223 | ES_QUERY, 224 | ES_NODES_RESOLVE_HOST_NAME, 225 | ES_KEYSTORE_LOCATION, 226 | ES_BATCH_WRITE_RETRY_POLICY_NONE, 227 | ES_BATCH_WRITE_RETRY_POLICY_SIMPLE, 228 | ES_SERIALIZATION_WRITER_VALUE_CLASS, 229 | ES_SERIALIZATION_WRITER_BYTES_CLASS, 230 | ES_SERIALIZATION_READER_VALUE_CLASS, 231 | ES_MAX_DOCS_PER_PARTITION, 232 | ES_MAPPING_METADATA_EXTRACTOR_CLASS, 233 | ES_MAPPING_ID, 234 | ES_MAPPING_ID_EXTRACTOR_CLASS, 235 | ES_MAPPING_PARENT, 236 | ES_MAPPING_PARENT_EXTRACTOR_CLASS, 237 | ES_MAPPING_JOIN, 238 | ES_MAPPING_JOIN_EXTRACTOR_CLASS, 239 | ES_MAPPING_VERSION, 240 | ES_MAPPING_VERSION_EXTRACTOR_CLASS, 241 | ES_MAPPING_ROUTING, 242 | ES_MAPPING_ROUTING_EXTRACTOR_CLASS, 243 | ES_MAPPING_TTL, 244 | ES_MAPPING_TTL_EXTRACTOR_CLASS, 245 | ES_MAPPING_TIMESTAMP, 246 | ES_MAPPING_TIMESTAMP_EXTRACTOR_CLASS, 247 | ES_MAPPING_INDEX_EXTRACTOR_CLASS, 248 | ES_MAPPING_INDEX_FORMATTER_CLASS, 249 | ES_MAPPING_PARAMS_EXTRACTOR_CLASS, 250 | ES_MAPPING_VERSION_TYPE, 251 | ES_MAPPING_VERSION_TYPE_INTERNAL, 252 | ES_MAPPING_VERSION_TYPE_EXTERNAL, 253 | ES_MAPPING_VERSION_TYPE_EXTERNAL_GT, 254 | ES_MAPPING_VERSION_TYPE_EXTERNAL_GTE, 255 | ES_MAPPING_VERSION_TYPE_FORCE, 256 | ES_READ_FIELD_EMPTY_AS_NULL_LEGACY, 257 | ES_READ_FIELD_VALIDATE_PRESENCE_LEGACY, 258 | ES_READ_FIELD_INCLUDE, 259 | ES_READ_FIELD_EXCLUDE, 260 | ES_READ_FIELD_AS_ARRAY_INCLUDE, 261 | ES_READ_FIELD_AS_ARRAY_EXCLUDE, 262 | ES_READ_SOURCE_FILTER, 263 | ES_OPERATION_INDEX, 264 | ES_OPERATION_CREATE, 265 | ES_OPERATION_UPDATE, 266 | ES_OPERATION_UPSERT, 267 | ES_OPERATION_DELETE, 268 | ES_UPDATE_SCRIPT_FILE, 269 | ES_UPDATE_SCRIPT_INLINE, 270 | ES_UPDATE_SCRIPT_STORED, 271 | ES_UPDATE_SCRIPT_LEGACY, 272 | ES_UPDATE_SCRIPT_LANG, 273 | ES_UPDATE_SCRIPT_PARAMS, 274 | ES_UPDATE_SCRIPT_PARAMS_JSON, 275 | ES_NET_SSL_KEYSTORE_LOCATION, 276 | ES_NET_SSL_KEYSTORE_PASS, 277 | ES_NET_SSL_TRUST_STORE_LOCATION, 278 | ES_NET_SSL_TRUST_STORE_PASS, 279 | ES_NET_HTTP_HEADER_PREFIX, 280 | ES_NET_HTTP_AUTH_USER, 281 | ES_NET_HTTP_AUTH_PASS, 282 | ES_NET_SPNEGO_AUTH_ELASTICSEARCH_PRINCIPAL, 283 | ES_NET_PROXY_HTTP_HOST, 284 | ES_NET_PROXY_HTTP_PORT, 285 | ES_NET_PROXY_HTTP_USER, 286 | ES_NET_PROXY_HTTP_PASS, 287 | ES_NET_PROXY_HTTPS_HOST, 288 | ES_NET_PROXY_HTTPS_PORT, 289 | ES_NET_PROXY_HTTPS_USER, 290 | ES_NET_PROXY_HTTPS_PASS, 291 | ES_NET_PROXY_SOCKS_HOST, 292 | ES_NET_PROXY_SOCKS_PORT, 293 | ES_NET_PROXY_SOCKS_USER, 294 | ES_NET_PROXY_SOCKS_PASS, 295 | ES_SECURITY_AUTHENTICATION, 296 | ES_SECURITY_USER_PROVIDER_CLASS, 297 | ES_MAPPING_DEFAULT_EXTRACTOR_CLASS 298 | }; 299 | 300 | } 301 | -------------------------------------------------------------------------------- /src/main/java/org/apache/flink/connector/elasticsearch/table/ElasticsearchSourceFactory.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.connector.elasticsearch.table; 2 | 3 | import org.apache.flink.configuration.ConfigOption; 4 | import org.apache.flink.connector.elasticsearch.ElasticsearchTableOptions; 5 | import org.apache.flink.table.api.TableSchema; 6 | import org.apache.flink.table.descriptors.DescriptorProperties; 7 | import org.apache.flink.table.factories.BatchTableSourceFactory; 8 | import org.apache.flink.table.sources.BatchTableSource; 9 | import org.apache.flink.table.utils.TableSchemaUtils; 10 | import org.apache.flink.types.Row; 11 | import org.apache.hadoop.conf.Configuration; 12 | import org.apache.hadoop.hdfs.HdfsConfiguration; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import java.util.ArrayList; 17 | import java.util.HashMap; 18 | import java.util.List; 19 | import java.util.Map; 20 | 21 | import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_PROPERTY_VERSION; 22 | import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_TYPE; 23 | import static org.apache.flink.table.descriptors.DescriptorProperties.*; 24 | import static org.apache.flink.table.descriptors.Schema.*; 25 | 26 | /** 27 | * @author cclient 28 | */ 29 | public class ElasticsearchSourceFactory implements BatchTableSourceFactory { 30 | 31 | private static final Logger log = LoggerFactory.getLogger(ElasticsearchSourceFactory.class); 32 | 33 | public void setConf(Configuration conf, DescriptorProperties descriptorProperties, String key) { 34 | if (descriptorProperties.containsKey(key)) { 35 | conf.set(key, descriptorProperties.getString(key)); 36 | } 37 | } 38 | 39 | public Configuration buildConfiguration(DescriptorProperties descriptorProperties) { 40 | Configuration conf = new Configuration(); 41 | for (ConfigOption esConfigOption : ElasticsearchTableOptions.ES_CONFIG_OPTIONS) { 42 | setConf(conf, descriptorProperties, esConfigOption.key()); 43 | } 44 | return conf; 45 | } 46 | 47 | @Override 48 | public BatchTableSource createBatchTableSource(Map properties) { 49 | final DescriptorProperties descriptorProperties = new DescriptorProperties(true); 50 | descriptorProperties.putProperties(properties); 51 | Configuration conf = buildConfiguration(descriptorProperties); 52 | TableSchema schema = 53 | TableSchemaUtils.getPhysicalSchema(descriptorProperties.getTableSchema(SCHEMA)); 54 | return ElasticsearchTableSource.builder() 55 | .setConfiguration(conf) 56 | .setSchema(schema).build(); 57 | } 58 | 59 | @Override 60 | public Map requiredContext() { 61 | Map context = new HashMap<>(); 62 | context.put(CONNECTOR_TYPE, "elasticsearch"); // elasticsearch 63 | context.put(CONNECTOR_PROPERTY_VERSION, "1"); // backwards compatibility 64 | return context; 65 | } 66 | 67 | @Override 68 | public List supportedProperties() { 69 | List properties = new ArrayList<>(); 70 | for (ConfigOption esConfigOption : ElasticsearchTableOptions.ES_CONFIG_OPTIONS) { 71 | properties.add(esConfigOption.key()); 72 | } 73 | // schema 74 | properties.add(SCHEMA + ".#." + SCHEMA_DATA_TYPE); 75 | properties.add(SCHEMA + ".#." + SCHEMA_TYPE); 76 | properties.add(SCHEMA + ".#." + SCHEMA_NAME); 77 | // computed column 78 | properties.add(SCHEMA + ".#." + EXPR); 79 | 80 | // watermark 81 | properties.add(SCHEMA + "." + WATERMARK + ".#." + WATERMARK_ROWTIME); 82 | properties.add(SCHEMA + "." + WATERMARK + ".#." + WATERMARK_STRATEGY_EXPR); 83 | properties.add(SCHEMA + "." + WATERMARK + ".#." + WATERMARK_STRATEGY_DATA_TYPE); 84 | 85 | // table constraint 86 | properties.add(SCHEMA + "." + DescriptorProperties.PRIMARY_KEY_NAME); 87 | properties.add(SCHEMA + "." + DescriptorProperties.PRIMARY_KEY_COLUMNS); 88 | 89 | return properties; 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/org/apache/flink/connector/elasticsearch/table/ElasticsearchTableSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.elasticsearch.table; 20 | 21 | import org.apache.flink.api.common.serialization.DeserializationSchema; 22 | import org.apache.flink.api.java.DataSet; 23 | import org.apache.flink.api.java.ExecutionEnvironment; 24 | import org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat; 25 | import org.apache.flink.api.java.tuple.Tuple2; 26 | import org.apache.flink.api.java.typeutils.RowTypeInfo; 27 | import org.apache.flink.formats.common.TimestampFormat; 28 | import org.apache.flink.formats.json.JsonRowDataDeserializationSchema; 29 | import org.apache.flink.hadoopcompatibility.HadoopInputs; 30 | import org.apache.flink.hadoopcompatibility.OutputRowFunction; 31 | import org.apache.flink.streaming.api.datastream.DataStream; 32 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 33 | import org.apache.flink.table.api.TableSchema; 34 | import org.apache.flink.table.data.RowData; 35 | import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; 36 | import org.apache.flink.table.sources.BatchTableSource; 37 | import org.apache.flink.table.sources.StreamTableSource; 38 | import org.apache.flink.table.types.DataType; 39 | import org.apache.flink.table.types.inference.TypeTransformations; 40 | import org.apache.flink.table.types.logical.LogicalType; 41 | import org.apache.flink.table.types.logical.RowType; 42 | import org.apache.flink.table.types.utils.DataTypeUtils; 43 | import org.apache.flink.table.utils.TableConnectorUtils; 44 | import org.apache.flink.types.Row; 45 | import org.apache.hadoop.conf.Configuration; 46 | import org.apache.hadoop.io.Text; 47 | import org.apache.hadoop.mapreduce.Job; 48 | import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; 49 | import org.elasticsearch.hadoop.mr.EsInputFormat; 50 | import org.slf4j.Logger; 51 | import org.slf4j.LoggerFactory; 52 | 53 | import java.io.IOException; 54 | 55 | import static org.apache.flink.table.types.utils.TypeConversions.fromDataTypeToLegacyInfo; 56 | import static org.apache.flink.util.Preconditions.checkNotNull; 57 | 58 | /** 59 | * @author cclient 60 | */ 61 | public class ElasticsearchTableSource 62 | implements BatchTableSource, StreamTableSource { 63 | private static final Logger log = LoggerFactory.getLogger(ElasticsearchTableSource.class); 64 | 65 | private final TableSchema schema; 66 | private final int[] selectFields; 67 | private final DataType producedDataType; 68 | private final RowType rowType; 69 | private final DeserializationSchema deserializationSchema; 70 | private final HadoopInputFormat hadoopInputFormat; 71 | private final boolean failOnMissingField = false; 72 | private final boolean ignoreParseErrors = false; 73 | private final TimestampFormat timestampFormat = TimestampFormat.ISO_8601; 74 | 75 | private ElasticsearchTableSource(Configuration configuration, 76 | TableSchema schema) { 77 | this(configuration, schema, null); 78 | } 79 | 80 | private ElasticsearchTableSource(Configuration configuration, 81 | TableSchema schema, 82 | int[] selectFields) { 83 | this.schema = schema; 84 | this.selectFields = selectFields; 85 | final DataType[] schemaDataTypes = schema.getFieldDataTypes(); 86 | final String[] schemaFieldNames = schema.getFieldNames(); 87 | if (selectFields != null) { 88 | DataType[] dataTypes = new DataType[selectFields.length]; 89 | String[] fieldNames = new String[selectFields.length]; 90 | for (int i = 0; i < selectFields.length; i++) { 91 | dataTypes[i] = schemaDataTypes[selectFields[i]]; 92 | fieldNames[i] = schemaFieldNames[selectFields[i]]; 93 | } 94 | this.producedDataType = 95 | TableSchema.builder().fields(fieldNames, dataTypes).build().toRowDataType(); 96 | this.rowType = buildRowType(dataTypes, fieldNames); 97 | } else { 98 | this.producedDataType = schema.toRowDataType(); 99 | this.rowType = buildRowType(schema); 100 | } 101 | Job job = buildJob(configuration); 102 | this.hadoopInputFormat = HadoopInputs.createHadoopInput(new EsInputFormat(), Text.class, Text.class, job); 103 | deserializationSchema = new JsonRowDataDeserializationSchema(rowType, InternalTypeInfo.of(rowType), failOnMissingField, ignoreParseErrors, timestampFormat); 104 | } 105 | 106 | public static Builder builder() { 107 | return new Builder(); 108 | } 109 | 110 | private Job buildJob(Configuration configuration) { 111 | Job job = null; 112 | try { 113 | //use Text and json parse,do not use MapWritable 114 | configuration.set("es.output.json", "true"); 115 | //es metadata _index,_type,_id 116 | configuration.set("es.read.metadata", "true"); 117 | job = new Job(configuration); 118 | //this.job.waitForCompletion(true); 119 | } catch (IOException e) { 120 | e.printStackTrace(); 121 | } 122 | job.setInputFormatClass(EsInputFormat.class); 123 | job.setMapOutputKeyClass(Text.class); 124 | job.setMapOutputValueClass(Text.class); 125 | job.setOutputFormatClass(NullOutputFormat.class); 126 | return job; 127 | } 128 | 129 | public RowType buildRowType(TableSchema schema) { 130 | return buildRowType(schema.getFieldDataTypes(), schema.getFieldNames()); 131 | } 132 | 133 | public RowType buildRowType(DataType[] fieldDataTypes, String[] fieldNames) { 134 | LogicalType[] logicalTypes = new LogicalType[fieldDataTypes.length]; 135 | for (int i = 0; i < fieldDataTypes.length; i++) { 136 | logicalTypes[i] = fieldDataTypes[i].getLogicalType(); 137 | } 138 | return RowType.of(logicalTypes, fieldNames); 139 | } 140 | 141 | @Override 142 | public DataType getProducedDataType() { 143 | return producedDataType; 144 | } 145 | 146 | @Override 147 | public TableSchema getTableSchema() { 148 | return schema; 149 | } 150 | 151 | @Override 152 | public String explainSource() { 153 | final RowTypeInfo rowTypeInfo = (RowTypeInfo) fromDataTypeToLegacyInfo(producedDataType); 154 | return TableConnectorUtils.generateRuntimeName(getClass(), rowTypeInfo.getFieldNames()); 155 | } 156 | 157 | @Override 158 | public DataSet getDataSet(ExecutionEnvironment executionEnvironment) { 159 | ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); 160 | DataSet> input = 161 | env.createInput(this.hadoopInputFormat); 162 | DataSet rowDataSet = input.map(new OutputRowFunction(deserializationSchema, producedDataType)).returns(new RowTypeInfo(this.schema.getFieldTypes(), this.schema.getFieldNames())); 163 | return rowDataSet; 164 | } 165 | 166 | @Override 167 | public DataStream getDataStream(StreamExecutionEnvironment execEnv) { 168 | return execEnv.createInput(this.hadoopInputFormat) 169 | .map(new OutputRowFunction(deserializationSchema, producedDataType), new RowTypeInfo(this.schema.getFieldTypes(), this.schema.getFieldNames())); 170 | } 171 | 172 | /** 173 | * Builder for a {@link ElasticsearchTableSource}. 174 | */ 175 | public static class Builder { 176 | protected Configuration configuration; 177 | 178 | protected TableSchema schema; 179 | 180 | public static TableSchema normalizeTableSchema(TableSchema schema) { 181 | TableSchema.Builder physicalSchemaBuilder = TableSchema.builder(); 182 | schema.getTableColumns() 183 | .forEach( 184 | c -> { 185 | if (c.isPhysical()) { 186 | final DataType type = 187 | DataTypeUtils.transform( 188 | c.getType(), TypeTransformations.timeToSqlTypes()); 189 | physicalSchemaBuilder.field(c.getName(), type); 190 | } 191 | }); 192 | return physicalSchemaBuilder.build(); 193 | } 194 | 195 | /** 196 | * required, table schema of this table source. 197 | */ 198 | public Builder setSchema(TableSchema schema) { 199 | this.schema = normalizeTableSchema(schema); 200 | return this; 201 | } 202 | 203 | public Builder setConfiguration(Configuration configuration) { 204 | this.configuration = configuration; 205 | return this; 206 | } 207 | 208 | /** 209 | * Finalizes the configuration and checks validity. 210 | * 211 | * @return Configured JdbcTableSource 212 | */ 213 | public ElasticsearchTableSource build() { 214 | checkNotNull(schema, "No schema supplied."); 215 | return new ElasticsearchTableSource(configuration, schema); 216 | } 217 | } 218 | } 219 | -------------------------------------------------------------------------------- /src/main/java/org/apache/flink/hadoopcompatibility/OutputRowFunction.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.hadoopcompatibility; 2 | 3 | import org.apache.flink.api.common.functions.MapFunction; 4 | import org.apache.flink.api.common.serialization.DeserializationSchema; 5 | import org.apache.flink.api.java.tuple.Tuple2; 6 | import org.apache.flink.table.data.RowData; 7 | import org.apache.flink.table.data.conversion.RowRowConverter; 8 | import org.apache.flink.table.types.DataType; 9 | import org.apache.flink.types.Row; 10 | import org.apache.hadoop.io.BinaryComparable; 11 | 12 | /** 13 | * @author cclient 14 | */ 15 | public class OutputRowFunction implements MapFunction, Row> { 16 | 17 | 18 | private final DataType producedDataType; 19 | private final DeserializationSchema deserializationSchema; 20 | 21 | public OutputRowFunction(DeserializationSchema deserializationSchema, DataType producedDataType) { 22 | this.deserializationSchema = deserializationSchema; 23 | this.producedDataType = producedDataType; 24 | } 25 | 26 | public Row parseToRow(byte[] message) throws Exception { 27 | RowData rowData = deserializationSchema.deserialize(message); 28 | RowRowConverter rowRowConverter = RowRowConverter.create(producedDataType); 29 | rowRowConverter.toExternal(rowData); 30 | Row row = rowRowConverter.toExternal(rowData); 31 | return row; 32 | } 33 | 34 | @Override 35 | public Row map(Tuple2 vTuple2) throws Exception { 36 | return parseToRow(vTuple2.f1.getBytes()); 37 | } 38 | } -------------------------------------------------------------------------------- /src/main/resources/META-INF/services/org.apache.flink.table.factories.TableFactory: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | org.apache.flink.connector.elasticsearch.table.ElasticsearchSourceFactory 17 | -------------------------------------------------------------------------------- /src/test/java/org/apache/flink/connector/elasticsearch/table/ElasticsearchTableSourceTest.java: -------------------------------------------------------------------------------- 1 | package org.apache.flink.connector.elasticsearch.table; 2 | 3 | import org.apache.flink.api.java.ExecutionEnvironment; 4 | import org.apache.flink.table.api.*; 5 | import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; 6 | import org.apache.flink.test.util.AbstractTestBase; 7 | import org.apache.flink.types.Row; 8 | import org.junit.Before; 9 | import org.junit.Test; 10 | 11 | import java.util.*; 12 | import java.util.stream.Collectors; 13 | import java.util.stream.StreamSupport; 14 | 15 | public class ElasticsearchTableSourceTest extends AbstractTestBase { 16 | 17 | public static final String INPUT_TABLE = "flink_es_table"; 18 | ExecutionEnvironment env; 19 | BatchTableEnvironment tEnv; 20 | 21 | private static List manifestResults(TableResult result) { 22 | Iterator resultIterator = result.collect(); 23 | return StreamSupport.stream( 24 | Spliterators.spliteratorUnknownSize(resultIterator, Spliterator.ORDERED), 25 | false) 26 | .map(Row::toString) 27 | .collect(Collectors.toList()); 28 | } 29 | 30 | @Before 31 | public void before() { 32 | env = ExecutionEnvironment.getExecutionEnvironment(); 33 | tEnv = BatchTableEnvironment.create(env); 34 | String esResource = INPUT_TABLE + "/_doc"; 35 | String esNodes="127.0.0.1:9200"; 36 | String esPort="9200"; 37 | String esQuery = "?q=*"; 38 | String clientOnly = "false"; 39 | String discovery = "false"; 40 | String wanOnly = "true"; 41 | 42 | String inputRowStr = "(" + 43 | "_metadata ROW<_index STRING,_type STRING,_id STRING>," 44 | // "_metadata MAP,\n" 45 | + "int_key INT," 46 | + "int_array ARRAY," 47 | + "int_object MAP," 48 | + "int_nested ARRAY>" 49 | + "," 50 | 51 | + "string_key STRING," 52 | + "string_array ARRAY," 53 | + "string_object MAP," 54 | + "string_nested ARRAY>" 55 | + "," 56 | 57 | + "double_key DOUBLE," 58 | + "double_array ARRAY," 59 | + "double_object MAP," 60 | + "double_nested ARRAY>" 61 | + "," 62 | 63 | + "time_key TIMESTAMP," 64 | + "time_array ARRAY," 65 | + "time_object MAP," 66 | + "time_nested ARRAY>" 67 | + "," 68 | 69 | + "bool_key BOOLEAN," 70 | + "bool_array ARRAY," 71 | + "bool_object MAP," 72 | + "bool_nested ARRAY>" 73 | + ")"; 74 | 75 | 76 | String createInputTable = "CREATE TABLE " 77 | + INPUT_TABLE 78 | + inputRowStr 79 | + " WITH (" 80 | + " 'connector.type'='elasticsearch'," 81 | + " 'es.resource'='" + esResource + "'," 82 | + " 'es.nodes'='" + esNodes + "'," 83 | + " 'es.port'='" + esPort + "'," 84 | + " 'es.query'='" + esQuery + "'," 85 | + " 'es.nodes.client.only'='" + clientOnly + "'," 86 | + " 'es.nodes.discovery'='" + discovery + "'," 87 | + " 'es.nodes.wan.only'='" + wanOnly + "'" 88 | + ")"; 89 | System.out.println(createInputTable); 90 | tEnv.executeSql(createInputTable); 91 | } 92 | 93 | @Test 94 | public void testEsSource() { 95 | String selectColumes = "SELECT _index,_type,_id," + 96 | // String selectColumes="SELECT _metadata['_index'],_metadata['_type'],_metadata['_id'],\n" + 97 | "int_key,int_array,int_object,int_nested," + 98 | "int_key,int_array,int_object,int_nested," + 99 | "string_key,string_array,string_object,string_nested," + 100 | "time_key,time_array,time_object,time_nested," + 101 | "bool_key,bool_array,bool_object,bool_nested FROM " + INPUT_TABLE; 102 | String selectSql = selectColumes; 103 | System.out.println(selectSql); 104 | TableResult tableResult = tEnv.executeSql(selectSql); 105 | List results = manifestResults(tableResult); 106 | results.forEach(System.out::println); 107 | } 108 | } -------------------------------------------------------------------------------- /src/test/resources/META-INF/services/org.apache.flink.table.factories.TableFactory: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | org.apache.flink.connector.elasticsearch.table.ElasticsearchSourceFactory 17 | --------------------------------------------------------------------------------