├── .asf.yaml ├── .github └── workflows │ ├── build-doris-kafka-connector.yml │ ├── checkstyle.yaml │ ├── kafka2doris-e2ecase.yaml │ └── license-eyes.yml ├── .gitignore ├── .licenserc.yaml ├── LICENSE ├── NOTICE.txt ├── README.md ├── build.sh ├── docs ├── en │ └── Doris-Connector-JMX.md └── zh-CN │ └── Doris-Connector-JMX.md ├── format.sh ├── pom.xml └── src ├── main └── java │ └── org │ └── apache │ └── doris │ └── kafka │ └── connector │ ├── DorisSinkConnector.java │ ├── DorisSinkTask.java │ ├── cfg │ ├── DorisOptions.java │ └── DorisSinkConnectorConfig.java │ ├── connection │ ├── ConnectionProvider.java │ └── JdbcConnectionProvider.java │ ├── converter │ ├── ConverterMode.java │ ├── RecordDescriptor.java │ ├── RecordService.java │ ├── RecordTypeRegister.java │ ├── schema │ │ ├── SchemaChangeManager.java │ │ └── SchemaEvolutionMode.java │ └── type │ │ ├── AbstractDateType.java │ │ ├── AbstractGeometryType.java │ │ ├── AbstractTemporalType.java │ │ ├── AbstractTimeType.java │ │ ├── AbstractTimestampType.java │ │ ├── AbstractType.java │ │ ├── Type.java │ │ ├── connect │ │ ├── AbstractConnectMapType.java │ │ ├── AbstractConnectSchemaType.java │ │ ├── ConnectBooleanType.java │ │ ├── ConnectBytesType.java │ │ ├── ConnectDateType.java │ │ ├── ConnectDecimalType.java │ │ ├── ConnectFloat32Type.java │ │ ├── ConnectFloat64Type.java │ │ ├── ConnectInt16Type.java │ │ ├── ConnectInt32Type.java │ │ ├── ConnectInt64Type.java │ │ ├── ConnectInt8Type.java │ │ ├── ConnectMapToConnectStringType.java │ │ ├── ConnectStringType.java │ │ ├── ConnectTimeType.java │ │ └── ConnectTimestampType.java │ │ ├── debezium │ │ ├── AbstractDebeziumTimeType.java │ │ ├── AbstractDebeziumTimestampType.java │ │ ├── ArrayType.java │ │ ├── DateType.java │ │ ├── GeographyType.java │ │ ├── GeometryType.java │ │ ├── MicroTimeType.java │ │ ├── MicroTimestampType.java │ │ ├── NanoTimeType.java │ │ ├── NanoTimestampType.java │ │ ├── PointType.java │ │ ├── TimeType.java │ │ ├── TimestampType.java │ │ ├── VariableScaleDecimalType.java │ │ ├── ZonedTimeType.java │ │ └── ZonedTimestampType.java │ │ ├── doris │ │ ├── DorisType.java │ │ └── DorisTypeProperties.java │ │ └── util │ │ ├── DateTimeUtils.java │ │ ├── GeoUtils.java │ │ └── SchemaUtils.java │ ├── decode │ ├── DorisConverter.java │ ├── DorisJsonSchema.java │ └── avro │ │ └── DorisAvroConverter.java │ ├── exception │ ├── ArgumentsException.java │ ├── ConnectedFailedException.java │ ├── CopyLoadException.java │ ├── DataConverterException.java │ ├── DataDecodeException.java │ ├── DataFormatException.java │ ├── DorisException.java │ ├── SchemaChangeException.java │ ├── StreamLoadException.java │ └── UploadException.java │ ├── metrics │ ├── DorisConnectMonitor.java │ ├── MetricsJmxReporter.java │ └── MetricsUtil.java │ ├── model │ ├── BackendV2.java │ ├── BaseResponse.java │ ├── BehaviorOnNullValues.java │ ├── ColumnDescriptor.java │ ├── CopyIntoResp.java │ ├── KafkaRespContent.java │ ├── LoadOperation.java │ ├── RespContent.java │ ├── TableDescriptor.java │ └── doris │ │ ├── Field.java │ │ └── Schema.java │ ├── service │ ├── DorisCombinedSinkService.java │ ├── DorisDefaultSinkService.java │ ├── DorisSinkService.java │ ├── DorisSinkServiceFactory.java │ ├── DorisSystemService.java │ └── RestService.java │ ├── utils │ ├── BackendUtils.java │ ├── BackoffAndRetryUtils.java │ ├── ConfigCheckUtils.java │ ├── FileNameUtils.java │ ├── HttpPostBuilder.java │ ├── HttpPutBuilder.java │ ├── HttpUtils.java │ └── Version.java │ └── writer │ ├── AsyncStreamLoadWriter.java │ ├── CopyIntoWriter.java │ ├── CopySQLBuilder.java │ ├── DeliveryGuarantee.java │ ├── DorisWriter.java │ ├── LabelGenerator.java │ ├── LoadConstants.java │ ├── LoadStatus.java │ ├── PartitionBuffer.java │ ├── RecordBuffer.java │ ├── ResponseUtil.java │ ├── StreamLoadWriter.java │ ├── commit │ ├── DorisCommittable.java │ └── DorisCommitter.java │ └── load │ ├── AsyncDorisStreamLoad.java │ ├── CopyLoad.java │ ├── DataLoad.java │ ├── DefaultThreadFactory.java │ ├── DorisStreamLoad.java │ ├── GroupCommitMode.java │ └── LoadModel.java └── test ├── java └── org │ └── apache │ └── doris │ └── kafka │ └── connector │ ├── DistributedTest.java │ ├── StandaloneTest.java │ ├── cfg │ ├── TestDorisOptions.java │ └── TestDorisSinkConnectorConfig.java │ ├── converter │ └── TestRecordService.java │ ├── decode │ └── avro │ │ └── DorisAvroConverterTest.java │ ├── e2e │ ├── doris │ │ ├── DorisContainerService.java │ │ └── DorisContainerServiceImpl.java │ ├── kafka │ │ ├── KafkaContainerService.java │ │ └── KafkaContainerServiceImpl.java │ └── sink │ │ ├── AbstractKafka2DorisSink.java │ │ ├── avro │ │ ├── AbstractAvroE2ESinkTest.java │ │ └── AvroMsgE2ETest.java │ │ └── stringconverter │ │ ├── AbstractStringE2ESinkTest.java │ │ ├── DorisSinkFailoverSinkTest.java │ │ └── StringMsgE2ETest.java │ ├── service │ └── TestDorisSinkService.java │ ├── utils │ ├── ConfigCheckUtilsTest.java │ ├── FileNameConfigCheckUtilsTest.java │ └── JdbcTest.java │ └── writer │ ├── HttpEntityMock.java │ ├── HttpTestUtil.java │ ├── TestCopyIntoWriter.java │ ├── TestCopyLoad.java │ ├── TestCopySQL.java │ ├── TestRecordBuffer.java │ └── TestStreamLoadWriter.java └── resources ├── connect-distributed.properties ├── connect-standalone.properties ├── decode └── avro │ ├── product.avsc │ └── user.avsc ├── docker └── doris │ ├── be.conf │ └── fe.conf ├── doris-connector-sink.properties ├── doris-connector-sink.sh ├── e2e ├── avro_converter │ ├── confluent_avro_convert.json │ ├── confluent_avro_tab.sql │ ├── doris_avro_convert.json │ └── doris_avro_tab.sql ├── string_converter │ ├── combine_flush_connector.json │ ├── combine_flush_connector_2pc.json │ ├── combine_flush_tab.sql │ ├── combine_flush_tab_2pc.sql │ ├── debezium_dml_event.json │ ├── debezium_dml_event_tab.sql │ ├── full_types.json │ ├── full_types_debezium_ingestion.sql │ ├── group_commit_connector.json │ ├── group_commit_tab.sql │ ├── insert_partial_update_tab.sql │ ├── null_values_default.json │ ├── null_values_fail.json │ ├── null_values_ignore.json │ ├── null_values_tab.sql │ ├── partial_update.json │ ├── partial_update_tab.sql │ ├── string_msg_connector.json │ ├── string_msg_failover_connector.json │ ├── string_msg_failover_connector_uniq.json │ ├── string_msg_tab.sql │ ├── string_msg_tab_failover.sql │ ├── string_msg_tab_failover_uniq.sql │ ├── table_field_config.json │ ├── table_field_config1.sql │ ├── table_field_config2.sql │ ├── time_types.json │ └── time_types.sql └── transforms │ ├── multiple_transforms_chain.json │ ├── multiple_transforms_chain.sql │ ├── regex_router_transforms.json │ ├── regex_router_transforms.sql │ ├── rename_transforms.json │ └── rename_transforms.sql └── log4j.properties /.asf.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | github: 21 | description: Kafka Connector for Apache Doris 22 | homepage: https://doris.apache.org 23 | labels: 24 | - olap 25 | - mpp 26 | - apache 27 | - doris 28 | - kafka 29 | - kafka-connect 30 | - etl 31 | - data-streaming 32 | 33 | enabled_merge_buttons: 34 | squash: true 35 | merge: false 36 | rebase: false 37 | protected_branches: 38 | master: 39 | required_pull_request_reviews: 40 | dismiss_stale_reviews: true 41 | required_approving_review_count: 1 42 | features: 43 | issues: true 44 | projects: true 45 | 46 | notifications: 47 | issues: commits@doris.apache.org 48 | commits: commits@doris.apache.org 49 | pullrequests_status: commits@doris.apache.org 50 | 51 | -------------------------------------------------------------------------------- /.github/workflows/build-doris-kafka-connector.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | --- 21 | name: Build Doris Kafka Connector 22 | on: 23 | pull_request: 24 | push: 25 | 26 | jobs: 27 | build-extension: 28 | name: "Build Doris Kafka Connector" 29 | runs-on: ubuntu-latest 30 | defaults: 31 | run: 32 | shell: bash 33 | steps: 34 | - name: Checkout 35 | uses: actions/checkout@master 36 | 37 | - name: Setup java 38 | uses: actions/setup-java@v2 39 | with: 40 | distribution: adopt 41 | java-version: '8' 42 | 43 | - name: Build Doris Kafka Connector 44 | run: | 45 | mvn clean package -------------------------------------------------------------------------------- /.github/workflows/checkstyle.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | name: Code Style Checker 21 | 22 | on: 23 | pull_request: 24 | push: 25 | branches: 26 | - master 27 | jobs: 28 | java-checkstyle: 29 | name: "CheckStyle" 30 | runs-on: ubuntu-latest 31 | steps: 32 | - name: Checkout 33 | uses: actions/checkout@v3 34 | with: 35 | persist-credentials: false 36 | submodules: recursive 37 | 38 | - name: Setup java 39 | uses: actions/setup-java@v2 40 | with: 41 | distribution: adopt 42 | java-version: '8' 43 | 44 | - name: Run java checkstyle 45 | run: 46 | mvn spotless:check -------------------------------------------------------------------------------- /.github/workflows/kafka2doris-e2ecase.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | name: E2E Test CI 21 | on: 22 | pull_request: 23 | push: 24 | 25 | jobs: 26 | build-extension: 27 | name: "e2e test case" 28 | runs-on: ubuntu-latest 29 | defaults: 30 | run: 31 | shell: bash 32 | steps: 33 | - name: Checkout 34 | uses: actions/checkout@master 35 | 36 | - name: Setup java 37 | uses: actions/setup-java@v2 38 | with: 39 | distribution: adopt 40 | java-version: '8' 41 | 42 | - name: Doris E2E Test 43 | run: | 44 | mvn test -Dtest='org.apache.doris.kafka.connector.e2e.**' -------------------------------------------------------------------------------- /.github/workflows/license-eyes.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | --- 20 | name: License Check 21 | on: 22 | pull_request: 23 | push: 24 | branches: 25 | - master 26 | 27 | jobs: 28 | license-check: 29 | name: "License Check" 30 | runs-on: ubuntu-latest 31 | steps: 32 | - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" 33 | uses: actions/checkout@v2 34 | 35 | - name: Check License 36 | uses: apache/skywalking-eyes@v0.2.0 37 | with: 38 | config-path: ./.licenserc.yaml 39 | env: 40 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | 3 | .idea 4 | *.iml 5 | *.DS_Store 6 | .cache 7 | dist 8 | dependency-reduced-pom.xml 9 | -------------------------------------------------------------------------------- /.licenserc.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | header: 20 | license: 21 | spdx-id: Apache-2.0 22 | copyright-owner: Apache Software Foundation 23 | 24 | paths-ignore: 25 | - 'LICENSE' 26 | - '.gitignore' 27 | - 'src/test/resources/decode/avro/**' 28 | - 'src/test/resources/e2e/**' 29 | 30 | comment: on-failure -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | Apache Doris Kafka Connector 2 | Copyright 2024-2025 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # Doris-Kafka-Connector 21 | 22 | [Kafka Connect](https://docs.confluent.io/platform/current/connect/index.html) is a scalable and reliable tool for data transmission between Apache Kafka and other systems. Connectors can be defined Move large amounts of data in and out of Kafka. 23 | 24 | Doris provides the Sink Connector plug-in, which can write data from Kafka topics to Doris. 25 | More information about usage, please visit [Doris Kafka Connector](https://doris.apache.org/docs/dev/ecosystem/doris-kafka-connector) 26 | 27 | ## License 28 | 29 | [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0) 30 | 31 | ## How to Build 32 | 33 | After running the following command, the jar package of Doris-Kafka-Connector will be generated in the dist directory. 34 | ``` 35 | sh build.sh 36 | ``` 37 | 38 | ## Code formatting 39 | Doris-Kafka-Connector uses the AOSP style in google-java-format version 1.7 as the formatting style of the project code. 40 | 41 | When you need to format your code, you have two formatting options: 42 | 43 | - Execute `sh format.sh` under the project 44 | - Execute `mvn spotless:apply` under the project 45 | 46 | After executing the above formatting command, you can use `mvn spotless:check` to check whether the code format meets the requirements. -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | mvn clean package -Dmaven.test.skip=true 21 | 22 | # Copy target files into `dist` directory 23 | rm -rf dist 24 | mkdir -p dist 25 | 26 | cp -r target/doris-kafka-connector-25.0.0-SNAPSHOT.jar dist 27 | -------------------------------------------------------------------------------- /format.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, 15 | # software distributed under the License is distributed on an 16 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | # KIND, either express or implied. See the License for the 18 | # specific language governing permissions and limitations 19 | # under the License. 20 | # 21 | 22 | set -euo pipefail 23 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 24 | cd "$SCRIPT_DIR" 25 | 26 | DOWNLOAD_URL="https://github.com/google/google-java-format/releases/download/google-java-format-1.7/google-java-format-1.7-all-deps.jar" 27 | JAR_FILE="./.cache/google-java-format-1.7-all-deps.jar" 28 | 29 | if [ ! -f "${JAR_FILE}" ]; then 30 | mkdir -p "$(dirname "${JAR_FILE}")" 31 | echo "Downloading Google Java format to ${JAR_FILE}" 32 | curl -# -L --fail "${DOWNLOAD_URL}" --output "${JAR_FILE}" 33 | fi 34 | 35 | if ! command -v java > /dev/null; then 36 | echo "Java not installed." 37 | exit 1 38 | fi 39 | echo "Running Google Java Format" 40 | find ./src -type f -name "*.java" -print0 | xargs -0 java -jar "${JAR_FILE}" --aosp --replace --set-exit-if-changed && echo "OK" 41 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/connection/ConnectionProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.connection; 21 | 22 | import java.sql.Connection; 23 | 24 | public interface ConnectionProvider { 25 | 26 | /** Get existing connection or establish an new one if there is none. */ 27 | Connection getOrEstablishConnection() throws Exception; 28 | 29 | /** Close possible existing connection. */ 30 | void closeConnection(); 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/ConverterMode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.converter; 21 | 22 | public enum ConverterMode { 23 | NORMAL("normal"), 24 | 25 | // kafka upstream data comes from debezium 26 | DEBEZIUM_INGESTION("debezium_ingestion"); 27 | 28 | private final String name; 29 | 30 | ConverterMode(String name) { 31 | this.name = name; 32 | } 33 | 34 | public static ConverterMode of(String name) { 35 | return ConverterMode.valueOf(name.toUpperCase()); 36 | } 37 | 38 | public String getName() { 39 | return name; 40 | } 41 | 42 | public static String[] instances() { 43 | return new String[] {NORMAL.name, DEBEZIUM_INGESTION.name}; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/schema/SchemaEvolutionMode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.converter.schema; 21 | 22 | public enum SchemaEvolutionMode { 23 | NONE("none"), 24 | 25 | BASIC("basic"); 26 | 27 | private final String name; 28 | 29 | SchemaEvolutionMode(String name) { 30 | this.name = name; 31 | } 32 | 33 | public static SchemaEvolutionMode of(String name) { 34 | return SchemaEvolutionMode.valueOf(name.toUpperCase()); 35 | } 36 | 37 | public String getName() { 38 | return name; 39 | } 40 | 41 | public static String[] instances() { 42 | return new String[] {NONE.name, BASIC.name}; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/AbstractDateType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type; 20 | 21 | import org.apache.doris.kafka.connector.converter.type.doris.DorisType; 22 | import org.apache.kafka.connect.data.Schema; 23 | 24 | /** An abstract base class for all temporal date implementations of {@link Type}. */ 25 | public abstract class AbstractDateType extends AbstractTemporalType { 26 | 27 | @Override 28 | public String getTypeName(Schema schema) { 29 | return DorisType.DATE; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/AbstractGeometryType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.converter.type; 21 | 22 | import org.apache.doris.kafka.connector.converter.type.doris.DorisType; 23 | import org.apache.kafka.connect.data.Schema; 24 | 25 | public abstract class AbstractGeometryType extends AbstractType { 26 | @Override 27 | public String getTypeName(Schema schema) { 28 | return DorisType.STRING; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/AbstractTemporalType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type; 20 | 21 | import java.time.ZoneId; 22 | import java.util.TimeZone; 23 | import org.apache.doris.kafka.connector.cfg.DorisOptions; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | /** An abstract base class for all temporal implementations of {@link Type}. */ 28 | public abstract class AbstractTemporalType extends AbstractType { 29 | 30 | private static final Logger LOGGER = LoggerFactory.getLogger(AbstractTemporalType.class); 31 | 32 | private TimeZone databaseTimeZone; 33 | 34 | @Override 35 | public void configure(DorisOptions dorisOptions) { 36 | final String databaseTimeZone = dorisOptions.getDatabaseTimeZone(); 37 | try { 38 | this.databaseTimeZone = TimeZone.getTimeZone(ZoneId.of(databaseTimeZone)); 39 | } catch (Exception e) { 40 | LOGGER.error( 41 | "Failed to resolve time zone '{}', please specify a correct time zone value", 42 | databaseTimeZone, 43 | e); 44 | throw e; 45 | } 46 | } 47 | 48 | protected TimeZone getDatabaseTimeZone() { 49 | return databaseTimeZone; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/AbstractTimeType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type; 20 | 21 | import java.util.Optional; 22 | import org.apache.doris.kafka.connector.converter.type.doris.DorisType; 23 | import org.apache.doris.kafka.connector.converter.type.doris.DorisTypeProperties; 24 | import org.apache.kafka.connect.data.Schema; 25 | 26 | /** An abstract temporal implementation of {@link Type} for {@code TIME} based columns. */ 27 | public abstract class AbstractTimeType extends AbstractTemporalType { 28 | 29 | @Override 30 | public String getTypeName(Schema schema) { 31 | // NOTE: 32 | // The MySQL connector does not use the __debezium.source.column.scale parameter to pass 33 | // the time column's precision but instead uses the __debezium.source.column.length key 34 | // which differs from all other connector implementations. 35 | // 36 | final int precision = getTimePrecision(schema); 37 | return String.format( 38 | "%s(%s)", 39 | DorisType.DATETIME, 40 | Math.min(precision, DorisTypeProperties.MAX_SUPPORTED_DATE_TIME_PRECISION)); 41 | } 42 | 43 | protected int getTimePrecision(Schema schema) { 44 | final String length = getSourceColumnLength(schema).orElse("0"); 45 | final Optional scale = getSourceColumnPrecision(schema); 46 | return scale.map(Integer::parseInt).orElseGet(() -> Integer.parseInt(length)); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/AbstractTimestampType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type; 20 | 21 | import java.util.Optional; 22 | import org.apache.doris.kafka.connector.converter.type.doris.DorisType; 23 | import org.apache.doris.kafka.connector.converter.type.doris.DorisTypeProperties; 24 | import org.apache.kafka.connect.data.Schema; 25 | 26 | /** An abstract temporal implementation of {@link Type} for {@code TIMESTAMP} based columns. */ 27 | public abstract class AbstractTimestampType extends AbstractTemporalType { 28 | 29 | @Override 30 | public String getTypeName(Schema schema) { 31 | final int precision = getTimePrecision(schema); 32 | return String.format( 33 | "%s(%s)", 34 | DorisType.DATETIME, 35 | Math.min(precision, DorisTypeProperties.MAX_SUPPORTED_DATE_TIME_PRECISION)); 36 | } 37 | 38 | protected int getTimePrecision(Schema schema) { 39 | final String length = getSourceColumnLength(schema).orElse("0"); 40 | final Optional scale = getSourceColumnPrecision(schema); 41 | return scale.map(Integer::parseInt).orElseGet(() -> Integer.parseInt(length)); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/AbstractType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type; 20 | 21 | import java.util.Objects; 22 | import java.util.Optional; 23 | import org.apache.doris.kafka.connector.cfg.DorisOptions; 24 | import org.apache.doris.kafka.connector.converter.type.util.SchemaUtils; 25 | import org.apache.kafka.connect.data.Schema; 26 | 27 | /** An abstract implementation of {@link Type}, which all types should extend. */ 28 | public abstract class AbstractType implements Type { 29 | 30 | @Override 31 | public void configure(DorisOptions dorisOptions) {} 32 | 33 | @Override 34 | public Object getValue(Object sourceValue) { 35 | return sourceValue; 36 | } 37 | 38 | @Override 39 | public boolean isNumber() { 40 | return false; 41 | } 42 | 43 | @Override 44 | public String toString() { 45 | return getClass().getSimpleName(); 46 | } 47 | 48 | protected Optional getSchemaParameter(Schema schema, String parameterName) { 49 | if (!Objects.isNull(schema.parameters())) { 50 | return Optional.ofNullable(schema.parameters().get(parameterName)); 51 | } 52 | return Optional.empty(); 53 | } 54 | 55 | protected Optional getSourceColumnType(Schema schema) { 56 | return SchemaUtils.getSourceColumnType(schema); 57 | } 58 | 59 | protected Optional getSourceColumnLength(Schema schema) { 60 | return SchemaUtils.getSourceColumnLength(schema); 61 | } 62 | 63 | protected Optional getSourceColumnPrecision(Schema schema) { 64 | return SchemaUtils.getSourceColumnPrecision(schema); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/Type.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type; 20 | 21 | import org.apache.doris.kafka.connector.cfg.DorisOptions; 22 | import org.apache.kafka.connect.data.Schema; 23 | 24 | /** 25 | * A type indicates the type of each column of kafka record, including various column types of 26 | * debezium and connect. 27 | */ 28 | public interface Type { 29 | 30 | /** Allows a type to perform initialization/configuration tasks based on user configs. */ 31 | void configure(DorisOptions dorisOptions); 32 | 33 | /** 34 | * Returns the names that this type will be mapped as. 35 | * 36 | *

For example, when creating a custom mapping for {@code io.debezium.data.Bits}, a type 37 | * could be registered using the {@code LOGICAL_NAME} of the schema if the type is to be used 38 | * when a schema name is identified; otherwise it could be registered as the raw column type 39 | * when column type propagation is enabled. 40 | */ 41 | String[] getRegistrationKeys(); 42 | 43 | /** Get the actual converted value based on the column type. */ 44 | Object getValue(Object sourceValue); 45 | 46 | default Object getValue(Object sourcevalue, Schema schema) { 47 | return getValue(sourcevalue); 48 | } 49 | 50 | String getTypeName(Schema schema); 51 | 52 | boolean isNumber(); 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/connect/AbstractConnectMapType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.connect; 20 | 21 | import com.fasterxml.jackson.core.JsonProcessingException; 22 | import com.fasterxml.jackson.databind.ObjectMapper; 23 | import org.apache.kafka.connect.errors.ConnectException; 24 | 25 | /** 26 | * An implementation of {@link org.apache.doris.kafka.connector.converter.type.Type} for {@code MAP} 27 | * schema types. 28 | */ 29 | public abstract class AbstractConnectMapType extends AbstractConnectSchemaType { 30 | 31 | private static final ObjectMapper MAPPER = new ObjectMapper(); 32 | 33 | @Override 34 | public String[] getRegistrationKeys() { 35 | return new String[] {"MAP"}; 36 | } 37 | 38 | protected String mapToJsonString(Object value) { 39 | try { 40 | return MAPPER.writeValueAsString(value); 41 | } catch (JsonProcessingException e) { 42 | throw new ConnectException("Failed to deserialize MAP data to JSON", e); 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/connect/AbstractConnectSchemaType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.connect; 20 | 21 | import org.apache.doris.kafka.connector.converter.type.AbstractType; 22 | 23 | /** 24 | * An abstract implementation of {@link org.apache.doris.kafka.connector.converter.type.Type} that 25 | * all Kafka Connect based schema types should be derived. 26 | * 27 | *

This abstract implementation is used as a marker object to designate types that are operating 28 | * on the raw {@link org.apache.kafka.connect.data.Schema.Type} values rather than custom schema 29 | * types that are contributed by Kafka Connect, such as Date or Time, or other third parties such as 30 | * Debezium. 31 | */ 32 | public abstract class AbstractConnectSchemaType extends AbstractType {} 33 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/connect/ConnectBooleanType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.connect; 20 | 21 | import org.apache.doris.kafka.connector.converter.type.doris.DorisType; 22 | import org.apache.kafka.connect.data.Schema; 23 | 24 | public class ConnectBooleanType extends AbstractConnectSchemaType { 25 | 26 | public static final ConnectBooleanType INSTANCE = new ConnectBooleanType(); 27 | 28 | @Override 29 | public String[] getRegistrationKeys() { 30 | return new String[] {"BOOLEAN"}; 31 | } 32 | 33 | @Override 34 | public String getTypeName(Schema schema) { 35 | return DorisType.BOOLEAN; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/connect/ConnectBytesType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.connect; 20 | 21 | import java.nio.ByteBuffer; 22 | import org.apache.doris.kafka.connector.converter.type.doris.DorisType; 23 | import org.apache.kafka.connect.data.Schema; 24 | 25 | public class ConnectBytesType extends AbstractConnectSchemaType { 26 | 27 | public static final ConnectBytesType INSTANCE = new ConnectBytesType(); 28 | 29 | @Override 30 | public String[] getRegistrationKeys() { 31 | return new String[] {"BYTES"}; 32 | } 33 | 34 | @Override 35 | public Object getValue(Object sourceValue) { 36 | if (sourceValue == null) { 37 | return null; 38 | } 39 | return bytesToHexString(getByteArrayFromValue(sourceValue)); 40 | } 41 | 42 | @Override 43 | public String getTypeName(Schema schema) { 44 | return DorisType.STRING; 45 | } 46 | 47 | private byte[] getByteArrayFromValue(Object value) { 48 | byte[] byteArray = null; 49 | if (value instanceof ByteBuffer) { 50 | final ByteBuffer buffer = ((ByteBuffer) value).slice(); 51 | byteArray = new byte[buffer.remaining()]; 52 | buffer.get(byteArray); 53 | } else if (value instanceof byte[]) { 54 | byteArray = (byte[]) value; 55 | } 56 | return byteArray; 57 | } 58 | 59 | /** Convert hexadecimal byte array to string */ 60 | private String bytesToHexString(byte[] bytes) { 61 | StringBuilder sb = new StringBuilder(); 62 | for (byte b : bytes) { 63 | sb.append(String.format("%02X", b)); 64 | } 65 | return sb.toString(); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/connect/ConnectDateType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.connect; 20 | 21 | import org.apache.doris.kafka.connector.converter.type.AbstractDateType; 22 | import org.apache.doris.kafka.connector.converter.type.util.DateTimeUtils; 23 | import org.apache.kafka.connect.data.Date; 24 | import org.apache.kafka.connect.errors.ConnectException; 25 | 26 | public class ConnectDateType extends AbstractDateType { 27 | 28 | public static final ConnectDateType INSTANCE = new ConnectDateType(); 29 | 30 | @Override 31 | public String[] getRegistrationKeys() { 32 | return new String[] {Date.LOGICAL_NAME}; 33 | } 34 | 35 | @Override 36 | public Object getValue(Object sourceValue) { 37 | if (sourceValue == null) { 38 | return null; 39 | } 40 | if (sourceValue instanceof java.util.Date) { 41 | return DateTimeUtils.toLocalDateFromDate((java.util.Date) sourceValue); 42 | } 43 | throw new ConnectException( 44 | String.format( 45 | "Unexpected %s value '%s' with type '%s'", 46 | getClass().getSimpleName(), sourceValue, sourceValue.getClass().getName())); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/connect/ConnectDecimalType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.connect; 20 | 21 | import org.apache.doris.kafka.connector.converter.type.AbstractType; 22 | import org.apache.doris.kafka.connector.converter.type.doris.DorisType; 23 | import org.apache.kafka.connect.data.Decimal; 24 | import org.apache.kafka.connect.data.Schema; 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | 28 | public class ConnectDecimalType extends AbstractType { 29 | 30 | private static final Logger LOGGER = LoggerFactory.getLogger(ConnectDecimalType.class); 31 | 32 | public static final ConnectDecimalType INSTANCE = new ConnectDecimalType(); 33 | 34 | @Override 35 | public String[] getRegistrationKeys() { 36 | return new String[] {Decimal.LOGICAL_NAME}; 37 | } 38 | 39 | @Override 40 | public String getTypeName(Schema schema) { 41 | int scale = Integer.parseInt(getSchemaParameter(schema, "scale").orElse("0")); 42 | int precision = 43 | Integer.parseInt( 44 | getSchemaParameter(schema, "connect.decimal.precision").orElse("0")); 45 | return precision <= 38 46 | ? String.format("%s(%s,%s)", DorisType.DECIMAL, precision, Math.max(scale, 0)) 47 | : DorisType.STRING; 48 | } 49 | 50 | @Override 51 | public boolean isNumber() { 52 | return true; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/connect/ConnectFloat32Type.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.connect; 20 | 21 | import org.apache.doris.kafka.connector.converter.type.doris.DorisType; 22 | import org.apache.kafka.connect.data.Schema; 23 | 24 | public class ConnectFloat32Type extends AbstractConnectSchemaType { 25 | 26 | public static final ConnectFloat32Type INSTANCE = new ConnectFloat32Type(); 27 | 28 | @Override 29 | public String[] getRegistrationKeys() { 30 | return new String[] {"FLOAT32"}; 31 | } 32 | 33 | @Override 34 | public String getTypeName(Schema schema) { 35 | return DorisType.FLOAT; 36 | } 37 | 38 | @Override 39 | public boolean isNumber() { 40 | return true; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/connect/ConnectFloat64Type.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.connect; 20 | 21 | import org.apache.doris.kafka.connector.converter.type.doris.DorisType; 22 | import org.apache.kafka.connect.data.Schema; 23 | 24 | public class ConnectFloat64Type extends AbstractConnectSchemaType { 25 | 26 | public static final ConnectFloat64Type INSTANCE = new ConnectFloat64Type(); 27 | 28 | @Override 29 | public String[] getRegistrationKeys() { 30 | return new String[] {"FLOAT64"}; 31 | } 32 | 33 | @Override 34 | public String getTypeName(Schema schema) { 35 | return DorisType.DOUBLE; 36 | } 37 | 38 | @Override 39 | public boolean isNumber() { 40 | return true; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/connect/ConnectInt16Type.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.connect; 20 | 21 | import org.apache.doris.kafka.connector.converter.type.doris.DorisType; 22 | import org.apache.kafka.connect.data.Schema; 23 | 24 | public class ConnectInt16Type extends AbstractConnectSchemaType { 25 | 26 | public static final ConnectInt16Type INSTANCE = new ConnectInt16Type(); 27 | 28 | @Override 29 | public String[] getRegistrationKeys() { 30 | return new String[] {"INT16"}; 31 | } 32 | 33 | @Override 34 | public String getTypeName(Schema schema) { 35 | return DorisType.SMALLINT; 36 | } 37 | 38 | @Override 39 | public boolean isNumber() { 40 | return true; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/connect/ConnectInt32Type.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.connect; 20 | 21 | import org.apache.doris.kafka.connector.converter.type.doris.DorisType; 22 | import org.apache.kafka.connect.data.Schema; 23 | 24 | public class ConnectInt32Type extends AbstractConnectSchemaType { 25 | 26 | public static final ConnectInt32Type INSTANCE = new ConnectInt32Type(); 27 | 28 | @Override 29 | public String[] getRegistrationKeys() { 30 | return new String[] {"INT32"}; 31 | } 32 | 33 | @Override 34 | public String getTypeName(Schema schema) { 35 | return DorisType.INT; 36 | } 37 | 38 | @Override 39 | public boolean isNumber() { 40 | return true; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/connect/ConnectInt64Type.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.connect; 20 | 21 | import org.apache.doris.kafka.connector.converter.type.doris.DorisType; 22 | import org.apache.kafka.connect.data.Schema; 23 | 24 | public class ConnectInt64Type extends AbstractConnectSchemaType { 25 | 26 | public static final ConnectInt64Type INSTANCE = new ConnectInt64Type(); 27 | 28 | @Override 29 | public String[] getRegistrationKeys() { 30 | return new String[] {"INT64"}; 31 | } 32 | 33 | @Override 34 | public String getTypeName(Schema schema) { 35 | return DorisType.BIGINT; 36 | } 37 | 38 | @Override 39 | public boolean isNumber() { 40 | return true; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/connect/ConnectInt8Type.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.connect; 20 | 21 | import org.apache.doris.kafka.connector.converter.type.doris.DorisType; 22 | import org.apache.kafka.connect.data.Schema; 23 | 24 | public class ConnectInt8Type extends AbstractConnectSchemaType { 25 | 26 | public static final ConnectInt8Type INSTANCE = new ConnectInt8Type(); 27 | 28 | @Override 29 | public String[] getRegistrationKeys() { 30 | return new String[] {"INT8"}; 31 | } 32 | 33 | @Override 34 | public String getTypeName(Schema schema) { 35 | return DorisType.TINYINT; 36 | } 37 | 38 | @Override 39 | public boolean isNumber() { 40 | return true; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/connect/ConnectMapToConnectStringType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.connect; 20 | 21 | import java.util.Map; 22 | import org.apache.kafka.connect.data.Schema; 23 | 24 | public class ConnectMapToConnectStringType extends AbstractConnectMapType { 25 | 26 | public static final ConnectMapToConnectStringType INSTANCE = 27 | new ConnectMapToConnectStringType(); 28 | 29 | @Override 30 | public String getTypeName(Schema schema) { 31 | return ConnectStringType.INSTANCE.getTypeName(schema); 32 | } 33 | 34 | @Override 35 | public Object getValue(Object sourceValue) { 36 | if (sourceValue instanceof Map) { 37 | sourceValue = mapToJsonString(sourceValue); 38 | } 39 | return ConnectStringType.INSTANCE.getValue(sourceValue); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/connect/ConnectStringType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.connect; 20 | 21 | import org.apache.doris.kafka.connector.converter.type.doris.DorisType; 22 | import org.apache.doris.kafka.connector.converter.type.doris.DorisTypeProperties; 23 | import org.apache.kafka.connect.data.Schema; 24 | 25 | /** 26 | * An implementation of {@link org.apache.doris.kafka.connector.converter.type.Type} that supports 27 | * {@code STRING} connect schema types. 28 | */ 29 | public class ConnectStringType extends AbstractConnectSchemaType { 30 | 31 | public static final ConnectStringType INSTANCE = new ConnectStringType(); 32 | 33 | @Override 34 | public String getTypeName(Schema schema) { 35 | int columnLength = getColumnLength(schema); 36 | if (columnLength > 0) { 37 | return columnLength * 3 > DorisTypeProperties.MAX_VARCHAR_SIZE 38 | ? DorisType.STRING 39 | : String.format("%s(%s)", DorisType.VARCHAR, columnLength * 3); 40 | } 41 | return DorisType.STRING; 42 | } 43 | 44 | @Override 45 | public String[] getRegistrationKeys() { 46 | return new String[] {"STRING"}; 47 | } 48 | 49 | private int getColumnLength(Schema schema) { 50 | return Integer.parseInt(getSourceColumnLength(schema).orElse("0")); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/connect/ConnectTimeType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.connect; 20 | 21 | import java.time.LocalDate; 22 | import java.time.LocalDateTime; 23 | import java.time.LocalTime; 24 | import java.util.Date; 25 | import org.apache.doris.kafka.connector.converter.type.AbstractTimeType; 26 | import org.apache.doris.kafka.connector.converter.type.util.DateTimeUtils; 27 | import org.apache.kafka.connect.data.Time; 28 | import org.apache.kafka.connect.errors.ConnectException; 29 | 30 | public class ConnectTimeType extends AbstractTimeType { 31 | 32 | public static final ConnectTimeType INSTANCE = new ConnectTimeType(); 33 | 34 | @Override 35 | public String[] getRegistrationKeys() { 36 | return new String[] {Time.LOGICAL_NAME}; 37 | } 38 | 39 | @Override 40 | public Object getValue(Object sourceValue) { 41 | if (sourceValue == null) { 42 | return null; 43 | } 44 | if (sourceValue instanceof Date) { 45 | 46 | final LocalTime localTime = DateTimeUtils.toLocalTimeFromUtcDate((Date) sourceValue); 47 | final LocalDateTime localDateTime = localTime.atDate(LocalDate.now()); 48 | return localDateTime.toLocalTime(); 49 | } 50 | 51 | throw new ConnectException( 52 | String.format( 53 | "Unexpected %s value '%s' with type '%s'", 54 | getClass().getSimpleName(), sourceValue, sourceValue.getClass().getName())); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/connect/ConnectTimestampType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.connect; 20 | 21 | import org.apache.doris.kafka.connector.converter.type.AbstractTimestampType; 22 | import org.apache.doris.kafka.connector.converter.type.util.DateTimeUtils; 23 | import org.apache.kafka.connect.data.Timestamp; 24 | import org.apache.kafka.connect.errors.ConnectException; 25 | 26 | public class ConnectTimestampType extends AbstractTimestampType { 27 | 28 | public static final ConnectTimestampType INSTANCE = new ConnectTimestampType(); 29 | 30 | @Override 31 | public String[] getRegistrationKeys() { 32 | return new String[] {Timestamp.LOGICAL_NAME}; 33 | } 34 | 35 | @Override 36 | public Object getValue(Object sourceValue) { 37 | if (sourceValue == null) { 38 | return null; 39 | } 40 | if (sourceValue instanceof java.util.Date) { 41 | return DateTimeUtils.toLocalDateTimeFromDate((java.util.Date) sourceValue); 42 | } 43 | 44 | throw new ConnectException( 45 | String.format( 46 | "Unexpected %s value '%s' with type '%s'", 47 | getClass().getSimpleName(), sourceValue, sourceValue.getClass().getName())); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/debezium/AbstractDebeziumTimeType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.debezium; 20 | 21 | import java.time.LocalTime; 22 | import java.time.format.DateTimeFormatter; 23 | import org.apache.doris.kafka.connector.converter.type.AbstractTimeType; 24 | import org.apache.kafka.connect.errors.ConnectException; 25 | 26 | public abstract class AbstractDebeziumTimeType extends AbstractTimeType { 27 | 28 | @Override 29 | public Object getValue(Object sourceValue) { 30 | if (sourceValue == null) { 31 | return null; 32 | } 33 | if (sourceValue instanceof Number) { 34 | final LocalTime localTime = getLocalTime((Number) sourceValue); 35 | return String.format("%s", DateTimeFormatter.ISO_TIME.format(localTime)); 36 | } 37 | throw new ConnectException( 38 | String.format( 39 | "Unexpected %s value '%s' with type '%s'", 40 | getClass().getSimpleName(), sourceValue, sourceValue.getClass().getName())); 41 | } 42 | 43 | protected abstract LocalTime getLocalTime(Number value); 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/debezium/AbstractDebeziumTimestampType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.debezium; 20 | 21 | import java.time.LocalDateTime; 22 | import org.apache.doris.kafka.connector.converter.type.AbstractTimestampType; 23 | import org.apache.kafka.connect.errors.ConnectException; 24 | 25 | public abstract class AbstractDebeziumTimestampType extends AbstractTimestampType { 26 | 27 | @Override 28 | public Object getValue(Object sourceValue) { 29 | if (sourceValue == null) { 30 | return null; 31 | } 32 | if (sourceValue instanceof Number) { 33 | return getLocalDateTime(((Number) sourceValue).longValue()); 34 | } 35 | throw new ConnectException( 36 | String.format( 37 | "Unexpected %s value '%s' with type '%s'", 38 | getClass().getSimpleName(), sourceValue, sourceValue.getClass().getName())); 39 | } 40 | 41 | protected abstract LocalDateTime getLocalDateTime(long value); 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/debezium/DateType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.debezium; 20 | 21 | import io.debezium.time.Date; 22 | import org.apache.doris.kafka.connector.converter.type.AbstractDateType; 23 | import org.apache.doris.kafka.connector.converter.type.util.DateTimeUtils; 24 | import org.apache.kafka.connect.errors.ConnectException; 25 | 26 | public class DateType extends AbstractDateType { 27 | 28 | public static final DateType INSTANCE = new DateType(); 29 | 30 | @Override 31 | public String[] getRegistrationKeys() { 32 | return new String[] {Date.SCHEMA_NAME}; 33 | } 34 | 35 | @Override 36 | public Object getValue(Object sourceValue) { 37 | if (sourceValue == null) { 38 | return null; 39 | } 40 | if (sourceValue instanceof Number) { 41 | return DateTimeUtils.toLocalDateOfEpochDays(((Number) sourceValue).longValue()); 42 | } 43 | 44 | throw new ConnectException( 45 | String.format( 46 | "Unexpected %s value '%s' with type '%s'", 47 | getClass().getSimpleName(), sourceValue, sourceValue.getClass().getName())); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/debezium/GeographyType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.converter.type.debezium; 21 | 22 | import io.debezium.data.geometry.Geography; 23 | import org.apache.doris.kafka.connector.converter.type.AbstractGeometryType; 24 | 25 | public class GeographyType extends AbstractGeometryType { 26 | public static final GeographyType INSTANCE = new GeographyType(); 27 | 28 | @Override 29 | public String[] getRegistrationKeys() { 30 | return new String[] {Geography.LOGICAL_NAME}; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/debezium/GeometryType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.converter.type.debezium; 21 | 22 | import io.debezium.data.geometry.Geometry; 23 | import org.apache.doris.kafka.connector.converter.type.AbstractGeometryType; 24 | import org.apache.doris.kafka.connector.converter.type.util.GeoUtils; 25 | import org.apache.kafka.connect.data.Struct; 26 | 27 | public class GeometryType extends AbstractGeometryType { 28 | public static final GeometryType INSTANCE = new GeometryType(); 29 | 30 | @Override 31 | public String[] getRegistrationKeys() { 32 | return new String[] {Geometry.LOGICAL_NAME}; 33 | } 34 | 35 | @Override 36 | public Object getValue(Object sourceValue) { 37 | if (sourceValue == null) { 38 | return null; 39 | } 40 | 41 | if (sourceValue instanceof Struct) { 42 | return GeoUtils.handleGeoStructData(sourceValue); 43 | } 44 | 45 | return sourceValue; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/debezium/MicroTimeType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.debezium; 20 | 21 | import io.debezium.time.MicroTime; 22 | import java.time.LocalTime; 23 | import org.apache.doris.kafka.connector.converter.type.util.DateTimeUtils; 24 | 25 | public class MicroTimeType extends AbstractDebeziumTimeType { 26 | 27 | public static final MicroTimeType INSTANCE = new MicroTimeType(); 28 | 29 | @Override 30 | public String[] getRegistrationKeys() { 31 | return new String[] {MicroTime.SCHEMA_NAME}; 32 | } 33 | 34 | @Override 35 | protected LocalTime getLocalTime(Number value) { 36 | return DateTimeUtils.toLocalTimeFromDurationMicroseconds(value.longValue()); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/debezium/MicroTimestampType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.debezium; 20 | 21 | import io.debezium.time.MicroTimestamp; 22 | import java.time.LocalDateTime; 23 | import org.apache.doris.kafka.connector.converter.type.util.DateTimeUtils; 24 | 25 | public class MicroTimestampType extends AbstractDebeziumTimestampType { 26 | 27 | public static final MicroTimestampType INSTANCE = new MicroTimestampType(); 28 | 29 | @Override 30 | public String[] getRegistrationKeys() { 31 | return new String[] {MicroTimestamp.SCHEMA_NAME}; 32 | } 33 | 34 | @Override 35 | protected LocalDateTime getLocalDateTime(long value) { 36 | return DateTimeUtils.toLocalDateTimeFromInstantEpochMicros(value); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/debezium/NanoTimeType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.debezium; 20 | 21 | import io.debezium.time.NanoTime; 22 | import java.time.LocalTime; 23 | import org.apache.doris.kafka.connector.converter.type.util.DateTimeUtils; 24 | 25 | public class NanoTimeType extends AbstractDebeziumTimeType { 26 | 27 | public static final NanoTimeType INSTANCE = new NanoTimeType(); 28 | 29 | @Override 30 | public String[] getRegistrationKeys() { 31 | return new String[] {NanoTime.SCHEMA_NAME}; 32 | } 33 | 34 | @Override 35 | protected LocalTime getLocalTime(Number value) { 36 | return DateTimeUtils.toLocalTimeFromDurationNanoseconds(value.longValue()); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/debezium/NanoTimestampType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.debezium; 20 | 21 | import io.debezium.time.MicroTimestamp; 22 | import io.debezium.time.NanoTimestamp; 23 | import java.time.LocalDateTime; 24 | import org.apache.doris.kafka.connector.converter.type.util.DateTimeUtils; 25 | 26 | /** 27 | * An implementation of {@link org.apache.doris.kafka.connector.converter.type.Type} for {@link 28 | * MicroTimestamp} values. 29 | */ 30 | public class NanoTimestampType extends AbstractDebeziumTimestampType { 31 | 32 | public static final NanoTimestampType INSTANCE = new NanoTimestampType(); 33 | 34 | @Override 35 | public String[] getRegistrationKeys() { 36 | return new String[] {NanoTimestamp.SCHEMA_NAME}; 37 | } 38 | 39 | @Override 40 | protected LocalDateTime getLocalDateTime(long value) { 41 | return DateTimeUtils.toLocalDateTimeFromInstantEpochNanos(value); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/debezium/PointType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.converter.type.debezium; 21 | 22 | import io.debezium.data.geometry.Point; 23 | import org.apache.doris.kafka.connector.converter.type.AbstractGeometryType; 24 | import org.apache.doris.kafka.connector.converter.type.util.GeoUtils; 25 | import org.apache.kafka.connect.data.Struct; 26 | 27 | public class PointType extends AbstractGeometryType { 28 | public static final PointType INSTANCE = new PointType(); 29 | 30 | @Override 31 | public String[] getRegistrationKeys() { 32 | return new String[] {Point.LOGICAL_NAME}; 33 | } 34 | 35 | @Override 36 | public Object getValue(Object sourceValue) { 37 | if (sourceValue == null) { 38 | return null; 39 | } 40 | if (sourceValue instanceof Struct) { 41 | return GeoUtils.handleGeoStructData(sourceValue); 42 | } 43 | 44 | return sourceValue; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/debezium/TimeType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.debezium; 20 | 21 | import io.debezium.time.Time; 22 | import java.time.LocalTime; 23 | import org.apache.doris.kafka.connector.converter.type.util.DateTimeUtils; 24 | 25 | public class TimeType extends AbstractDebeziumTimeType { 26 | 27 | public static final TimeType INSTANCE = new TimeType(); 28 | 29 | @Override 30 | public String[] getRegistrationKeys() { 31 | return new String[] {Time.SCHEMA_NAME}; 32 | } 33 | 34 | @Override 35 | protected LocalTime getLocalTime(Number value) { 36 | return DateTimeUtils.toLocalTimeFromDurationMilliseconds(value.longValue()); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/debezium/TimestampType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.debezium; 20 | 21 | import io.debezium.time.Timestamp; 22 | import java.time.Instant; 23 | import java.time.LocalDateTime; 24 | import java.time.ZoneOffset; 25 | 26 | public class TimestampType extends AbstractDebeziumTimestampType { 27 | 28 | public static final TimestampType INSTANCE = new TimestampType(); 29 | 30 | @Override 31 | public String[] getRegistrationKeys() { 32 | return new String[] {Timestamp.SCHEMA_NAME}; 33 | } 34 | 35 | @Override 36 | protected LocalDateTime getLocalDateTime(long value) { 37 | return LocalDateTime.ofInstant(Instant.ofEpochMilli(value), ZoneOffset.UTC); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/debezium/ZonedTimeType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.debezium; 20 | 21 | import io.debezium.time.ZonedTime; 22 | import java.time.OffsetTime; 23 | import java.time.ZonedDateTime; 24 | import java.time.format.DateTimeFormatter; 25 | import org.apache.doris.kafka.connector.converter.type.AbstractTimeType; 26 | import org.apache.kafka.connect.errors.ConnectException; 27 | 28 | public class ZonedTimeType extends AbstractTimeType { 29 | 30 | public static final ZonedTimeType INSTANCE = new ZonedTimeType(); 31 | // The ZonedTime of debezium type only contains three types of hours, minutes and seconds 32 | private final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("HH:mm:ss"); 33 | 34 | @Override 35 | public String[] getRegistrationKeys() { 36 | return new String[] {ZonedTime.SCHEMA_NAME}; 37 | } 38 | 39 | @Override 40 | public Object getValue(Object sourceValue) { 41 | if (sourceValue == null) { 42 | return null; 43 | } 44 | if (sourceValue instanceof String) { 45 | OffsetTime offsetTime = 46 | OffsetTime.parse((String) sourceValue, ZonedTime.FORMATTER) 47 | .withOffsetSameInstant( 48 | ZonedDateTime.now(getDatabaseTimeZone().toZoneId()) 49 | .getOffset()); 50 | return offsetTime.format(TIME_FORMATTER); 51 | } 52 | 53 | throw new ConnectException( 54 | String.format( 55 | "Unexpected %s value '%s' with type '%s'", 56 | getClass().getSimpleName(), sourceValue, sourceValue.getClass().getName())); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/debezium/ZonedTimestampType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.converter.type.debezium; 20 | 21 | import io.debezium.time.ZonedTimestamp; 22 | import java.time.ZonedDateTime; 23 | import org.apache.doris.kafka.connector.converter.type.AbstractTimestampType; 24 | import org.apache.kafka.connect.errors.ConnectException; 25 | 26 | public class ZonedTimestampType extends AbstractTimestampType { 27 | 28 | public static final ZonedTimestampType INSTANCE = new ZonedTimestampType(); 29 | 30 | @Override 31 | public String[] getRegistrationKeys() { 32 | return new String[] {ZonedTimestamp.SCHEMA_NAME}; 33 | } 34 | 35 | @Override 36 | public Object getValue(Object sourceValue) { 37 | if (sourceValue == null) { 38 | return null; 39 | } 40 | if (sourceValue instanceof String) { 41 | final ZonedDateTime zdt = 42 | ZonedDateTime.parse((String) sourceValue, ZonedTimestamp.FORMATTER) 43 | .withZoneSameInstant(getDatabaseTimeZone().toZoneId()); 44 | return zdt.toOffsetDateTime(); 45 | } 46 | 47 | throw new ConnectException( 48 | String.format( 49 | "Unexpected %s value '%s' with type '%s'", 50 | getClass().getSimpleName(), sourceValue, sourceValue.getClass().getName())); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/doris/DorisType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.converter.type.doris; 21 | 22 | public class DorisType { 23 | public static final String BOOLEAN = "BOOLEAN"; 24 | public static final String TINYINT = "TINYINT"; 25 | public static final String SMALLINT = "SMALLINT"; 26 | public static final String INT = "INT"; 27 | public static final String BIGINT = "BIGINT"; 28 | public static final String LARGEINT = "LARGEINT"; 29 | public static final String FLOAT = "FLOAT"; 30 | public static final String DOUBLE = "DOUBLE"; 31 | public static final String DECIMAL = "DECIMAL"; 32 | public static final String DATE = "DATE"; 33 | public static final String DATETIME = "DATETIME"; 34 | public static final String CHAR = "CHAR"; 35 | public static final String VARCHAR = "VARCHAR"; 36 | public static final String STRING = "STRING"; 37 | public static final String HLL = "HLL"; 38 | public static final String BITMAP = "BITMAP"; 39 | public static final String ARRAY = "ARRAY"; 40 | public static final String JSONB = "JSONB"; 41 | public static final String JSON = "JSON"; 42 | public static final String MAP = "MAP"; 43 | public static final String STRUCT = "STRUCT"; 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/doris/DorisTypeProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.converter.type.doris; 21 | 22 | public class DorisTypeProperties { 23 | 24 | /* Max precision of datetime type of Doris. */ 25 | public static final int MAX_SUPPORTED_DATE_TIME_PRECISION = 6; 26 | 27 | public static final int TIMESTAMP_TYPE_MAX_PRECISION = 9; 28 | 29 | public static final int MAX_VARCHAR_SIZE = 65533; 30 | 31 | public static final int MAX_CHAR_SIZE = 255; 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/converter/type/util/SchemaUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | * 19 | * Copied from 20 | * https://github.com/debezium/debezium-connector-jdbc/blob/main/src/main/java/io/debezium/connector/jdbc/util/SchemaUtils.java 21 | * modified by doris. 22 | */ 23 | 24 | package org.apache.doris.kafka.connector.converter.type.util; 25 | 26 | import java.util.Objects; 27 | import java.util.Optional; 28 | import org.apache.kafka.connect.data.Schema; 29 | 30 | public class SchemaUtils { 31 | private static final String SCHEMA_PARAMETER_COLUMN_TYPE = "__debezium.source.column.type"; 32 | private static final String SCHEMA_PARAMETER_COLUMN_LENGTH = "__debezium.source.column.length"; 33 | private static final String SCHEMA_PARAMETER_COLUMN_PRECISION = 34 | "__debezium.source.column.scale"; 35 | private static final String SCHEMA_PARAMETER_COLUMN_NAME = "__debezium.source.column.name"; 36 | 37 | public static Optional getSourceColumnType(Schema schema) { 38 | return getSchemaParameter(schema, SCHEMA_PARAMETER_COLUMN_TYPE); 39 | } 40 | 41 | public static Optional getSourceColumnLength(Schema schema) { 42 | return getSchemaParameter(schema, SCHEMA_PARAMETER_COLUMN_LENGTH); 43 | } 44 | 45 | public static Optional getSourceColumnPrecision(Schema schema) { 46 | return getSchemaParameter(schema, SCHEMA_PARAMETER_COLUMN_PRECISION); 47 | } 48 | 49 | public static Optional getSourceColumnName(Schema schema) { 50 | return getSchemaParameter(schema, SCHEMA_PARAMETER_COLUMN_NAME); 51 | } 52 | 53 | public static Optional getSchemaParameter(Schema schema, String parameterName) { 54 | if (!Objects.isNull(schema.parameters())) { 55 | return Optional.ofNullable(schema.parameters().get(parameterName)); 56 | } 57 | return Optional.empty(); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/decode/DorisConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.decode; 21 | 22 | import java.util.Map; 23 | import org.apache.doris.kafka.connector.exception.DataDecodeException; 24 | import org.apache.kafka.connect.data.Schema; 25 | import org.apache.kafka.connect.storage.Converter; 26 | 27 | public abstract class DorisConverter implements Converter { 28 | 29 | /** unused */ 30 | @Override 31 | public void configure(final Map map, final boolean b) { 32 | // not necessary 33 | } 34 | 35 | /** doesn't support data source connector */ 36 | @Override 37 | public byte[] fromConnectData(String topic, Schema schema, Object value) { 38 | throw new DataDecodeException("DorisConverter doesn't support data source connector yet."); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/decode/DorisJsonSchema.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.decode; 21 | 22 | import java.util.List; 23 | import java.util.Map; 24 | import org.apache.kafka.connect.data.Field; 25 | import org.apache.kafka.connect.data.Schema; 26 | 27 | public class DorisJsonSchema implements Schema { 28 | static String NAME = "DORIS_JSON_SCHEMA"; 29 | static int VERSION = 1; 30 | 31 | @Override 32 | public Schema.Type type() { 33 | return Type.STRUCT; 34 | } 35 | 36 | @Override 37 | public boolean isOptional() { 38 | return false; 39 | } 40 | 41 | @Override 42 | public Object defaultValue() { 43 | return null; 44 | } 45 | 46 | @Override 47 | public String name() { 48 | return NAME; 49 | } 50 | 51 | @Override 52 | public Integer version() { 53 | return VERSION; 54 | } 55 | 56 | @Override 57 | public String doc() { 58 | return null; 59 | } 60 | 61 | @Override 62 | public Map parameters() { 63 | return null; 64 | } 65 | 66 | @Override 67 | public Schema keySchema() { 68 | return null; 69 | } 70 | 71 | @Override 72 | public Schema valueSchema() { 73 | return null; 74 | } 75 | 76 | @Override 77 | public List fields() { 78 | return null; 79 | } 80 | 81 | @Override 82 | public Field field(final String s) { 83 | return null; 84 | } 85 | 86 | @Override 87 | public Schema schema() { 88 | return null; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/exception/ArgumentsException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.exception; 21 | 22 | public class ArgumentsException extends DorisException { 23 | 24 | public ArgumentsException() {} 25 | 26 | public ArgumentsException(String message) { 27 | super(message); 28 | } 29 | 30 | public ArgumentsException(String message, Throwable cause) { 31 | super(message, cause); 32 | } 33 | 34 | public ArgumentsException(Throwable cause) { 35 | super(cause); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/exception/ConnectedFailedException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.exception; 21 | 22 | public class ConnectedFailedException extends DorisException { 23 | 24 | public ConnectedFailedException(String message) { 25 | super(message); 26 | } 27 | 28 | public ConnectedFailedException(String server, Throwable cause) { 29 | super("Connect to " + server + " failed.", cause); 30 | } 31 | 32 | public ConnectedFailedException(String server, int statusCode, Throwable cause) { 33 | super("Connect to " + server + " failed, status code is " + statusCode + ".", cause); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/exception/CopyLoadException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.exception; 21 | 22 | public class CopyLoadException extends DorisException { 23 | 24 | public CopyLoadException() {} 25 | 26 | public CopyLoadException(String message) { 27 | super(message); 28 | } 29 | 30 | public CopyLoadException(String message, Throwable cause) { 31 | super(message, cause); 32 | } 33 | 34 | public CopyLoadException(Throwable cause) { 35 | super(cause); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/exception/DataConverterException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.exception; 21 | 22 | public class DataConverterException extends RuntimeException { 23 | 24 | public DataConverterException() { 25 | super(); 26 | } 27 | 28 | public DataConverterException(String message) { 29 | super(message); 30 | } 31 | 32 | public DataConverterException(String message, Throwable cause) { 33 | super(message, cause); 34 | } 35 | 36 | public DataConverterException(Throwable cause) { 37 | super(cause); 38 | } 39 | 40 | protected DataConverterException( 41 | String message, 42 | Throwable cause, 43 | boolean enableSuppression, 44 | boolean writableStackTrace) { 45 | super(message, cause, enableSuppression, writableStackTrace); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/exception/DataDecodeException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.exception; 21 | 22 | public class DataDecodeException extends DorisException { 23 | 24 | public DataDecodeException(String message) { 25 | super(message); 26 | } 27 | 28 | public DataDecodeException(String message, Throwable cause) { 29 | super(message, cause); 30 | } 31 | 32 | public DataDecodeException(Throwable cause) { 33 | super(cause); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/exception/DataFormatException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.exception; 21 | 22 | public class DataFormatException extends DorisException { 23 | 24 | public DataFormatException() {} 25 | 26 | public DataFormatException(String message) { 27 | super(message); 28 | } 29 | 30 | public DataFormatException(String message, Throwable cause) { 31 | super(message, cause); 32 | } 33 | 34 | public DataFormatException(Throwable cause) { 35 | super(cause); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/exception/DorisException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.exception; 21 | 22 | public class DorisException extends RuntimeException { 23 | 24 | public DorisException() {} 25 | 26 | public DorisException(String message) { 27 | super(message); 28 | } 29 | 30 | public DorisException(String message, Throwable cause) { 31 | super(message, cause); 32 | } 33 | 34 | public DorisException(Throwable cause) { 35 | super(cause); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/exception/SchemaChangeException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.apache.doris.kafka.connector.exception; 20 | 21 | /** Doris Schema Change run exception. */ 22 | public class SchemaChangeException extends RuntimeException { 23 | public SchemaChangeException() { 24 | super(); 25 | } 26 | 27 | public SchemaChangeException(String message) { 28 | super(message); 29 | } 30 | 31 | public SchemaChangeException(String message, Throwable cause) { 32 | super(message, cause); 33 | } 34 | 35 | public SchemaChangeException(Throwable cause) { 36 | super(cause); 37 | } 38 | 39 | protected SchemaChangeException( 40 | String message, 41 | Throwable cause, 42 | boolean enableSuppression, 43 | boolean writableStackTrace) { 44 | super(message, cause, enableSuppression, writableStackTrace); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/exception/StreamLoadException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.exception; 21 | 22 | public class StreamLoadException extends RuntimeException { 23 | 24 | public StreamLoadException() { 25 | super(); 26 | } 27 | 28 | public StreamLoadException(String message) { 29 | super(message); 30 | } 31 | 32 | public StreamLoadException(String message, Throwable cause) { 33 | super(message, cause); 34 | } 35 | 36 | public StreamLoadException(Throwable cause) { 37 | super(cause); 38 | } 39 | 40 | protected StreamLoadException( 41 | String message, 42 | Throwable cause, 43 | boolean enableSuppression, 44 | boolean writableStackTrace) { 45 | super(message, cause, enableSuppression, writableStackTrace); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/exception/UploadException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.exception; 21 | 22 | public class UploadException extends DorisException { 23 | 24 | public UploadException() {} 25 | 26 | public UploadException(String message) { 27 | super(message); 28 | } 29 | 30 | public UploadException(String message, Throwable cause) { 31 | super(message, cause); 32 | } 33 | 34 | public UploadException(Throwable cause) { 35 | super(cause); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/metrics/MetricsUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.metrics; 21 | 22 | public class MetricsUtil { 23 | public static final String JMX_METRIC_PREFIX = "kafka.connector.doris"; 24 | 25 | // Offset related constants 26 | public static final String TOTAL_PROCESSED_DOMAIN = "total-processed"; 27 | 28 | // total number of data successfully imported to doris through stream-load (or the total number 29 | // of data files uploaded through copy-into). 30 | public static final String TOTAL_LOAD_COUNT = "total-load-count"; 31 | public static final String TOTAL_RECORD_COUNT = "total-record-count"; 32 | public static final String TOTAL_DATA_SIZE = "total-data-size"; 33 | 34 | // file count related constants 35 | public static final String OFFSET_DOMAIN = "offsets"; 36 | // Successfully submitted data to doris' offset 37 | public static final String COMMITTED_OFFSET = "committed-offset"; 38 | 39 | // Buffer related constants 40 | public static final String BUFFER_DOMAIN = "buffer"; 41 | public static final String BUFFER_MEMORY_USAGE = "buffer-memory-usage"; 42 | public static final String BUFFER_SIZE_BYTES = "buffer-size-bytes"; 43 | public static final String BUFFER_RECORD_COUNT = "buffer-record-count"; 44 | 45 | public static String constructMetricName( 46 | final Integer taskId, final String domain, final String metricName) { 47 | return String.format("%s/%s/%s", taskId, domain, metricName); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/model/BackendV2.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.model; 21 | 22 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 23 | import com.fasterxml.jackson.annotation.JsonProperty; 24 | import java.util.List; 25 | 26 | /** Be response model. */ 27 | @JsonIgnoreProperties(ignoreUnknown = true) 28 | public class BackendV2 { 29 | 30 | @JsonProperty(value = "backends") 31 | private List backends; 32 | 33 | public List getBackends() { 34 | return backends; 35 | } 36 | 37 | public void setBackends(List backends) { 38 | this.backends = backends; 39 | } 40 | 41 | public static class BackendRowV2 { 42 | @JsonProperty("ip") 43 | public String ip; 44 | 45 | @JsonProperty("http_port") 46 | public int httpPort; 47 | 48 | @JsonProperty("is_alive") 49 | public boolean isAlive; 50 | 51 | public String getIp() { 52 | return ip; 53 | } 54 | 55 | public void setIp(String ip) { 56 | this.ip = ip; 57 | } 58 | 59 | public int getHttpPort() { 60 | return httpPort; 61 | } 62 | 63 | public void setHttpPort(int httpPort) { 64 | this.httpPort = httpPort; 65 | } 66 | 67 | public boolean isAlive() { 68 | return isAlive; 69 | } 70 | 71 | public void setAlive(boolean alive) { 72 | isAlive = alive; 73 | } 74 | 75 | public String toBackendString() { 76 | return ip + ":" + httpPort; 77 | } 78 | 79 | public static BackendRowV2 of(String ip, int httpPort, boolean alive) { 80 | BackendRowV2 rowV2 = new BackendRowV2(); 81 | rowV2.setIp(ip); 82 | rowV2.setHttpPort(httpPort); 83 | rowV2.setAlive(alive); 84 | return rowV2; 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/model/BaseResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.model; 21 | 22 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 23 | 24 | @JsonIgnoreProperties(ignoreUnknown = true) 25 | public class BaseResponse { 26 | private int code; 27 | private String msg; 28 | private T data; 29 | private int count; 30 | 31 | public int getCode() { 32 | return code; 33 | } 34 | 35 | public String getMsg() { 36 | return msg; 37 | } 38 | 39 | public T getData() { 40 | return data; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/model/BehaviorOnNullValues.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.model; 21 | 22 | public enum BehaviorOnNullValues { 23 | IGNORE("ignore"), 24 | 25 | FAIL("fail"); 26 | 27 | private String name; 28 | 29 | BehaviorOnNullValues(String name) { 30 | this.name = name; 31 | } 32 | 33 | public static BehaviorOnNullValues of(String name) { 34 | return BehaviorOnNullValues.valueOf(name.toUpperCase()); 35 | } 36 | 37 | public static String[] instances() { 38 | return new String[] {IGNORE.name, FAIL.name}; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/model/CopyIntoResp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.model; 21 | 22 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 23 | import java.util.Map; 24 | 25 | @JsonIgnoreProperties(ignoreUnknown = true) 26 | public class CopyIntoResp extends BaseResponse { 27 | private String code; 28 | private String exception; 29 | 30 | private Map result; 31 | 32 | public String getDataCode() { 33 | return code; 34 | } 35 | 36 | public String getException() { 37 | return exception; 38 | } 39 | 40 | public Map getResult() { 41 | return result; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/model/KafkaRespContent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.model; 21 | 22 | public class KafkaRespContent extends RespContent { 23 | 24 | private String topic; 25 | private String database; 26 | private String table; 27 | private long lastOffset; 28 | 29 | public String getTopic() { 30 | return topic; 31 | } 32 | 33 | public void setTopic(String topic) { 34 | this.topic = topic; 35 | } 36 | 37 | public String getDatabase() { 38 | return database; 39 | } 40 | 41 | public void setDatabase(String database) { 42 | this.database = database; 43 | } 44 | 45 | public String getTable() { 46 | return table; 47 | } 48 | 49 | public void setTable(String table) { 50 | this.table = table; 51 | } 52 | 53 | public long getLastOffset() { 54 | return lastOffset; 55 | } 56 | 57 | public void setLastOffset(long lastOffset) { 58 | this.lastOffset = lastOffset; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/model/LoadOperation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.model; 21 | 22 | public enum LoadOperation { 23 | SEND_REQUEST_TO_DORIS, 24 | 25 | COMMIT_TRANSACTION, 26 | 27 | GET_UPLOAD_ADDRESS, 28 | 29 | UPLOAD_FILE, 30 | 31 | EXECUTE_COPY; 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/service/DorisSinkService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.service; 21 | 22 | import java.util.Collection; 23 | import java.util.Map; 24 | import org.apache.kafka.clients.consumer.OffsetAndMetadata; 25 | import org.apache.kafka.common.TopicPartition; 26 | import org.apache.kafka.connect.sink.SinkRecord; 27 | 28 | /** Background service of data sink, responsible to create/drop table and insert/delete files */ 29 | public interface DorisSinkService { 30 | 31 | /** init task for writer */ 32 | void init(); 33 | 34 | /** 35 | * Start the Task. 36 | * 37 | * @param topicPartition TopicPartition passed from Kafka 38 | */ 39 | void startTask(TopicPartition topicPartition); 40 | 41 | /** 42 | * Start the Task. This should handle any configuration parsing and one-time setup of the task. 43 | * 44 | * @param tableName destination table name 45 | * @param topicPartition TopicPartition passed from Kafka 46 | */ 47 | void startTask(String tableName, TopicPartition topicPartition); 48 | 49 | /** 50 | * insert a collections of JSON records will trigger time based flush 51 | * 52 | * @param records record content 53 | */ 54 | void insert(final Collection records); 55 | 56 | /** 57 | * insert a JSON record will not trigger time based flush 58 | * 59 | * @param record record content 60 | */ 61 | void insert(final SinkRecord record); 62 | 63 | /** 64 | * retrieve offset of last loaded record 65 | * 66 | * @param topicPartition topic and partition 67 | * @return offset, or -1 for empty 68 | */ 69 | long getOffset(TopicPartition topicPartition); 70 | 71 | /** 72 | * get the number of partitions assigned to this sink service 73 | * 74 | * @return number of partitions 75 | */ 76 | int getPartitionCount(); 77 | 78 | /** commit data to doris. */ 79 | void commit(Map offsets); 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/service/DorisSinkServiceFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.service; 21 | 22 | import java.util.Map; 23 | import org.apache.doris.kafka.connector.cfg.DorisOptions; 24 | import org.apache.kafka.connect.sink.SinkTaskContext; 25 | 26 | /** A factory to create {@link DorisSinkService} */ 27 | public class DorisSinkServiceFactory { 28 | 29 | public static DorisSinkService getDorisSinkService( 30 | Map connectorConfig, SinkTaskContext context, DorisOptions options) { 31 | if (options.isEnableCombineFlush()) { 32 | return new DorisCombinedSinkService(connectorConfig, context); 33 | } else { 34 | return new DorisDefaultSinkService(connectorConfig, context); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/utils/HttpPostBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.utils; 21 | 22 | import java.nio.charset.StandardCharsets; 23 | import java.util.HashMap; 24 | import java.util.Map; 25 | import java.util.Objects; 26 | import org.apache.commons.codec.binary.Base64; 27 | import org.apache.commons.lang3.StringUtils; 28 | import org.apache.http.HttpEntity; 29 | import org.apache.http.HttpHeaders; 30 | import org.apache.http.client.methods.HttpPost; 31 | 32 | public class HttpPostBuilder { 33 | String url; 34 | Map header; 35 | HttpEntity httpEntity; 36 | 37 | public HttpPostBuilder() { 38 | header = new HashMap<>(); 39 | } 40 | 41 | public HttpPostBuilder setUrl(String url) { 42 | this.url = url; 43 | return this; 44 | } 45 | 46 | public HttpPostBuilder addCommonHeader() { 47 | header.put(HttpHeaders.EXPECT, "100-continue"); 48 | return this; 49 | } 50 | 51 | public HttpPostBuilder baseAuth(String user, String password) { 52 | final String authInfo = user + ":" + password; 53 | byte[] encoded = Base64.encodeBase64(authInfo.getBytes(StandardCharsets.UTF_8)); 54 | header.put(HttpHeaders.AUTHORIZATION, "Basic " + new String(encoded)); 55 | return this; 56 | } 57 | 58 | public HttpPostBuilder setEntity(HttpEntity httpEntity) { 59 | this.httpEntity = httpEntity; 60 | return this; 61 | } 62 | 63 | public HttpPost build() { 64 | StringUtils.isNotEmpty(url); 65 | Objects.nonNull(httpEntity); 66 | HttpPost put = new HttpPost(url); 67 | header.forEach(put::setHeader); 68 | put.setEntity(httpEntity); 69 | return put; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/utils/HttpUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.utils; 21 | 22 | import org.apache.http.impl.client.CloseableHttpClient; 23 | import org.apache.http.impl.client.DefaultRedirectStrategy; 24 | import org.apache.http.impl.client.HttpClientBuilder; 25 | import org.apache.http.impl.client.HttpClients; 26 | 27 | /** util to build http client. */ 28 | public class HttpUtils { 29 | private final HttpClientBuilder httpClientBuilder = 30 | HttpClients.custom() 31 | .setRedirectStrategy( 32 | new DefaultRedirectStrategy() { 33 | @Override 34 | protected boolean isRedirectable(String method) { 35 | return true; 36 | } 37 | }); 38 | 39 | public CloseableHttpClient getHttpClient() { 40 | return httpClientBuilder.build(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/utils/Version.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.utils; 21 | 22 | public class Version { 23 | public static String getVersion() { 24 | try { 25 | return Version.class.getPackage().getImplementationVersion(); 26 | } catch (Exception ex) { 27 | return "25.0.0"; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/writer/CopySQLBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.writer; 21 | 22 | import java.util.List; 23 | import java.util.Map; 24 | import java.util.Properties; 25 | import java.util.StringJoiner; 26 | 27 | public class CopySQLBuilder { 28 | private static final String COPY_SYNC = "copy.async"; 29 | private final String database; 30 | private final String table; 31 | private final List fileList; 32 | private Properties properties; 33 | 34 | private final boolean enableDelete; 35 | 36 | public CopySQLBuilder( 37 | String database, String table, List fileList, boolean enableDelete) { 38 | this.database = database; 39 | this.table = table; 40 | this.fileList = fileList; 41 | this.enableDelete = enableDelete; 42 | this.properties = new Properties(); 43 | } 44 | 45 | public String buildCopySQL() { 46 | StringBuilder sb = new StringBuilder(); 47 | sb.append("COPY INTO ") 48 | .append(database) 49 | .append(".") 50 | .append(table) 51 | .append(" FROM @~('{") 52 | .append(String.join(",", fileList)) 53 | .append("}') ") 54 | .append("PROPERTIES ("); 55 | 56 | // copy into must be sync 57 | properties.put(COPY_SYNC, false); 58 | // default json 59 | properties.put("file.type", "json"); 60 | properties.put("file.strip_outer_array", "false"); 61 | properties.put("copy.use_delete_sign", enableDelete); 62 | StringJoiner props = new StringJoiner(","); 63 | for (Map.Entry entry : properties.entrySet()) { 64 | String key = String.valueOf(entry.getKey()); 65 | String value = String.valueOf(entry.getValue()); 66 | String prop = String.format("'%s'='%s'", key, value); 67 | props.add(prop); 68 | } 69 | sb.append(props).append(")"); 70 | return sb.toString(); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/writer/DeliveryGuarantee.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.writer; 21 | 22 | public enum DeliveryGuarantee { 23 | EXACTLY_ONCE("exactly_once"), 24 | 25 | AT_LEAST_ONCE("at_least_once"); 26 | 27 | private final String name; 28 | 29 | DeliveryGuarantee(String name) { 30 | this.name = name; 31 | } 32 | 33 | public static DeliveryGuarantee of(String name) { 34 | return DeliveryGuarantee.valueOf(name.toUpperCase()); 35 | } 36 | 37 | public String getName() { 38 | return name; 39 | } 40 | 41 | public static String[] instances() { 42 | return new String[] {EXACTLY_ONCE.name, AT_LEAST_ONCE.name}; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/writer/LoadConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.writer; 21 | 22 | public class LoadConstants { 23 | public static final String DORIS_DELETE_SIGN = "__DORIS_DELETE_SIGN__"; 24 | public static final String DORIS_DEL_TRUE = "1"; 25 | public static final String DORIS_DEL_FALSE = "0"; 26 | 27 | // Special identifier, label separator used for kafka-connect sink data 28 | public static final String FILE_DELIM_DEFAULT = "__KC_"; 29 | 30 | // since apache doris 2.1.0, support stream load with group commit mode. 31 | public static final String GROUP_COMMIT = "group_commit"; 32 | public static final String PARTIAL_COLUMNS = "partial_columns"; 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/writer/LoadStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.writer; 21 | 22 | /** enum of LoadStatus. */ 23 | public class LoadStatus { 24 | public static final String SUCCESS = "Success"; 25 | public static final String PUBLISH_TIMEOUT = "Publish Timeout"; 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/writer/RecordBuffer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.writer; 21 | 22 | import java.nio.charset.StandardCharsets; 23 | import java.util.StringJoiner; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | /** record buffer */ 28 | public class RecordBuffer extends PartitionBuffer { 29 | private static final Logger LOG = LoggerFactory.getLogger(RecordBuffer.class); 30 | public static final String LINE_SEPARATOR = "\n"; 31 | private final StringJoiner buffer; 32 | private String label; 33 | 34 | public RecordBuffer() { 35 | super(); 36 | buffer = new StringJoiner(LINE_SEPARATOR); 37 | } 38 | 39 | @Override 40 | public void insert(String record) { 41 | buffer.add(record); 42 | setNumOfRecords(getNumOfRecords() + 1); 43 | setBufferSizeBytes(getBufferSizeBytes() + record.getBytes(StandardCharsets.UTF_8).length); 44 | } 45 | 46 | public String getData() { 47 | String result = buffer.toString(); 48 | LOG.debug( 49 | "flush buffer: {} records, {} bytes, offset {} - {}", 50 | getNumOfRecords(), 51 | getBufferSizeBytes(), 52 | getFirstOffset(), 53 | getLastOffset()); 54 | return result; 55 | } 56 | 57 | public String getLabel() { 58 | return label; 59 | } 60 | 61 | public void setLabel(String label) { 62 | this.label = label; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/writer/ResponseUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.writer; 21 | 22 | import java.util.regex.Pattern; 23 | 24 | /** util for handle response. */ 25 | public class ResponseUtil { 26 | public static final Pattern COMMITTED_PATTERN = 27 | Pattern.compile( 28 | "transaction \\[(\\d+)\\] is already \\b(COMMITTED|committed|VISIBLE|visible)\\b, not pre-committed."); 29 | 30 | public static boolean isCommitted(String msg) { 31 | return COMMITTED_PATTERN.matcher(msg).find(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/writer/load/DataLoad.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.writer.load; 21 | 22 | import com.fasterxml.jackson.databind.ObjectMapper; 23 | import java.io.Serializable; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | public abstract class DataLoad implements Serializable { 28 | private static final Logger LOG = LoggerFactory.getLogger(DataLoad.class); 29 | protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); 30 | protected String database; 31 | protected String table; 32 | protected String user; 33 | protected String password; 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/writer/load/DefaultThreadFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.writer.load; 21 | 22 | import java.util.concurrent.ThreadFactory; 23 | import java.util.concurrent.atomic.AtomicInteger; 24 | 25 | public class DefaultThreadFactory implements ThreadFactory { 26 | private static final AtomicInteger poolNumber = new AtomicInteger(1); 27 | private final AtomicInteger threadNumber = new AtomicInteger(1); 28 | private final String namePrefix; 29 | 30 | public DefaultThreadFactory(String name) { 31 | namePrefix = "pool-" + poolNumber.getAndIncrement() + "-" + name + "-"; 32 | } 33 | 34 | public Thread newThread(Runnable r) { 35 | Thread t = new Thread(r, namePrefix + threadNumber.getAndIncrement()); 36 | t.setDaemon(false); 37 | return t; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/writer/load/GroupCommitMode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.writer.load; 21 | 22 | import java.util.Arrays; 23 | import java.util.List; 24 | 25 | public enum GroupCommitMode { 26 | OFF_MODE("off_mode"), 27 | SYNC_MODE("sync_mode"), 28 | ASYNC_MODE("async_mode"); 29 | 30 | private final String name; 31 | 32 | GroupCommitMode(String name) { 33 | this.name = name; 34 | } 35 | 36 | public String getName() { 37 | return name; 38 | } 39 | 40 | public static LoadModel of(String name) { 41 | return LoadModel.valueOf(name.toUpperCase()); 42 | } 43 | 44 | public static List instances() { 45 | return Arrays.asList(OFF_MODE.name, SYNC_MODE.name, ASYNC_MODE.name); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/org/apache/doris/kafka/connector/writer/load/LoadModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.writer.load; 21 | 22 | public enum LoadModel { 23 | STREAM_LOAD("stream_load"), 24 | 25 | COPY_INTO("copy_into"); 26 | 27 | private String name; 28 | 29 | LoadModel(String name) { 30 | this.name = name; 31 | } 32 | 33 | public String getName() { 34 | return name; 35 | } 36 | 37 | public static LoadModel of(String name) { 38 | return LoadModel.valueOf(name.toUpperCase()); 39 | } 40 | 41 | public static String[] instances() { 42 | return new String[] {STREAM_LOAD.name, COPY_INTO.name}; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/test/java/org/apache/doris/kafka/connector/DistributedTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector; 21 | 22 | import org.apache.kafka.connect.cli.ConnectDistributed; 23 | import org.junit.Ignore; 24 | import org.junit.Test; 25 | 26 | @Ignore 27 | public class DistributedTest { 28 | 29 | @Test 30 | public void runConnector1() throws Exception { 31 | String[] params = new String[1]; 32 | String distributed = 33 | this.getClass() 34 | .getClassLoader() 35 | .getResource("connect-distributed.properties") 36 | .getPath(); 37 | System.out.println(distributed); 38 | params[0] = distributed; 39 | ConnectDistributed.main(params); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/test/java/org/apache/doris/kafka/connector/StandaloneTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector; 21 | 22 | import org.apache.kafka.connect.cli.ConnectStandalone; 23 | import org.junit.Ignore; 24 | import org.junit.Test; 25 | 26 | @Ignore 27 | public class StandaloneTest { 28 | 29 | @Test 30 | public void runTest() throws Exception { 31 | String[] params = new String[2]; 32 | String standalone = 33 | this.getClass() 34 | .getClassLoader() 35 | .getResource("connect-standalone.properties") 36 | .getPath(); 37 | System.out.println(standalone); 38 | params[0] = standalone; 39 | String sink = 40 | this.getClass() 41 | .getClassLoader() 42 | .getResource("doris-connector-sink.properties") 43 | .getPath(); 44 | System.out.println(sink); 45 | params[1] = sink; 46 | ConnectStandalone.main(params); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/test/java/org/apache/doris/kafka/connector/e2e/doris/DorisContainerService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.e2e.doris; 21 | 22 | public interface DorisContainerService { 23 | 24 | void startContainer(); 25 | 26 | String getInstanceHost(); 27 | 28 | void close(); 29 | } 30 | -------------------------------------------------------------------------------- /src/test/java/org/apache/doris/kafka/connector/e2e/kafka/KafkaContainerService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.e2e.kafka; 21 | 22 | import java.io.IOException; 23 | 24 | public interface KafkaContainerService { 25 | 26 | void startContainer(); 27 | 28 | void startSchemaRegistry(); 29 | 30 | String getSchemaRegistryUrl(); 31 | 32 | void startConnector(); 33 | 34 | String getInstanceHostAndPort(); 35 | 36 | void registerKafkaConnector(String name, String msg) throws IOException, InterruptedException; 37 | 38 | void deleteKafkaConnector(String name); 39 | 40 | String getConnectorTaskStatus(String name); 41 | 42 | void close(); 43 | } 44 | -------------------------------------------------------------------------------- /src/test/java/org/apache/doris/kafka/connector/utils/ConfigCheckUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.utils; 21 | 22 | import java.util.HashMap; 23 | import java.util.Map; 24 | import org.junit.Test; 25 | 26 | public class ConfigCheckUtilsTest { 27 | 28 | @Test 29 | public void testIsValidDorisApplicationName() { 30 | assert ConfigCheckUtils.isValidDorisApplicationName("-_aA1"); 31 | assert ConfigCheckUtils.isValidDorisApplicationName("aA_1-"); 32 | assert !ConfigCheckUtils.isValidDorisApplicationName("_1.a$"); 33 | assert !ConfigCheckUtils.isValidDorisApplicationName("(1.f$-_"); 34 | } 35 | 36 | @Test 37 | public void testTableName() { 38 | Map topic2table = 39 | ConfigCheckUtils.parseTopicToTableMap("ab@cd:abcd, 1234:_1234"); 40 | 41 | assert ConfigCheckUtils.tableName("ab@cd", topic2table).equals("abcd"); 42 | assert ConfigCheckUtils.tableName("1234", topic2table).equals("_1234"); 43 | } 44 | 45 | @Test 46 | public void testTableNameMapEmpty() { 47 | Map topic2table = new HashMap<>(); 48 | 49 | assert ConfigCheckUtils.tableName("name.db.tbl", topic2table).equals("tbl"); 50 | assert ConfigCheckUtils.tableName("table123", topic2table).equals("table123"); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/test/java/org/apache/doris/kafka/connector/utils/FileNameConfigCheckUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.utils; 21 | 22 | import org.junit.Assert; 23 | import org.junit.Test; 24 | 25 | public class FileNameConfigCheckUtilsTest { 26 | 27 | String name = "doris-connector-test"; 28 | String topic = "avro-complex10"; 29 | public String fileName1 = 30 | "doris-connector-test__KC_avro-complex10__KC_0__KC_411__KC_1706149860394"; 31 | public String fileName2 = 32 | "doris-connector-test__KC_avro-complex10__KC_2__KC_348__KC_1706149860395"; 33 | public String fileName3 = 34 | "doris-connector-test__KC_avro-complex10__KC_5__KC_311__KC_1706149860394"; 35 | 36 | @Test 37 | public void verifyFileNameTest() { 38 | Assert.assertTrue(FileNameUtils.verifyFileName(name, topic, 0, fileName1)); 39 | Assert.assertFalse(FileNameUtils.verifyFileName(name, topic, 1, fileName1)); 40 | 41 | Assert.assertTrue(FileNameUtils.verifyFileName(name, topic, 2, fileName2)); 42 | Assert.assertFalse(FileNameUtils.verifyFileName(name, topic, 12, fileName2)); 43 | 44 | Assert.assertFalse(FileNameUtils.verifyFileName(name, topic, 15, fileName3)); 45 | Assert.assertTrue(FileNameUtils.verifyFileName(name, topic, 5, fileName3)); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/org/apache/doris/kafka/connector/writer/HttpEntityMock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.writer; 21 | 22 | import java.io.ByteArrayInputStream; 23 | import java.io.IOException; 24 | import java.io.InputStream; 25 | import java.io.OutputStream; 26 | import java.nio.charset.StandardCharsets; 27 | import org.apache.http.Header; 28 | import org.apache.http.HttpEntity; 29 | import org.apache.http.message.BasicHeader; 30 | 31 | /** Mock Class for HttpEntity. */ 32 | public class HttpEntityMock implements HttpEntity { 33 | private String value; 34 | 35 | public HttpEntityMock() {} 36 | 37 | public void setValue(String value) { 38 | this.value = value; 39 | } 40 | 41 | @Override 42 | public boolean isRepeatable() { 43 | return false; 44 | } 45 | 46 | @Override 47 | public boolean isChunked() { 48 | return false; 49 | } 50 | 51 | @Override 52 | public long getContentLength() { 53 | return 0; 54 | } 55 | 56 | @Override 57 | public Header getContentType() { 58 | return new BasicHeader("header", "text/html;charset=utf-8;"); 59 | } 60 | 61 | @Override 62 | public Header getContentEncoding() { 63 | return null; 64 | } 65 | 66 | @Override 67 | public InputStream getContent() throws IOException, UnsupportedOperationException { 68 | return new ByteArrayInputStream(value.getBytes(StandardCharsets.UTF_8)); 69 | } 70 | 71 | @Override 72 | public void writeTo(OutputStream outputStream) throws IOException {} 73 | 74 | @Override 75 | public boolean isStreaming() { 76 | return false; 77 | } 78 | 79 | @Override 80 | public void consumeContent() throws IOException {} 81 | } 82 | -------------------------------------------------------------------------------- /src/test/java/org/apache/doris/kafka/connector/writer/HttpTestUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.writer; 21 | 22 | import static org.mockito.Mockito.mock; 23 | import static org.mockito.Mockito.when; 24 | 25 | import org.apache.http.ProtocolVersion; 26 | import org.apache.http.StatusLine; 27 | import org.apache.http.client.methods.CloseableHttpResponse; 28 | import org.apache.http.message.BasicStatusLine; 29 | 30 | /** Test Util for Http. */ 31 | public class HttpTestUtil { 32 | 33 | public static StatusLine normalLine = 34 | new BasicStatusLine(new ProtocolVersion("http", 1, 0), 200, ""); 35 | public static StatusLine abnormalLine = 36 | new BasicStatusLine(new ProtocolVersion("http", 1, 0), 404, ""); 37 | public static StatusLine redirectLine = 38 | new BasicStatusLine(new ProtocolVersion("http", 1, 0), 307, ""); 39 | 40 | public static CloseableHttpResponse getResponse( 41 | String response, boolean ok, boolean isRedirect) { 42 | HttpEntityMock httpEntityMock = new HttpEntityMock(); 43 | httpEntityMock.setValue(response); 44 | CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); 45 | if (isRedirect) { 46 | when(httpResponse.getStatusLine()).thenReturn(redirectLine); 47 | } else if (ok) { 48 | when(httpResponse.getStatusLine()).thenReturn(normalLine); 49 | } else { 50 | when(httpResponse.getStatusLine()).thenReturn(abnormalLine); 51 | } 52 | when(httpResponse.getEntity()).thenReturn(httpEntityMock); 53 | return httpResponse; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/org/apache/doris/kafka/connector/writer/TestCopySQL.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.writer; 21 | 22 | import java.util.Arrays; 23 | import org.junit.Assert; 24 | import org.junit.Test; 25 | 26 | public class TestCopySQL { 27 | 28 | @Test 29 | public void testCopySQL() { 30 | CopySQLBuilder builder = 31 | new CopySQLBuilder("db", "tbl", Arrays.asList("fileName", "fileName2"), false); 32 | String copySQL = builder.buildCopySQL(); 33 | String except = 34 | "COPY INTO db.tbl FROM @~('{fileName,fileName2}') PROPERTIES ('copy.async'='false','file.type'='json','file.strip_outer_array'='false','copy.use_delete_sign'='false')"; 35 | Assert.assertEquals(copySQL, except); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/test/java/org/apache/doris/kafka/connector/writer/TestRecordBuffer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.doris.kafka.connector.writer; 21 | 22 | import org.apache.kafka.connect.data.Schema; 23 | import org.apache.kafka.connect.sink.SinkRecord; 24 | import org.junit.Assert; 25 | import org.junit.Test; 26 | 27 | public class TestRecordBuffer { 28 | 29 | private RecordBuffer buffer = new RecordBuffer(); 30 | 31 | @Test 32 | public void insert() { 33 | long offset = 1l; 34 | buffer.insert(newSinkRecord("doris", offset).value().toString()); 35 | String data = buffer.getData(); 36 | Assert.assertEquals(data, "doris"); 37 | offset++; 38 | buffer.insert(newSinkRecord("doris", offset).value().toString()); 39 | Assert.assertEquals(2, buffer.getNumOfRecords()); 40 | } 41 | 42 | public static SinkRecord newSinkRecord(Object value, long offset) { 43 | SinkRecord record = 44 | new SinkRecord( 45 | "topic", 46 | 0, 47 | Schema.OPTIONAL_STRING_SCHEMA, 48 | "key", 49 | Schema.OPTIONAL_STRING_SCHEMA, 50 | value, 51 | offset); 52 | return record; 53 | } 54 | 55 | public static SinkRecord newSinkRecord( 56 | String topic, Object value, long offset, Schema valueSchema) { 57 | SinkRecord record = 58 | new SinkRecord( 59 | topic, 0, Schema.OPTIONAL_STRING_SCHEMA, "key", valueSchema, value, offset); 60 | return record; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/test/resources/connect-standalone.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | # These are defaults. This file just demonstrates how to override some settings. 21 | bootstrap.servers=127.0.0.1:9092 22 | 23 | # The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will 24 | # need to configure these based on the format they want their data in when loaded from or stored into Kafka 25 | 26 | key.converter=org.apache.kafka.connect.json.JsonConverter 27 | value.converter=org.apache.kafka.connect.json.JsonConverter 28 | 29 | # Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply 30 | # it to 31 | key.converter.schemas.enable=true 32 | value.converter.schemas.enable=true 33 | 34 | # The internal converter used for offsets and config data is configurable and must be specified, but most users will 35 | # always want to use the built-in default. Offset and config data is never visible outside of Kafka Connect in this format. 36 | 37 | internal.key.converter=org.apache.kafka.connect.json.JsonConverter 38 | internal.value.converter=org.apache.kafka.connect.json.JsonConverter 39 | 40 | #internal.key.converter=org.apache.kafka.connect.storage.StringConverter 41 | #internal.value.converter=org.apache.kafka.connect.storage.StringConverter 42 | 43 | internal.key.converter.schemas.enable=false 44 | internal.value.converter.schemas.enable=false 45 | 46 | 47 | #producer.offset.storage.file.filename=/home/work/lt/kafka-connect-offset/producer.connect.offsets 48 | #consumer.offset.storage.file.filename=/home/work/lt/kafka-connect-offset/consumer.connect.offsets 49 | 50 | offset.storage.file.filename=connect.offsets 51 | 52 | 53 | # Flush much faster than normal, which is useful for testing/debugging 54 | offset.flush.interval.ms=10000 55 | 56 | rest.port=8083 57 | 58 | #transforms=unwrap 59 | #transforms.unwrap.type=io.debezium.transforms.UnwrapFromEnvelope 60 | #transforms.unwrap.drop.tombstones=false 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /src/test/resources/decode/avro/product.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "Product", 4 | "fields": [ 5 | { 6 | "name": "id", 7 | "type": "int" 8 | }, 9 | { 10 | "name": "name", 11 | "type": "string" 12 | }, 13 | { 14 | "name": "price", 15 | "type": "double" 16 | } 17 | ] 18 | } -------------------------------------------------------------------------------- /src/test/resources/decode/avro/user.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "User", 4 | "fields": [ 5 | { 6 | "name": "id", 7 | "type": "int" 8 | }, 9 | { 10 | "name": "name", 11 | "type": "string" 12 | }, 13 | { 14 | "name": "age", 15 | "type": "int" 16 | } 17 | ] 18 | } -------------------------------------------------------------------------------- /src/test/resources/doris-connector-sink.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | name=test-doris-sink 21 | connector.class=org.apache.doris.kafka.connector.DorisSinkConnector 22 | topics=topic_test 23 | doris.topic2table.map=topic_test:test_kafka_tbl 24 | buffer.count.records=10000 25 | buffer.flush.time=120 26 | buffer.size.bytes=5000000 27 | doris.urls=10.10.10.1 28 | doris.http.port=8030 29 | doris.query.port=9030 30 | doris.user=root 31 | doris.password= 32 | doris.database=test_db 33 | key.converter=org.apache.kafka.connect.storage.StringConverter 34 | value.converter=org.apache.kafka.connect.json.JsonConverter 35 | key.converter.schemas.enable=false 36 | value.converter.schemas.enable=false 37 | 38 | errors.tolerance=all 39 | errors.deadletterqueue.topic.name=test_error 40 | errors.deadletterqueue.context.headers.enable = true 41 | errors.deadletterqueue.topic.replication.factor=1 -------------------------------------------------------------------------------- /src/test/resources/doris-connector-sink.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | #add connector 21 | curl -i http://127.0.0.1:8083/connectors -H "Content-Type: application/json" -X POST -d '{ 22 | "name":"test-doris-sink-cluster", 23 | "config":{ 24 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 25 | "topics":"topic_test", 26 | "doris.topic2table.map": "topic_test:test_kafka_tbl", 27 | "buffer.count.records":"10000", 28 | "buffer.flush.time":"120", 29 | "buffer.size.bytes":"5000000", 30 | "doris.url":"10.10.10.1", 31 | "doris.user":"root", 32 | "doris.password":"", 33 | "doris.http.port":"8030", 34 | "doris.query.port":"9030", 35 | "doris.database":"test_db", 36 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 37 | "value.converter":"org.apache.kafka.connect.json.JsonConverter", 38 | "key.converter.schemas.enable":"false", 39 | "value.converter.schemas.enable":"false", 40 | } 41 | }' 42 | 43 | # remove connector 44 | curl -i http://127.0.0.1:8083/connectors/test-doris-sink-cluster -X DELETE 45 | 46 | # pause connector 47 | curl -i http://127.0.0.1:8083/connectors/test-doris-sink-cluster/pause -X PUT 48 | 49 | # resume connector 50 | curl -i http://127.0.0.1:8083/connectors/test-doris-sink-cluster/resume -X PUT 51 | 52 | # restart connector 53 | curl -i http://127.0.0.1:8083/connectors/test-doris-sink-cluster/restart -X POST 54 | 55 | # restart task 56 | curl -i http://127.0.0.1:8083/connectors/test-doris-sink-cluster/tasks/${taskid}/restart -X POST 57 | -------------------------------------------------------------------------------- /src/test/resources/e2e/avro_converter/confluent_avro_convert.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"confluent-avro-test", 3 | "config":{ 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"avro-user-confluent", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "avro-user-confluent:confluent_avro_tab", 8 | "buffer.count.records":"1", 9 | "buffer.flush.time":"10", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"confluent_avro_convert", 17 | "load.model":"stream_load", 18 | "key.converter":"io.confluent.connect.avro.AvroConverter", 19 | "key.converter.schema.registry.url":"http://127.0.0.1:8081", 20 | "value.converter":"io.confluent.connect.avro.AvroConverter", 21 | "value.converter.schema.registry.url":"http://127.0.0.1:8081" 22 | } 23 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/avro_converter/confluent_avro_tab.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE confluent_avro_convert.confluent_avro_tab 2 | ( 3 | `id` INT, 4 | `name` VARCHAR(256), 5 | `age` SMALLINT, 6 | )DUPLICATE KEY(`id`) 7 | DISTRIBUTED BY HASH(`id`) BUCKETS 1 8 | PROPERTIES ( 9 | "replication_allocation" = "tag.location.default: 1", 10 | "light_schema_change" = "true" 11 | ); -------------------------------------------------------------------------------- /src/test/resources/e2e/avro_converter/doris_avro_convert.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"avro_sink-connector", 3 | "config":{ 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"avro-user", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "avro-user:doris_avro_tab", 8 | "buffer.count.records":"1", 9 | "buffer.flush.time":"10", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"avro_convert", 17 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 18 | "value.converter":"org.apache.doris.kafka.connector.decode.avro.DorisAvroConverter", 19 | "value.converter.avro.topic2schema.filepath":"avro-user:file:///opt/avro_user.avsc" 20 | } 21 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/avro_converter/doris_avro_tab.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE avro_convert.doris_avro_tab 2 | ( 3 | `id` INT, 4 | `name` VARCHAR(256), 5 | `age` INT, 6 | )DUPLICATE KEY(`id`) 7 | DISTRIBUTED BY HASH(`id`) BUCKETS 1 8 | PROPERTIES ( 9 | "replication_allocation" = "tag.location.default: 1", 10 | "light_schema_change" = "true" 11 | ); -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/combine_flush_connector.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"combine_flush_connector", 3 | "config":{ 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"combine_test", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "combine_test:combine_flush_tab", 8 | "buffer.count.records":"100", 9 | "buffer.flush.time":"1", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"combine_flush", 17 | "load.model":"stream_load", 18 | "enable.combine.flush":"true", 19 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 20 | "value.converter":"org.apache.kafka.connect.storage.StringConverter" 21 | } 22 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/combine_flush_connector_2pc.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"combine_flush_connector_2pc", 3 | "config":{ 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"combine_test_2pc", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "combine_test_2pc:combine_flush_tab_2pc", 8 | "buffer.count.records":"100", 9 | "buffer.flush.time":"1", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"combine_flush_2pc", 17 | "load.model":"stream_load", 18 | "enable.combine.flush":"true", 19 | "enable.2pc": "true", 20 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 21 | "value.converter":"org.apache.kafka.connect.storage.StringConverter" 22 | } 23 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/combine_flush_tab.sql: -------------------------------------------------------------------------------- 1 | -- Please note that the database here should be consistent with doris.database in the file where the connector is registered. 2 | CREATE TABLE combine_flush.combine_flush_tab ( 3 | id INT NULL, 4 | name VARCHAR(100) NULL, 5 | age INT NULL 6 | ) ENGINE=OLAP 7 | UNIQUE KEY(`id`) 8 | COMMENT 'OLAP' 9 | DISTRIBUTED BY HASH(`id`) BUCKETS AUTO 10 | PROPERTIES ( 11 | "replication_allocation" = "tag.location.default: 1" 12 | ); -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/combine_flush_tab_2pc.sql: -------------------------------------------------------------------------------- 1 | -- Please note that the database here should be consistent with doris.database in the file where the connector is registered. 2 | CREATE TABLE combine_flush_2pc.combine_flush_tab_2pc ( 3 | id INT NULL, 4 | name VARCHAR(100) NULL, 5 | age INT NULL 6 | ) ENGINE=OLAP 7 | UNIQUE KEY(`id`) 8 | COMMENT 'OLAP' 9 | DISTRIBUTED BY HASH(`id`) BUCKETS AUTO 10 | PROPERTIES ( 11 | "replication_allocation" = "tag.location.default: 1" 12 | ); -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/debezium_dml_event.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"debezium_dml_event", 3 | "config": { 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"debezium_dml_event", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "debezium_dml_event:debezium_dml_event_tab", 8 | "buffer.count.records":"1", 9 | "buffer.flush.time":"10", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"debezium_ingestion_msg", 17 | "converter.mode": "debezium_ingestion", 18 | "load.model":"stream_load", 19 | "delivery.guarantee":"exactly_once", 20 | "enable.2pc": "true", 21 | "enable.delete": "true", 22 | "key.converter":"org.apache.kafka.connect.json.JsonConverter", 23 | "value.converter":"org.apache.kafka.connect.json.JsonConverter" 24 | } 25 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/debezium_dml_event_tab.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE debezium_ingestion_msg.debezium_dml_event_tab 2 | ( 3 | `id` INT, 4 | `name` VARCHAR(256), 5 | `age` SMALLINT, 6 | )UNIQUE KEY(`id`) 7 | DISTRIBUTED BY HASH(`id`) BUCKETS 1 8 | PROPERTIES ( 9 | "replication_allocation" = "tag.location.default: 1", 10 | "light_schema_change" = "true" 11 | ); -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/full_types.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"mysql_all_types", 3 | "config":{ 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"full_types", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "full_types:full_types_tab", 8 | "buffer.count.records":"1", 9 | "buffer.flush.time":"10", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"debezium_ingestion_msg", 17 | "converter.mode": "debezium_ingestion", 18 | "load.model":"stream_load", 19 | "key.converter":"org.apache.kafka.connect.json.JsonConverter", 20 | "value.converter":"org.apache.kafka.connect.json.JsonConverter" 21 | } 22 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/full_types_debezium_ingestion.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE debezium_ingestion_msg.full_types_tab 2 | ( 3 | `id` LARGEINT NULL, 4 | `tiny_c` TINYINT NULL, 5 | `tiny_un_c` SMALLINT NULL, 6 | `tiny_un_z_c` SMALLINT NULL, 7 | `small_c` SMALLINT NULL, 8 | `small_un_c` INT NULL, 9 | `small_un_z_c` INT NULL, 10 | `medium_c` INT NULL, 11 | `medium_un_c` BIGINT NULL, 12 | `medium_un_z_c` BIGINT NULL, 13 | `int_c` INT NULL, 14 | `int_un_c` BIGINT NULL, 15 | `int_un_z_c` BIGINT NULL, 16 | `int11_c` INT NULL, 17 | `big_c` BIGINT NULL, 18 | `big_un_c` LARGEINT NULL, 19 | `big_un_z_c` LARGEINT NULL, 20 | `varchar_c` VARCHAR(765) NULL, 21 | `char_c` VARCHAR(9) NULL, 22 | `real_c` DOUBLE NULL, 23 | `float_c` FLOAT NULL, 24 | `float_un_c` FLOAT NULL, 25 | `float_un_z_c` FLOAT NULL, 26 | `double_c` DOUBLE NULL, 27 | `double_un_c` DOUBLE NULL, 28 | `double_un_z_c` DOUBLE NULL, 29 | `decimal_c` DECIMAL(8, 4) NULL, 30 | `decimal_un_c` DECIMAL(8, 4) NULL, 31 | `decimal_un_z_c` DECIMAL(8, 4) NULL, 32 | `numeric_c` DECIMAL(6, 0) NULL, 33 | `big_decimal_c` TEXT NULL, 34 | `bit1_c` BOOLEAN NULL, 35 | `tiny1_c` BOOLEAN NULL, 36 | `boolean_c` BOOLEAN NULL, 37 | `date_c` DATE NULL, 38 | `time_c` TEXT NULL, 39 | `datetime_c` DATETIME NULL, 40 | `timestamp_c` DATETIME NULL, 41 | `text_c` TEXT NULL, 42 | `year_c` INT NULL, 43 | `enum_c` TEXT NULL, 44 | `set_c` TEXT NULL, 45 | `json_c` JSON NULL, 46 | `point_c` TEXT NULL, 47 | `geometry_c` TEXT NULL, 48 | `linestring_c` TEXT NULL, 49 | `polygon_c` TEXT NULL, 50 | `multipoint_c` TEXT NULL, 51 | `multiline_c` TEXT NULL, 52 | `multipolygon_c` TEXT NULL, 53 | `geometrycollection_c` TEXT NULL 54 | )UNIQUE KEY(`id`) 55 | DISTRIBUTED BY HASH(`id`) BUCKETS 1 56 | PROPERTIES ( 57 | "replication_allocation" = "tag.location.default: 1", 58 | "light_schema_change" = "true" 59 | ); -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/group_commit_connector.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"group_commit_connector", 3 | "config":{ 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"group_commit_test", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "group_commit_test:group_commit_tab", 8 | "buffer.count.records":"2", 9 | "buffer.flush.time":"120", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"group_commit", 17 | "sink.properties.group_commit":"sync_mode", 18 | "enable.2pc": "false", 19 | "load.model":"stream_load", 20 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 21 | "value.converter":"org.apache.kafka.connect.storage.StringConverter" 22 | } 23 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/group_commit_tab.sql: -------------------------------------------------------------------------------- 1 | -- Please note that the database here should be consistent with doris.database in the file where the connector is registered. 2 | CREATE TABLE group_commit.group_commit_tab ( 3 | id INT NULL, 4 | name VARCHAR(100) NULL, 5 | age INT NULL 6 | ) ENGINE=OLAP 7 | UNIQUE KEY(`id`) 8 | COMMENT 'OLAP' 9 | DISTRIBUTED BY HASH(`id`) BUCKETS AUTO 10 | PROPERTIES ( 11 | "replication_allocation" = "tag.location.default: 1", 12 | "light_schema_change"="true" 13 | ); -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/insert_partial_update_tab.sql: -------------------------------------------------------------------------------- 1 | insert into string_msg.partial_update_tab (id, col1, col2, col3) 2 | values (1, "before_update_col1_1", "before_update_col2_1", "before_update_col3_1"), 3 | (2, "before_update_col1_2", "before_update_col2_2", "before_update_col3_2"); -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/null_values_default.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"null_values_default_connector", 3 | "config":{ 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"behavior_on_null_values_test", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "behavior_on_null_values_test:null_values_tab", 8 | "buffer.count.records":"1", 9 | "buffer.flush.time":"10", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"string_msg", 17 | "enable.2pc": "false", 18 | "load.model":"stream_load", 19 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 20 | "value.converter":"org.apache.kafka.connect.storage.StringConverter", 21 | "value.converter.schemas.enable": "false" 22 | } 23 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/null_values_fail.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"null_values_fail_connector", 3 | "config":{ 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"behavior_on_null_values_test", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "behavior_on_null_values_test:null_values_tab", 8 | "buffer.count.records":"1", 9 | "buffer.flush.time":"10", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"string_msg", 17 | "enable.2pc": "false", 18 | "load.model":"stream_load", 19 | "behavior.on.null.values":"fail", 20 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 21 | "value.converter":"org.apache.kafka.connect.storage.StringConverter", 22 | "value.converter.schemas.enable": "false" 23 | } 24 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/null_values_ignore.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"null_values_ignore_connector", 3 | "config":{ 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"behavior_on_null_values_test", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "behavior_on_null_values_test:null_values_tab", 8 | "buffer.count.records":"1", 9 | "buffer.flush.time":"10", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"string_msg", 17 | "enable.2pc": "false", 18 | "load.model":"stream_load", 19 | "behavior.on.null.values":"ignore", 20 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 21 | "value.converter":"org.apache.kafka.connect.storage.StringConverter", 22 | "value.converter.schemas.enable": "false" 23 | } 24 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/null_values_tab.sql: -------------------------------------------------------------------------------- 1 | -- Please note that the database here should be consistent with doris.database in the file where the connector is registered. 2 | CREATE TABLE string_msg.null_values_tab ( 3 | id INT NULL, 4 | col1 VARCHAR(20) NULL, 5 | col2 varchar(20) NULL 6 | ) ENGINE=OLAP 7 | UNIQUE KEY(`id`) 8 | COMMENT 'OLAP' 9 | DISTRIBUTED BY HASH(`id`) BUCKETS AUTO 10 | PROPERTIES ( 11 | "replication_allocation" = "tag.location.default: 1" 12 | ); -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/partial_update.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"partial_update_connector", 3 | "config":{ 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"partial_update_test", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "partial_update_test:partial_update_tab", 8 | "buffer.count.records":"2", 9 | "buffer.flush.time":"10", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"string_msg", 17 | "sink.properties.partial_columns":"true", 18 | "sink.properties.columns": "id,col1,col2", 19 | "enable.2pc": "false", 20 | "load.model":"stream_load", 21 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 22 | "value.converter":"org.apache.kafka.connect.storage.StringConverter" 23 | } 24 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/partial_update_tab.sql: -------------------------------------------------------------------------------- 1 | -- Please note that the database here should be consistent with doris.database in the file where the connector is registered. 2 | CREATE TABLE string_msg.partial_update_tab ( 3 | id INT NULL, 4 | col1 VARCHAR(20) NULL, 5 | col2 varchar(20) NULL, 6 | col3 varchar(20) NUll 7 | ) ENGINE=OLAP 8 | UNIQUE KEY(`id`) 9 | COMMENT 'OLAP' 10 | DISTRIBUTED BY HASH(`id`) BUCKETS AUTO 11 | PROPERTIES ( 12 | "replication_allocation" = "tag.location.default: 1", 13 | "light_schema_change"="true", 14 | "enable_unique_key_merge_on_write" = "true" 15 | ); -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/string_msg_connector.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"string_msg_connector", 3 | "config":{ 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"string_test", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "string_test:string_msg_tab", 8 | "buffer.count.records":"1", 9 | "buffer.flush.time":"120", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"string_msg", 17 | "load.model":"stream_load", 18 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 19 | "value.converter":"org.apache.kafka.connect.storage.StringConverter" 20 | } 21 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/string_msg_failover_connector.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"string_msg_failover_connector", 3 | "config":{ 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"string_test_failover", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "string_test_failover:string_msg_tab_failover", 8 | "buffer.count.records":"1", 9 | "buffer.flush.time":"1200", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"string_msg_failover", 17 | "load.model":"stream_load", 18 | "delivery.guarantee":"exactly_once", 19 | "enable.2pc": "true", 20 | "max.retries": "10", 21 | "retry.interval.ms": "5000", 22 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 23 | "value.converter":"org.apache.kafka.connect.storage.StringConverter" 24 | } 25 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/string_msg_failover_connector_uniq.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"string_msg_failover_connector_uniq", 3 | "config":{ 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"string_test_failover_uniq", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "string_test_failover_uniq:string_msg_tab_failover_uniq", 8 | "buffer.count.records":"1", 9 | "buffer.flush.time":"1200", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"string_msg_failover", 17 | "load.model":"stream_load", 18 | "enable.2pc": "true", 19 | "enable.combine.flush":"true", 20 | "max.retries": "10", 21 | "retry.interval.ms": "5000", 22 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 23 | "value.converter":"org.apache.kafka.connect.storage.StringConverter" 24 | } 25 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/string_msg_tab.sql: -------------------------------------------------------------------------------- 1 | -- Please note that the database here should be consistent with doris.database in the file where the connector is registered. 2 | CREATE TABLE string_msg.string_msg_tab ( 3 | id INT NULL, 4 | name VARCHAR(100) NULL, 5 | age INT NULL 6 | ) ENGINE=OLAP 7 | UNIQUE KEY(`id`) 8 | COMMENT 'OLAP' 9 | DISTRIBUTED BY HASH(`id`) BUCKETS AUTO 10 | PROPERTIES ( 11 | "replication_allocation" = "tag.location.default: 1" 12 | ); -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/string_msg_tab_failover.sql: -------------------------------------------------------------------------------- 1 | -- Please note that the database here should be consistent with doris.database in the file where the connector is registered. 2 | CREATE TABLE string_msg_failover.string_msg_tab_failover ( 3 | id INT NULL, 4 | name VARCHAR(100) NULL, 5 | age INT NULL 6 | ) ENGINE=OLAP 7 | DUPLICATE KEY(`id`) 8 | COMMENT 'OLAP' 9 | DISTRIBUTED BY HASH(`id`) BUCKETS AUTO 10 | PROPERTIES ( 11 | "replication_allocation" = "tag.location.default: 1" 12 | ); -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/string_msg_tab_failover_uniq.sql: -------------------------------------------------------------------------------- 1 | -- Please note that the database here should be consistent with doris.database in the file where the connector is registered. 2 | CREATE TABLE string_msg_failover.string_msg_tab_failover_uniq ( 3 | id INT NULL, 4 | name VARCHAR(100) NULL, 5 | age INT NULL 6 | ) ENGINE=OLAP 7 | UNIQUE KEY(`id`) 8 | COMMENT 'OLAP' 9 | DISTRIBUTED BY HASH(`id`) BUCKETS AUTO 10 | PROPERTIES ( 11 | "replication_allocation" = "tag.location.default: 1" 12 | ); -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/table_field_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"table_field_config_connector", 3 | "config":{ 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"table_field_config_test", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "table_field_config_test:field_config_tab1", 8 | "buffer.count.records":"2", 9 | "buffer.flush.time":"10", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"string_msg", 17 | "record.tablename.field": "table_name", 18 | "enable.2pc": "false", 19 | "load.model":"stream_load", 20 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 21 | "value.converter":"org.apache.kafka.connect.json.JsonConverter", 22 | "value.converter.schemas.enable": "false" 23 | } 24 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/table_field_config1.sql: -------------------------------------------------------------------------------- 1 | -- Please note that the database here should be consistent with doris.database in the file where the connector is registered. 2 | CREATE TABLE string_msg.field_config_tab1 ( 3 | id INT NULL, 4 | col1 VARCHAR(20) NULL, 5 | col2 varchar(20) NULL 6 | ) ENGINE=OLAP 7 | UNIQUE KEY(`id`) 8 | COMMENT 'OLAP' 9 | DISTRIBUTED BY HASH(`id`) BUCKETS AUTO 10 | PROPERTIES ( 11 | "replication_allocation" = "tag.location.default: 1" 12 | ); -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/table_field_config2.sql: -------------------------------------------------------------------------------- 1 | -- Please note that the database here should be consistent with doris.database in the file where the connector is registered. 2 | CREATE TABLE string_msg.field_config_tab2 ( 3 | id INT NULL, 4 | col1 VARCHAR(20) NULL, 5 | col2 varchar(20) NULL 6 | ) ENGINE=OLAP 7 | UNIQUE KEY(`id`) 8 | COMMENT 'OLAP' 9 | DISTRIBUTED BY HASH(`id`) BUCKETS AUTO 10 | PROPERTIES ( 11 | "replication_allocation" = "tag.location.default: 1" 12 | ); -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/time_types.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"postgres_time_example", 3 | "config":{ 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"time_example", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "time_example:time_example", 8 | "buffer.count.records":"1", 9 | "buffer.flush.time":"10", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"test_time", 17 | "converter.mode": "debezium_ingestion", 18 | "database.time_zone":"Asia/Shanghai", 19 | "key.converter":"org.apache.kafka.connect.json.JsonConverter", 20 | "value.converter":"org.apache.kafka.connect.json.JsonConverter" 21 | } 22 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/string_converter/time_types.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE test_time.time_example ( 2 | id int, 3 | timestamp_without_timezone datetime, 4 | timestamp_with_timezone datetime, 5 | date_only DATE, 6 | time_without_timezone text, 7 | time_with_timezone text, 8 | interval_period bigint 9 | )ENGINE=OLAP 10 | UNIQUE KEY(`id`) 11 | COMMENT 'OLAP' 12 | DISTRIBUTED BY HASH(`id`) BUCKETS AUTO 13 | PROPERTIES ( 14 | "replication_allocation" = "tag.location.default: 1" 15 | ); 16 | -------------------------------------------------------------------------------- /src/test/resources/e2e/transforms/multiple_transforms_chain.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"multiple_transforms_chain_connector", 3 | "config":{ 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"multiple_transforms_chain_test", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "multiple_transforms_chain_test:multiple_transforms_chain_tab", 8 | "buffer.count.records":"2", 9 | "buffer.flush.time":"11", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"transforms_msg", 17 | "load.model":"stream_load", 18 | "transforms":"extractField,renameField", 19 | "transforms.extractField.type": "org.apache.kafka.connect.transforms.ExtractField$Value", 20 | "transforms.extractField.field": "content", 21 | "transforms.renameField.type":"org.apache.kafka.connect.transforms.ReplaceField$Value", 22 | "transforms.renameField.renames":"old_col1:col1", 23 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 24 | "value.converter":"org.apache.kafka.connect.json.JsonConverter", 25 | "value.converter.schemas.enable": "false" 26 | } 27 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/transforms/multiple_transforms_chain.sql: -------------------------------------------------------------------------------- 1 | -- Please note that the database here should be consistent with doris.database in the file where the connector is registered. 2 | CREATE TABLE transforms_msg.multiple_transforms_chain_tab ( 3 | id INT NULL, 4 | col1 VARCHAR(20) NULL 5 | ) ENGINE=OLAP 6 | UNIQUE KEY(`id`) 7 | COMMENT 'OLAP' 8 | DISTRIBUTED BY HASH(`id`) BUCKETS AUTO 9 | PROPERTIES ( 10 | "replication_allocation" = "tag.location.default: 1" 11 | ); -------------------------------------------------------------------------------- /src/test/resources/e2e/transforms/regex_router_transforms.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"regex_router_transforms_connector", 3 | "config":{ 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"p-regex_router_transform_msg", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "p-regex_router_transform_msg:regex_router_transform_msg,regex_router_transform_msg:regex_router_transform_msg", 8 | "buffer.count.records":"2", 9 | "buffer.flush.time":"11", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"transforms_msg", 17 | "load.model":"stream_load", 18 | "transforms": "dropPrefix", 19 | "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter", 20 | "transforms.dropPrefix.regex": "p-(.*)", 21 | "transforms.dropPrefix.replacement": "$1", 22 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 23 | "value.converter":"org.apache.kafka.connect.json.JsonConverter", 24 | "value.converter.schemas.enable": "false" 25 | } 26 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/transforms/regex_router_transforms.sql: -------------------------------------------------------------------------------- 1 | -- Please note that the database here should be consistent with doris.database in the file where the connector is registered. 2 | CREATE TABLE transforms_msg.regex_router_transform_msg ( 3 | id INT NULL, 4 | col1 VARCHAR(20) NULL, 5 | col2 varchar(20) NULL 6 | ) ENGINE=OLAP 7 | UNIQUE KEY(`id`) 8 | COMMENT 'OLAP' 9 | DISTRIBUTED BY HASH(`id`) BUCKETS AUTO 10 | PROPERTIES ( 11 | "replication_allocation" = "tag.location.default: 1" 12 | ); -------------------------------------------------------------------------------- /src/test/resources/e2e/transforms/rename_transforms.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"rename_transforms_connector", 3 | "config":{ 4 | "connector.class":"org.apache.doris.kafka.connector.DorisSinkConnector", 5 | "topics":"kf_rename_transform_msg", 6 | "tasks.max":"1", 7 | "doris.topic2table.map": "kf_rename_transform_msg:rename_transform_msg", 8 | "buffer.count.records":"2", 9 | "buffer.flush.time":"11", 10 | "buffer.size.bytes":"10000000", 11 | "doris.urls":"127.0.0.1", 12 | "doris.user":"root", 13 | "doris.password":"", 14 | "doris.http.port":"8030", 15 | "doris.query.port":"9030", 16 | "doris.database":"transforms_msg", 17 | "load.model":"stream_load", 18 | "transforms":"renameField", 19 | "transforms.renameField.type":"org.apache.kafka.connect.transforms.ReplaceField$Value", 20 | "transforms.renameField.renames":"old_col1:col1", 21 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 22 | "value.converter":"org.apache.kafka.connect.json.JsonConverter", 23 | "value.converter.schemas.enable": "false" 24 | } 25 | } -------------------------------------------------------------------------------- /src/test/resources/e2e/transforms/rename_transforms.sql: -------------------------------------------------------------------------------- 1 | -- Please note that the database here should be consistent with doris.database in the file where the connector is registered. 2 | CREATE TABLE transforms_msg.rename_transform_msg ( 3 | id INT NULL, 4 | col1 VARCHAR(20) NULL, 5 | col2 varchar(20) NULL 6 | ) ENGINE=OLAP 7 | UNIQUE KEY(`id`) 8 | COMMENT 'OLAP' 9 | DISTRIBUTED BY HASH(`id`) BUCKETS AUTO 10 | PROPERTIES ( 11 | "replication_allocation" = "tag.location.default: 1" 12 | ); -------------------------------------------------------------------------------- /src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | log4j.rootLogger=INFO, console 21 | 22 | log4j.appender.console=org.apache.log4j.ConsoleAppender 23 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 24 | log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %c{1}(%L) [%t] - %m%n 25 | 26 | log4j.appender.file=org.apache.log4j.RollingFileAppender 27 | log4j.appender.file.File=doris-kafka-connector.log 28 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 29 | log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %c{1}(%L) [%t] - %m%n 30 | 31 | --------------------------------------------------------------------------------