├── .asf.yaml ├── .github ├── boring-cyborg.yml └── workflows │ ├── push_pr.yml │ └── weekly.yml ├── .gitignore ├── .gitmodules ├── .idea └── vcs.xml ├── LICENSE ├── NOTICE ├── README.md ├── docs ├── content.zh │ └── docs │ │ └── connectors │ │ └── table │ │ └── hbase.md ├── content │ └── docs │ │ └── connectors │ │ └── table │ │ └── hbase.md └── data │ └── hbase.yml ├── flink-connector-hbase-2.2 ├── archunit-violations │ ├── 346fff47-b126-4f8c-ba73-799d99856912 │ ├── ac3990d6-8d39-4299-8765-3ae2f91d76af │ └── stored.rules ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── flink │ │ │ └── connector │ │ │ └── hbase2 │ │ │ ├── HBase2DynamicTableFactory.java │ │ │ ├── sink │ │ │ └── HBaseDynamicTableSink.java │ │ │ └── source │ │ │ ├── AbstractTableInputFormat.java │ │ │ ├── HBaseDynamicTableSource.java │ │ │ ├── HBaseRowDataAsyncLookupFunction.java │ │ │ └── HBaseRowDataInputFormat.java │ └── resources │ │ └── META-INF │ │ └── services │ │ └── org.apache.flink.table.factories.Factory │ └── test │ ├── java │ └── org │ │ ├── apache │ │ └── flink │ │ │ ├── architecture │ │ │ └── TestCodeArchitectureTest.java │ │ │ └── connector │ │ │ └── hbase2 │ │ │ ├── HBaseConnectorITCase.java │ │ │ ├── HBaseDynamicTableFactoryTest.java │ │ │ ├── HBaseTablePlanTest.java │ │ │ ├── source │ │ │ └── HBaseRowDataAsyncLookupFunctionTest.java │ │ │ └── util │ │ │ ├── HBaseTestBase.java │ │ │ └── HBaseTestingClusterAutoStarter.java │ │ ├── eclipse │ │ └── jetty │ │ │ └── util │ │ │ └── JavaVersion.java │ │ └── slf4j │ │ └── impl │ │ └── Log4jLoggerAdapter.java │ └── resources │ ├── archunit.properties │ ├── hbase-site.xml │ ├── log4j2-test.properties │ └── org │ └── apache │ └── flink │ └── connector │ └── hbase2 │ └── HBaseTablePlanTest.xml ├── flink-connector-hbase-base ├── archunit-violations │ ├── f8f77b71-7087-4b3f-88c5-29588fadcc52 │ ├── fd028eab-0e49-4e91-9c40-477a55f378f2 │ └── stored.rules ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── flink │ │ └── connector │ │ └── hbase │ │ ├── options │ │ └── HBaseWriteOptions.java │ │ ├── sink │ │ ├── HBaseMutationConverter.java │ │ ├── HBaseSinkFunction.java │ │ ├── RowDataToMutationConverter.java │ │ └── WritableMetadata.java │ │ ├── source │ │ ├── AbstractHBaseDynamicTableSource.java │ │ ├── HBaseRowDataLookupFunction.java │ │ └── TableInputSplit.java │ │ ├── table │ │ ├── HBaseConnectorOptions.java │ │ └── HBaseConnectorOptionsUtil.java │ │ └── util │ │ ├── HBaseConfigurationUtil.java │ │ ├── HBaseSerde.java │ │ ├── HBaseTableSchema.java │ │ └── HBaseTypeUtils.java │ └── test │ ├── java │ └── org │ │ └── apache │ │ └── flink │ │ ├── architecture │ │ └── TestCodeArchitectureTest.java │ │ └── connector │ │ └── hbase │ │ └── util │ │ ├── HBaseConfigLoadingTest.java │ │ └── HBaseSerdeTest.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.junit.jupiter.api.extension.Extension │ ├── archunit.properties │ ├── hbase-site.xml │ └── log4j2-test.properties ├── flink-connector-hbase-e2e-tests ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── flink │ │ └── streaming │ │ └── tests │ │ └── HBaseContainer.java │ └── test │ ├── java │ └── org │ │ └── apache │ │ └── flink │ │ └── streaming │ │ └── tests │ │ └── HBaseITCase.java │ └── resources │ ├── hbase_e2e.sql │ └── log4j2-test.properties ├── flink-sql-connector-hbase-2.2 ├── pom.xml └── src │ └── main │ └── resources │ ├── META-INF │ ├── NOTICE │ └── licenses │ │ └── LICENSE.protobuf │ └── hbase-default.xml ├── pom.xml └── tools ├── ci └── log4j.properties └── maven ├── checkstyle.xml └── suppressions.xml /.asf.yaml: -------------------------------------------------------------------------------- 1 | github: 2 | enabled_merge_buttons: 3 | squash: true 4 | merge: false 5 | rebase: true 6 | labels: 7 | - flink 8 | - hbase 9 | - connector 10 | - datastream 11 | - table 12 | - sql 13 | autolink_jira: FLINK 14 | collaborators: 15 | - flinkbot 16 | notifications: 17 | commits: commits@flink.apache.org 18 | issues: issues@flink.apache.org 19 | pullrequests: issues@flink.apache.org 20 | jobs: builds@flink.apache.org 21 | jira_options: link label 22 | -------------------------------------------------------------------------------- /.github/boring-cyborg.yml: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | ################################################################################ 18 | 19 | labelPRBasedOnFilePath: 20 | component=BuildSystem: 21 | - .github/**/* 22 | - tools/maven/* 23 | 24 | component=Documentation: 25 | - docs/**/* 26 | 27 | component=Connectors/HBase: 28 | - flink-connector-hbase*/**/* 29 | - flink-sql-connector-hbase*/**/* 30 | 31 | ###### IssueLink Adder ################################################################################################# 32 | # Insert Issue (Jira/Github etc) link in PR description based on the Issue ID in PR title. 33 | insertIssueLinkInPrDescription: 34 | # specify the placeholder for the issue link that should be present in the description 35 | descriptionIssuePlaceholderRegexp: "^Issue link: (.*)$" 36 | matchers: 37 | # you can have several matches - for different types of issues 38 | # only the first matching entry is replaced 39 | jiraIssueMatch: 40 | # specify the regexp of issue id that you can find in the title of the PR 41 | # the match groups can be used to build the issue id (${1}, ${2}, etc.). 42 | titleIssueIdRegexp: \[(FLINK-[0-9]+)\] 43 | # the issue link to be added. ${1}, ${2} ... are replaced with the match groups from the 44 | # title match (remember to use quotes) 45 | descriptionIssueLink: "[${1}](https://issues.apache.org/jira/browse/${1}/)" 46 | docOnlyIssueMatch: 47 | titleIssueIdRegexp: \[hotfix\] 48 | descriptionIssueLink: "`Documentation only change, no JIRA issue`" 49 | 50 | ###### Title Validator ################################################################################################# 51 | # Verifies if commit/PR titles match the regexp specified 52 | verifyTitles: 53 | # Regular expression that should be matched by titles of commits or PR 54 | titleRegexp: ^\[FLINK-[0-9]+\].*$|^\[FLINK-XXXXX\].*$|^\[hotfix].*$ 55 | # If set to true, it will always check the PR title (as opposed to the individual commits). 56 | alwaysUsePrTitle: false 57 | # If set to true, it will only check the commit in case there is a single commit. 58 | # In case of multiple commits it will check PR title. 59 | # This reflects the standard behaviour of Github that for `Squash & Merge` GitHub 60 | # uses the PR title rather than commit messages for the squashed commit ¯\_(ツ)_/¯ 61 | # For single-commit PRs it takes the squashed commit message from the commit as expected. 62 | # 63 | # If set to false it will check all commit messages. This is useful when you do not squash commits at merge. 64 | validateEitherPrOrSingleCommitTitle: true 65 | # The title the GitHub status should appear from. 66 | statusTitle: "Title Validator" 67 | # A custom message to be displayed when the title passes validation. 68 | successMessage: "Validation successful!" 69 | # A custom message to be displayed when the title fails validation. 70 | # Allows insertion of ${type} (commit/PR), ${title} (the title validated) and ${regex} (the titleRegexp above). 71 | failureMessage: "Wrong ${type} title: ${title}" 72 | 73 | # Various Flags to control behaviour of the "Labeler" 74 | labelerFlags: 75 | # If this flag is changed to 'false', labels would only be added when the PR is first created 76 | # and not when existing PR is updated. 77 | # The default is 'true' which means the labels would be added when PR is updated even if they 78 | # were removed by the user 79 | labelOnPRUpdates: true 80 | 81 | # Comment to be posted to welcome users when they open their first PR 82 | firstPRWelcomeComment: > 83 | Thanks for opening this pull request! Please check out our contributing guidelines. (https://flink.apache.org/contributing/how-to-contribute.html) 84 | 85 | # Comment to be posted to congratulate user on their first merged PR 86 | firstPRMergeComment: > 87 | Awesome work, congrats on your first merged pull request! -------------------------------------------------------------------------------- /.github/workflows/push_pr.yml: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | ################################################################################ 18 | 19 | # We need to specify repo related information here since Apache INFRA doesn't differentiate 20 | # between several workflows with the same names while preparing a report for GHA usage 21 | # https://infra-reports.apache.org/#ghactions 22 | name: Flink Connector HBase CI 23 | on: [push, pull_request] 24 | concurrency: 25 | group: ${{ github.workflow }}-${{ github.ref }} 26 | cancel-in-progress: true 27 | jobs: 28 | compile_and_test: 29 | strategy: 30 | matrix: 31 | flink: [ 1.20.0 ] 32 | jdk: [ '8, 11' ] 33 | uses: apache/flink-connector-shared-utils/.github/workflows/ci.yml@ci_utils 34 | with: 35 | flink_version: ${{ matrix.flink }} 36 | jdk_version: ${{ matrix.jdk }} 37 | timeout_global: 120 38 | timeout_test: 80 39 | -------------------------------------------------------------------------------- /.github/workflows/weekly.yml: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | ################################################################################ 18 | 19 | # We need to specify repo related information here since Apache INFRA doesn't differentiate 20 | # between several workflows with the same names while preparing a report for GHA usage 21 | # https://infra-reports.apache.org/#ghactions 22 | name: Weekly Flink Connector HBase CI 23 | on: 24 | schedule: 25 | - cron: "0 0 * * 0" 26 | workflow_dispatch: 27 | jobs: 28 | compile_and_test: 29 | if: github.repository_owner == 'apache' 30 | strategy: 31 | matrix: 32 | flink_branches: [{ 33 | flink: 1.20-SNAPSHOT, 34 | branch: main 35 | }, { 36 | flink: 1.19.1, 37 | branch: v4.0 38 | }, { 39 | flink: 1.18.1, 40 | branch: v4.0 41 | }] 42 | uses: apache/flink-connector-shared-utils/.github/workflows/ci.yml@ci_utils 43 | with: 44 | flink_version: ${{ matrix.flink_branches.flink }} 45 | connector_branch: ${{ matrix.flink_branches.branch }} 46 | jdk_version: ${{ matrix.flink_branches.jdk || '8, 11' }} 47 | run_dependency_convergence: false 48 | timeout_global: 120 49 | timeout_test: 80 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .eslintcache 2 | .cache 3 | scalastyle-output.xml 4 | .classpath 5 | .idea/* 6 | !.idea/vcs.xml 7 | .metadata 8 | .settings 9 | .project 10 | .version.properties 11 | filter.properties 12 | logs.zip 13 | .mvn/wrapper/*.jar 14 | target 15 | tmp 16 | *.class 17 | *.iml 18 | *.swp 19 | *.jar 20 | *.zip 21 | *.log 22 | *.pyc 23 | .DS_Store 24 | build-target 25 | atlassian-ide-plugin.xml 26 | out/ 27 | /docs/api 28 | /docs/.bundle 29 | /docs/.rubydeps 30 | /docs/ruby2/.bundle 31 | /docs/ruby2/.rubydeps 32 | /docs/.jekyll-metadata 33 | *.ipr 34 | *.iws 35 | tools/flink 36 | tools/flink-* 37 | tools/releasing/release 38 | tools/japicmp-output -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "tools/releasing/shared"] 2 | path = tools/releasing/shared 3 | url = https://github.com/apache/flink-connector-shared-utils 4 | branch = release_utils 5 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Apache Flink HBase Connector 2 | Copyright 2014-2024 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | 7 | Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby 8 | granted, provided that this permission notice appear in all copies. 9 | 10 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING 11 | ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, 12 | DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, 13 | WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE 14 | USE OR PERFORMANCE OF THIS SOFTWARE. 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Apache Flink HBase Connector 2 | 3 | This repository contains the official Apache Flink HBase connector. 4 | 5 | ## Apache Flink 6 | 7 | Apache Flink is an open source stream processing framework with powerful stream- and batch-processing capabilities. 8 | 9 | Learn more about Flink at [https://flink.apache.org/](https://flink.apache.org/) 10 | 11 | ## Building the Apache Flink HBase Connector from Source 12 | 13 | Prerequisites: 14 | 15 | * Unix-like environment (we use Linux, Mac OS X) 16 | * Git 17 | * Maven (we recommend version 3.8.6) 18 | * Java 11 19 | 20 | ``` 21 | git clone https://github.com/apache/flink-connector-hbase.git 22 | cd flink-connector-hbase 23 | mvn clean package -DskipTests 24 | ``` 25 | 26 | The resulting jars can be found in the `target` directory of the respective module. 27 | 28 | ## Developing Flink 29 | 30 | The Flink committers use IntelliJ IDEA to develop the Flink codebase. 31 | We recommend IntelliJ IDEA for developing projects that involve Scala code. 32 | 33 | Minimal requirements for an IDE are: 34 | * Support for Java and Scala (also mixed projects) 35 | * Support for Maven with Java and Scala 36 | 37 | ### IntelliJ IDEA 38 | 39 | The IntelliJ IDE supports Maven out of the box and offers a plugin for Scala development. 40 | 41 | * IntelliJ download: [https://www.jetbrains.com/idea/](https://www.jetbrains.com/idea/) 42 | * IntelliJ Scala Plugin: [https://plugins.jetbrains.com/plugin/?id=1347](https://plugins.jetbrains.com/plugin/?id=1347) 43 | 44 | Check out our [Setting up IntelliJ](https://nightlies.apache.org/flink/flink-docs-master/flinkDev/ide_setup.html#intellij-idea) guide for details. 45 | 46 | ## Support 47 | 48 | Don’t hesitate to ask! 49 | 50 | Contact the developers and community on the [mailing lists](https://flink.apache.org/community.html#mailing-lists) if you need any help. 51 | 52 | [Open an issue](https://issues.apache.org/jira/browse/FLINK) if you found a bug in Flink. 53 | 54 | ## Documentation 55 | 56 | The documentation of Apache Flink is located on the website: [https://flink.apache.org](https://flink.apache.org) 57 | or in the `docs/` directory of the source code. 58 | 59 | ## Fork and Contribute 60 | 61 | This is an active open-source project. We are always open to people who want to use the system or contribute to it. 62 | Contact us if you are looking for implementation tasks that fit your skills. 63 | This article describes [how to contribute to Apache Flink](https://flink.apache.org/contributing/how-to-contribute.html). 64 | 65 | ## About 66 | 67 | Apache Flink is an open source project of The Apache Software Foundation (ASF). 68 | The Apache Flink project originated from the [Stratosphere](http://stratosphere.eu) research project. 69 | -------------------------------------------------------------------------------- /docs/data/hbase.yml: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | ################################################################################ 18 | 19 | version: 4.0.0 20 | flink_compatibility: ["1.18", "1.19"] 21 | variants: 22 | - maven: flink-connector-hbase-2.2 23 | sql_url: https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-hbase-2.2/$full_version/flink-sql-connector-hbase-2.2-$full_version.jar 24 | -------------------------------------------------------------------------------- /flink-connector-hbase-2.2/archunit-violations/346fff47-b126-4f8c-ba73-799d99856912: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/flink-connector-hbase/ca5a4aae3e953841371e9a568ad8472a5ef8c1d2/flink-connector-hbase-2.2/archunit-violations/346fff47-b126-4f8c-ba73-799d99856912 -------------------------------------------------------------------------------- /flink-connector-hbase-2.2/archunit-violations/ac3990d6-8d39-4299-8765-3ae2f91d76af: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/flink-connector-hbase/ca5a4aae3e953841371e9a568ad8472a5ef8c1d2/flink-connector-hbase-2.2/archunit-violations/ac3990d6-8d39-4299-8765-3ae2f91d76af -------------------------------------------------------------------------------- /flink-connector-hbase-2.2/archunit-violations/stored.rules: -------------------------------------------------------------------------------- 1 | # 2 | #Tue Feb 22 12:17:50 CET 2022 3 | Tests\ inheriting\ from\ AbstractTestBase\ should\ have\ name\ ending\ with\ ITCase=346fff47-b126-4f8c-ba73-799d99856912 4 | ITCASE\ tests\ should\ use\ a\ MiniCluster\ resource\ or\ extension=ac3990d6-8d39-4299-8765-3ae2f91d76af 5 | -------------------------------------------------------------------------------- /flink-connector-hbase-2.2/src/main/java/org/apache/flink/connector/hbase2/HBase2DynamicTableFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase2; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | import org.apache.flink.configuration.ConfigOption; 23 | import org.apache.flink.configuration.ReadableConfig; 24 | import org.apache.flink.connector.hbase.options.HBaseWriteOptions; 25 | import org.apache.flink.connector.hbase.util.HBaseTableSchema; 26 | import org.apache.flink.connector.hbase2.sink.HBaseDynamicTableSink; 27 | import org.apache.flink.connector.hbase2.source.HBaseDynamicTableSource; 28 | import org.apache.flink.table.connector.sink.DynamicTableSink; 29 | import org.apache.flink.table.connector.source.DynamicTableSource; 30 | import org.apache.flink.table.connector.source.lookup.LookupOptions; 31 | import org.apache.flink.table.connector.source.lookup.cache.DefaultLookupCache; 32 | import org.apache.flink.table.connector.source.lookup.cache.LookupCache; 33 | import org.apache.flink.table.factories.DynamicTableSinkFactory; 34 | import org.apache.flink.table.factories.DynamicTableSourceFactory; 35 | import org.apache.flink.table.factories.FactoryUtil.TableFactoryHelper; 36 | 37 | import org.apache.hadoop.conf.Configuration; 38 | 39 | import java.time.Duration; 40 | import java.util.HashSet; 41 | import java.util.Set; 42 | import java.util.stream.Collectors; 43 | import java.util.stream.Stream; 44 | 45 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.LOOKUP_ASYNC; 46 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.LOOKUP_CACHE_MAX_ROWS; 47 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.LOOKUP_CACHE_TTL; 48 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.LOOKUP_MAX_RETRIES; 49 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.NULL_STRING_LITERAL; 50 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.SINK_BUFFER_FLUSH_INTERVAL; 51 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.SINK_BUFFER_FLUSH_MAX_ROWS; 52 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.SINK_BUFFER_FLUSH_MAX_SIZE; 53 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.SINK_IGNORE_NULL_VALUE; 54 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.SINK_PARALLELISM; 55 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.TABLE_NAME; 56 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.ZOOKEEPER_QUORUM; 57 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.ZOOKEEPER_ZNODE_PARENT; 58 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptionsUtil.PROPERTIES_PREFIX; 59 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptionsUtil.getHBaseConfiguration; 60 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptionsUtil.getHBaseWriteOptions; 61 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptionsUtil.validatePrimaryKey; 62 | import static org.apache.flink.table.factories.FactoryUtil.createTableFactoryHelper; 63 | 64 | /** HBase connector factory. */ 65 | @Internal 66 | public class HBase2DynamicTableFactory 67 | implements DynamicTableSourceFactory, DynamicTableSinkFactory { 68 | 69 | private static final String IDENTIFIER = "hbase-2.2"; 70 | 71 | @Override 72 | public DynamicTableSource createDynamicTableSource(Context context) { 73 | TableFactoryHelper helper = createTableFactoryHelper(this, context); 74 | helper.validateExcept(PROPERTIES_PREFIX); 75 | 76 | final ReadableConfig tableOptions = helper.getOptions(); 77 | 78 | validatePrimaryKey(context.getPhysicalRowDataType(), context.getPrimaryKeyIndexes()); 79 | 80 | String tableName = tableOptions.get(TABLE_NAME); 81 | Configuration hbaseConf = getHBaseConfiguration(tableOptions); 82 | String nullStringLiteral = tableOptions.get(NULL_STRING_LITERAL); 83 | HBaseTableSchema hbaseSchema = 84 | HBaseTableSchema.fromDataType(context.getPhysicalRowDataType()); 85 | 86 | LookupCache cache = null; 87 | 88 | // Backward compatible to legacy caching options 89 | if (tableOptions.get(LOOKUP_CACHE_MAX_ROWS) > 0 90 | && tableOptions.get(LOOKUP_CACHE_TTL).compareTo(Duration.ZERO) > 0) { 91 | cache = 92 | DefaultLookupCache.newBuilder() 93 | .maximumSize(tableOptions.get(LOOKUP_CACHE_MAX_ROWS)) 94 | .expireAfterWrite(tableOptions.get(LOOKUP_CACHE_TTL)) 95 | .build(); 96 | } 97 | 98 | if (tableOptions 99 | .get(LookupOptions.CACHE_TYPE) 100 | .equals(LookupOptions.LookupCacheType.PARTIAL)) { 101 | cache = DefaultLookupCache.fromConfig(tableOptions); 102 | } 103 | 104 | return new HBaseDynamicTableSource( 105 | hbaseConf, 106 | tableName, 107 | hbaseSchema, 108 | nullStringLiteral, 109 | tableOptions.get(LookupOptions.MAX_RETRIES), 110 | tableOptions.get(LOOKUP_ASYNC), 111 | cache); 112 | } 113 | 114 | @Override 115 | public DynamicTableSink createDynamicTableSink(Context context) { 116 | TableFactoryHelper helper = createTableFactoryHelper(this, context); 117 | helper.validateExcept(PROPERTIES_PREFIX); 118 | 119 | final ReadableConfig tableOptions = helper.getOptions(); 120 | 121 | validatePrimaryKey(context.getPhysicalRowDataType(), context.getPrimaryKeyIndexes()); 122 | 123 | String tableName = tableOptions.get(TABLE_NAME); 124 | Configuration hbaseConf = getHBaseConfiguration(tableOptions); 125 | HBaseWriteOptions hBaseWriteOptions = getHBaseWriteOptions(tableOptions); 126 | String nullStringLiteral = tableOptions.get(NULL_STRING_LITERAL); 127 | 128 | return new HBaseDynamicTableSink( 129 | tableName, 130 | context.getPhysicalRowDataType(), 131 | hbaseConf, 132 | hBaseWriteOptions, 133 | nullStringLiteral); 134 | } 135 | 136 | @Override 137 | public String factoryIdentifier() { 138 | return IDENTIFIER; 139 | } 140 | 141 | @Override 142 | public Set> requiredOptions() { 143 | Set> set = new HashSet<>(); 144 | set.add(TABLE_NAME); 145 | return set; 146 | } 147 | 148 | @Override 149 | public Set> optionalOptions() { 150 | Set> set = new HashSet<>(); 151 | set.add(ZOOKEEPER_ZNODE_PARENT); 152 | set.add(ZOOKEEPER_QUORUM); 153 | set.add(NULL_STRING_LITERAL); 154 | set.add(SINK_BUFFER_FLUSH_MAX_SIZE); 155 | set.add(SINK_BUFFER_FLUSH_MAX_ROWS); 156 | set.add(SINK_BUFFER_FLUSH_INTERVAL); 157 | set.add(SINK_PARALLELISM); 158 | set.add(SINK_IGNORE_NULL_VALUE); 159 | set.add(LOOKUP_ASYNC); 160 | set.add(LOOKUP_CACHE_MAX_ROWS); 161 | set.add(LOOKUP_CACHE_TTL); 162 | set.add(LOOKUP_MAX_RETRIES); 163 | set.add(LookupOptions.CACHE_TYPE); 164 | set.add(LookupOptions.MAX_RETRIES); 165 | set.add(LookupOptions.PARTIAL_CACHE_EXPIRE_AFTER_ACCESS); 166 | set.add(LookupOptions.PARTIAL_CACHE_EXPIRE_AFTER_WRITE); 167 | set.add(LookupOptions.PARTIAL_CACHE_CACHE_MISSING_KEY); 168 | set.add(LookupOptions.PARTIAL_CACHE_MAX_ROWS); 169 | return set; 170 | } 171 | 172 | @Override 173 | public Set> forwardOptions() { 174 | return Stream.of( 175 | TABLE_NAME, 176 | ZOOKEEPER_ZNODE_PARENT, 177 | ZOOKEEPER_QUORUM, 178 | NULL_STRING_LITERAL, 179 | LOOKUP_CACHE_MAX_ROWS, 180 | LOOKUP_CACHE_TTL, 181 | LOOKUP_MAX_RETRIES, 182 | SINK_BUFFER_FLUSH_MAX_SIZE, 183 | SINK_BUFFER_FLUSH_MAX_ROWS, 184 | SINK_BUFFER_FLUSH_INTERVAL, 185 | SINK_IGNORE_NULL_VALUE) 186 | .collect(Collectors.toSet()); 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /flink-connector-hbase-2.2/src/main/java/org/apache/flink/connector/hbase2/sink/HBaseDynamicTableSink.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase2.sink; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | import org.apache.flink.annotation.VisibleForTesting; 23 | import org.apache.flink.connector.hbase.options.HBaseWriteOptions; 24 | import org.apache.flink.connector.hbase.sink.HBaseSinkFunction; 25 | import org.apache.flink.connector.hbase.sink.RowDataToMutationConverter; 26 | import org.apache.flink.connector.hbase.sink.WritableMetadata; 27 | import org.apache.flink.connector.hbase.util.HBaseTableSchema; 28 | import org.apache.flink.table.connector.ChangelogMode; 29 | import org.apache.flink.table.connector.sink.DynamicTableSink; 30 | import org.apache.flink.table.connector.sink.SinkFunctionProvider; 31 | import org.apache.flink.table.connector.sink.abilities.SupportsWritingMetadata; 32 | import org.apache.flink.table.data.RowData; 33 | import org.apache.flink.table.types.DataType; 34 | import org.apache.flink.types.RowKind; 35 | 36 | import org.apache.hadoop.conf.Configuration; 37 | 38 | import java.util.Collections; 39 | import java.util.List; 40 | import java.util.Map; 41 | 42 | /** HBase table sink implementation. */ 43 | @Internal 44 | public class HBaseDynamicTableSink implements DynamicTableSink, SupportsWritingMetadata { 45 | 46 | private final String tableName; 47 | private final HBaseTableSchema hbaseTableSchema; 48 | private final Configuration hbaseConf; 49 | private final HBaseWriteOptions writeOptions; 50 | private final String nullStringLiteral; 51 | private final DataType physicalDataType; 52 | 53 | /** Metadata that is appended at the end of a physical sink row. */ 54 | private List metadataKeys; 55 | 56 | public HBaseDynamicTableSink( 57 | String tableName, 58 | DataType physicalDataType, 59 | Configuration hbaseConf, 60 | HBaseWriteOptions writeOptions, 61 | String nullStringLiteral) { 62 | this.tableName = tableName; 63 | this.physicalDataType = physicalDataType; 64 | this.hbaseTableSchema = HBaseTableSchema.fromDataType(physicalDataType); 65 | this.metadataKeys = Collections.emptyList(); 66 | this.hbaseConf = hbaseConf; 67 | this.writeOptions = writeOptions; 68 | this.nullStringLiteral = nullStringLiteral; 69 | } 70 | 71 | @Override 72 | public SinkRuntimeProvider getSinkRuntimeProvider(Context context) { 73 | HBaseSinkFunction sinkFunction = 74 | new HBaseSinkFunction<>( 75 | tableName, 76 | hbaseConf, 77 | new RowDataToMutationConverter( 78 | hbaseTableSchema, 79 | physicalDataType, 80 | metadataKeys, 81 | nullStringLiteral, 82 | writeOptions.isIgnoreNullValue()), 83 | writeOptions.getBufferFlushMaxSizeInBytes(), 84 | writeOptions.getBufferFlushMaxRows(), 85 | writeOptions.getBufferFlushIntervalMillis()); 86 | return SinkFunctionProvider.of(sinkFunction, writeOptions.getParallelism()); 87 | } 88 | 89 | @Override 90 | public ChangelogMode getChangelogMode(ChangelogMode requestedMode) { 91 | // UPSERT mode 92 | ChangelogMode.Builder builder = ChangelogMode.newBuilder(); 93 | for (RowKind kind : requestedMode.getContainedKinds()) { 94 | if (kind != RowKind.UPDATE_BEFORE) { 95 | builder.addContainedKind(kind); 96 | } 97 | } 98 | return builder.build(); 99 | } 100 | 101 | @Override 102 | public Map listWritableMetadata() { 103 | return WritableMetadata.list(); 104 | } 105 | 106 | @Override 107 | public void applyWritableMetadata(List metadataKeys, DataType consumedDataType) { 108 | this.metadataKeys = metadataKeys; 109 | } 110 | 111 | @Override 112 | public DynamicTableSink copy() { 113 | return new HBaseDynamicTableSink( 114 | tableName, physicalDataType, hbaseConf, writeOptions, nullStringLiteral); 115 | } 116 | 117 | @Override 118 | public String asSummaryString() { 119 | return "HBase"; 120 | } 121 | 122 | // ------------------------------------------------------------------------------------------- 123 | 124 | @VisibleForTesting 125 | public HBaseTableSchema getHBaseTableSchema() { 126 | return this.hbaseTableSchema; 127 | } 128 | 129 | @VisibleForTesting 130 | public HBaseWriteOptions getWriteOptions() { 131 | return writeOptions; 132 | } 133 | 134 | @VisibleForTesting 135 | public Configuration getConfiguration() { 136 | return this.hbaseConf; 137 | } 138 | 139 | @VisibleForTesting 140 | public String getTableName() { 141 | return this.tableName; 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /flink-connector-hbase-2.2/src/main/java/org/apache/flink/connector/hbase2/source/HBaseDynamicTableSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase2.source; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | import org.apache.flink.api.common.io.InputFormat; 23 | import org.apache.flink.connector.hbase.source.AbstractHBaseDynamicTableSource; 24 | import org.apache.flink.connector.hbase.source.HBaseRowDataLookupFunction; 25 | import org.apache.flink.connector.hbase.util.HBaseTableSchema; 26 | import org.apache.flink.table.connector.source.DynamicTableSource; 27 | import org.apache.flink.table.connector.source.lookup.AsyncLookupFunctionProvider; 28 | import org.apache.flink.table.connector.source.lookup.LookupFunctionProvider; 29 | import org.apache.flink.table.connector.source.lookup.PartialCachingAsyncLookupProvider; 30 | import org.apache.flink.table.connector.source.lookup.PartialCachingLookupProvider; 31 | import org.apache.flink.table.connector.source.lookup.cache.LookupCache; 32 | import org.apache.flink.table.data.RowData; 33 | import org.apache.flink.table.types.DataType; 34 | 35 | import org.apache.hadoop.conf.Configuration; 36 | 37 | import javax.annotation.Nullable; 38 | 39 | import java.util.Objects; 40 | 41 | import static org.apache.flink.util.Preconditions.checkArgument; 42 | 43 | /** HBase table source implementation. */ 44 | @Internal 45 | public class HBaseDynamicTableSource extends AbstractHBaseDynamicTableSource { 46 | 47 | private final boolean lookupAsync; 48 | 49 | public HBaseDynamicTableSource( 50 | Configuration conf, 51 | String tableName, 52 | HBaseTableSchema hbaseSchema, 53 | String nullStringLiteral, 54 | int maxRetryTimes, 55 | boolean lookupAsync, 56 | @Nullable LookupCache cache) { 57 | super(conf, tableName, hbaseSchema, nullStringLiteral, maxRetryTimes, cache); 58 | this.lookupAsync = lookupAsync; 59 | } 60 | 61 | @Override 62 | public LookupRuntimeProvider getLookupRuntimeProvider(LookupContext context) { 63 | checkArgument( 64 | context.getKeys().length == 1 && context.getKeys()[0].length == 1, 65 | "Currently, HBase table can only be lookup by single rowkey."); 66 | checkArgument( 67 | hbaseSchema.getRowKeyName().isPresent(), 68 | "HBase schema must have a row key when used in lookup mode."); 69 | checkArgument( 70 | DataType.getFieldNames(hbaseSchema.convertToDataType()) 71 | .get(context.getKeys()[0][0]) 72 | .equals(hbaseSchema.getRowKeyName().get()), 73 | "Currently, HBase table only supports lookup by rowkey field."); 74 | if (lookupAsync) { 75 | HBaseRowDataAsyncLookupFunction asyncLookupFunction = 76 | new HBaseRowDataAsyncLookupFunction( 77 | conf, tableName, hbaseSchema, nullStringLiteral, maxRetryTimes); 78 | if (cache != null) { 79 | return PartialCachingAsyncLookupProvider.of(asyncLookupFunction, cache); 80 | } else { 81 | return AsyncLookupFunctionProvider.of(asyncLookupFunction); 82 | } 83 | } else { 84 | HBaseRowDataLookupFunction lookupFunction = 85 | new HBaseRowDataLookupFunction( 86 | conf, tableName, hbaseSchema, nullStringLiteral, maxRetryTimes); 87 | if (cache != null) { 88 | return PartialCachingLookupProvider.of(lookupFunction, cache); 89 | } else { 90 | return LookupFunctionProvider.of(lookupFunction); 91 | } 92 | } 93 | } 94 | 95 | @Override 96 | public DynamicTableSource copy() { 97 | return new HBaseDynamicTableSource( 98 | conf, tableName, hbaseSchema, nullStringLiteral, maxRetryTimes, lookupAsync, cache); 99 | } 100 | 101 | @Override 102 | protected InputFormat getInputFormat() { 103 | return new HBaseRowDataInputFormat(conf, tableName, hbaseSchema, nullStringLiteral); 104 | } 105 | 106 | @Override 107 | public boolean equals(Object o) { 108 | if (!(o instanceof HBaseDynamicTableSource)) { 109 | return false; 110 | } 111 | HBaseDynamicTableSource that = (HBaseDynamicTableSource) o; 112 | return Objects.equals(conf, that.conf) 113 | && Objects.equals(tableName, that.tableName) 114 | && Objects.equals(hbaseSchema, that.hbaseSchema) 115 | && Objects.equals(nullStringLiteral, that.nullStringLiteral) 116 | && Objects.equals(maxRetryTimes, that.maxRetryTimes) 117 | && Objects.equals(cache, that.cache) 118 | && Objects.equals(lookupAsync, that.lookupAsync); 119 | } 120 | 121 | @Override 122 | public int hashCode() { 123 | return Objects.hash( 124 | conf, tableName, hbaseSchema, nullStringLiteral, maxRetryTimes, cache, lookupAsync); 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /flink-connector-hbase-2.2/src/main/java/org/apache/flink/connector/hbase2/source/HBaseRowDataInputFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase2.source; 20 | 21 | import org.apache.flink.api.common.io.InputFormat; 22 | import org.apache.flink.connector.hbase.util.HBaseSerde; 23 | import org.apache.flink.connector.hbase.util.HBaseTableSchema; 24 | import org.apache.flink.table.data.RowData; 25 | 26 | import org.apache.hadoop.hbase.TableName; 27 | import org.apache.hadoop.hbase.TableNotFoundException; 28 | import org.apache.hadoop.hbase.client.ConnectionFactory; 29 | import org.apache.hadoop.hbase.client.Result; 30 | import org.apache.hadoop.hbase.client.Scan; 31 | import org.slf4j.Logger; 32 | import org.slf4j.LoggerFactory; 33 | 34 | import java.io.IOException; 35 | 36 | /** 37 | * {@link InputFormat} subclass that wraps the access for HTables. Returns the result as {@link 38 | * RowData} 39 | */ 40 | public class HBaseRowDataInputFormat extends AbstractTableInputFormat { 41 | private static final long serialVersionUID = 1L; 42 | private static final Logger LOG = LoggerFactory.getLogger(HBaseRowDataInputFormat.class); 43 | 44 | private final String tableName; 45 | private final HBaseTableSchema schema; 46 | private final String nullStringLiteral; 47 | 48 | private transient HBaseSerde serde; 49 | 50 | public HBaseRowDataInputFormat( 51 | org.apache.hadoop.conf.Configuration conf, 52 | String tableName, 53 | HBaseTableSchema schema, 54 | String nullStringLiteral) { 55 | super(conf); 56 | this.tableName = tableName; 57 | this.schema = schema; 58 | this.nullStringLiteral = nullStringLiteral; 59 | } 60 | 61 | @Override 62 | protected void initTable() throws IOException { 63 | this.serde = new HBaseSerde(schema, nullStringLiteral); 64 | if (table == null) { 65 | connectToTable(); 66 | } 67 | if (table != null && scan == null) { 68 | scan = getScanner(); 69 | } 70 | } 71 | 72 | @Override 73 | protected Scan getScanner() { 74 | return serde.createScan(); 75 | } 76 | 77 | @Override 78 | public String getTableName() { 79 | return tableName; 80 | } 81 | 82 | @Override 83 | protected RowData mapResultToOutType(Result res) { 84 | return serde.convertToReusedRow(res); 85 | } 86 | 87 | private void connectToTable() throws IOException { 88 | if (connection == null) { 89 | connection = ConnectionFactory.createConnection(getHadoopConfiguration()); 90 | } 91 | TableName name = TableName.valueOf(tableName); 92 | if (!connection.getAdmin().tableExists(name)) { 93 | throw new TableNotFoundException("HBase table '" + tableName + "' not found."); 94 | } 95 | table = connection.getTable(name); 96 | regionLocator = connection.getRegionLocator(name); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /flink-connector-hbase-2.2/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | org.apache.flink.connector.hbase2.HBase2DynamicTableFactory 17 | -------------------------------------------------------------------------------- /flink-connector-hbase-2.2/src/test/java/org/apache/flink/architecture/TestCodeArchitectureTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.architecture; 20 | 21 | import org.apache.flink.architecture.common.ImportOptions; 22 | 23 | import com.tngtech.archunit.core.importer.ImportOption; 24 | import com.tngtech.archunit.junit.AnalyzeClasses; 25 | import com.tngtech.archunit.junit.ArchTest; 26 | import com.tngtech.archunit.junit.ArchTests; 27 | 28 | /** Architecture tests for test code. */ 29 | @AnalyzeClasses( 30 | packages = "org.apache.flink.connector.hbase2", 31 | importOptions = { 32 | ImportOption.OnlyIncludeTests.class, 33 | ImportOptions.ExcludeScalaImportOption.class, 34 | ImportOptions.ExcludeShadedImportOption.class 35 | }) 36 | public class TestCodeArchitectureTest { 37 | 38 | @ArchTest 39 | public static final ArchTests COMMON_TESTS = ArchTests.in(TestCodeArchitectureTestBase.class); 40 | } 41 | -------------------------------------------------------------------------------- /flink-connector-hbase-2.2/src/test/java/org/apache/flink/connector/hbase2/HBaseTablePlanTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase2; 20 | 21 | import org.apache.flink.table.api.TableConfig; 22 | import org.apache.flink.table.planner.utils.StreamTableTestUtil; 23 | import org.apache.flink.table.planner.utils.TableTestBase; 24 | 25 | import org.junit.jupiter.api.BeforeEach; 26 | import org.junit.jupiter.api.Test; 27 | import org.junit.jupiter.api.TestInfo; 28 | import org.junit.rules.TestName; 29 | 30 | import static org.assertj.core.api.Assertions.assertThatThrownBy; 31 | 32 | /** Plan tests for HBase connector, for example, testing projection push down. */ 33 | public class HBaseTablePlanTest extends TableTestBase { 34 | 35 | private final StreamTableTestUtil util = streamTestUtil(TableConfig.getDefault()); 36 | 37 | private TestInfo testInfo; 38 | 39 | @BeforeEach 40 | public void setup(TestInfo testInfo) { 41 | this.testInfo = testInfo; 42 | } 43 | 44 | // A workaround to get the test method name for Flink versions not completely migrated to JUnit5 45 | public TestName name() { 46 | return new TestName() { 47 | @Override 48 | public String getMethodName() { 49 | return testInfo.getTestMethod().get().getName(); 50 | } 51 | }; 52 | } 53 | 54 | @Test 55 | public void testMultipleRowKey() { 56 | util.tableEnv() 57 | .executeSql( 58 | "CREATE TABLE hTable (" 59 | + " family1 ROW," 60 | + " family2 ROW," 61 | + " rowkey INT," 62 | + " rowkey2 STRING " 63 | + ") WITH (" 64 | + " 'connector' = 'hbase-2.2'," 65 | + " 'table-name' = 'my_table'," 66 | + " 'zookeeper.quorum' = 'localhost:2021'" 67 | + ")"); 68 | 69 | assertThatThrownBy(() -> util.verifyExecPlan("SELECT * FROM hTable")) 70 | .hasRootCauseInstanceOf(IllegalArgumentException.class) 71 | .hasRootCauseMessage("Row key can't be set multiple times."); 72 | } 73 | 74 | @Test 75 | public void testNoneRowKey() { 76 | util.tableEnv() 77 | .executeSql( 78 | "CREATE TABLE hTable (" 79 | + " family1 ROW," 80 | + " family2 ROW" 81 | + ") WITH (" 82 | + " 'connector' = 'hbase-2.2'," 83 | + " 'table-name' = 'my_table'," 84 | + " 'zookeeper.quorum' = 'localhost:2021'" 85 | + ")"); 86 | 87 | assertThatThrownBy(() -> util.verifyExecPlan("SELECT * FROM hTable")) 88 | .hasRootCauseInstanceOf(IllegalArgumentException.class) 89 | .hasRootCauseMessage( 90 | "HBase table requires to define a row key field. " 91 | + "A row key field is defined as an atomic type, " 92 | + "column families and qualifiers are defined as ROW type."); 93 | } 94 | 95 | @Test 96 | public void testInvalidPrimaryKey() { 97 | util.tableEnv() 98 | .executeSql( 99 | "CREATE TABLE hTable (" 100 | + " family1 ROW," 101 | + " family2 ROW," 102 | + " rowkey STRING, " 103 | + " PRIMARY KEY (family1) NOT ENFORCED " 104 | + ") WITH (" 105 | + " 'connector' = 'hbase-2.2'," 106 | + " 'table-name' = 'my_table'," 107 | + " 'zookeeper.quorum' = 'localhost:2021'" 108 | + ")"); 109 | 110 | assertThatThrownBy(() -> util.verifyExecPlan("SELECT * FROM hTable")) 111 | .hasRootCauseInstanceOf(IllegalArgumentException.class) 112 | .hasRootCauseMessage( 113 | "Primary key of HBase table must be defined on the row key field. " 114 | + "A row key field is defined as an atomic type, " 115 | + "column families and qualifiers are defined as ROW type."); 116 | } 117 | 118 | @Test 119 | public void testUnsupportedDataType() { 120 | util.tableEnv() 121 | .executeSql( 122 | "CREATE TABLE hTable (" 123 | + " family1 ROW," 124 | + " family2 ROW," 125 | + " col1 ARRAY, " 126 | + " rowkey STRING, " 127 | + " PRIMARY KEY (rowkey) NOT ENFORCED " 128 | + ") WITH (" 129 | + " 'connector' = 'hbase-2.2'," 130 | + " 'table-name' = 'my_table'," 131 | + " 'zookeeper.quorum' = 'localhost:2021'" 132 | + ")"); 133 | 134 | assertThatThrownBy(() -> util.verifyExecPlan("SELECT * FROM hTable")) 135 | .hasRootCauseInstanceOf(IllegalArgumentException.class) 136 | .hasRootCauseMessage("Unsupported field type 'ARRAY' for HBase."); 137 | } 138 | 139 | @Test 140 | public void testProjectionPushDown() { 141 | util.tableEnv() 142 | .executeSql( 143 | "CREATE TABLE hTable (" 144 | + " family1 ROW," 145 | + " family2 ROW," 146 | + " family3 ROW," 147 | + " rowkey INT," 148 | + " PRIMARY KEY (rowkey) NOT ENFORCED" 149 | + ") WITH (" 150 | + " 'connector' = 'hbase-2.2'," 151 | + " 'table-name' = 'my_table'," 152 | + " 'zookeeper.quorum' = 'localhost:2021'" 153 | + ")"); 154 | util.verifyExecPlan("SELECT h.family3, h.family2.col2 FROM hTable AS h"); 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /flink-connector-hbase-2.2/src/test/java/org/apache/flink/connector/hbase2/source/HBaseRowDataAsyncLookupFunctionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase2.source; 20 | 21 | import org.apache.flink.connector.hbase.util.HBaseTableSchema; 22 | import org.apache.flink.connector.hbase2.util.HBaseTestBase; 23 | import org.apache.flink.table.api.DataTypes; 24 | import org.apache.flink.table.connector.source.lookup.LookupOptions; 25 | import org.apache.flink.table.data.RowData; 26 | import org.apache.flink.table.types.DataType; 27 | 28 | import org.junit.jupiter.api.Test; 29 | 30 | import java.util.ArrayList; 31 | import java.util.Collection; 32 | import java.util.Collections; 33 | import java.util.List; 34 | import java.util.concurrent.CompletableFuture; 35 | import java.util.concurrent.CountDownLatch; 36 | 37 | import static org.apache.flink.table.api.DataTypes.BIGINT; 38 | import static org.apache.flink.table.api.DataTypes.DOUBLE; 39 | import static org.apache.flink.table.api.DataTypes.FIELD; 40 | import static org.apache.flink.table.api.DataTypes.INT; 41 | import static org.apache.flink.table.api.DataTypes.ROW; 42 | import static org.apache.flink.table.api.DataTypes.STRING; 43 | import static org.assertj.core.api.Assertions.assertThat; 44 | 45 | /** Test suite for {@link HBaseRowDataAsyncLookupFunction}. */ 46 | class HBaseRowDataAsyncLookupFunctionTest extends HBaseTestBase { 47 | @Test 48 | void testEval() throws Exception { 49 | HBaseRowDataAsyncLookupFunction lookupFunction = buildRowDataAsyncLookupFunction(); 50 | 51 | lookupFunction.open(null); 52 | final List result = new ArrayList<>(); 53 | int[] rowkeys = {1, 2, 1, 12, 3, 12, 4, 3}; 54 | CountDownLatch latch = new CountDownLatch(rowkeys.length); 55 | for (int rowkey : rowkeys) { 56 | CompletableFuture> future = new CompletableFuture<>(); 57 | lookupFunction.eval(future, rowkey); 58 | future.whenComplete( 59 | (rs, t) -> { 60 | synchronized (result) { 61 | if (rs.isEmpty()) { 62 | result.add(rowkey + ": null"); 63 | } else { 64 | rs.forEach(row -> result.add(rowkey + ": " + row.toString())); 65 | } 66 | } 67 | latch.countDown(); 68 | }); 69 | } 70 | // this verifies lookup calls are async 71 | assertThat(result.size()).isLessThan(rowkeys.length); 72 | latch.await(); 73 | lookupFunction.close(); 74 | Collections.sort(result); 75 | 76 | assertThat(result) 77 | .containsExactly( 78 | "12: null", 79 | "12: null", 80 | "1: +I(1,+I(10),+I(Hello-1,100),+I(1.01,false,Welt-1))", 81 | "1: +I(1,+I(10),+I(Hello-1,100),+I(1.01,false,Welt-1))", 82 | "2: +I(2,+I(20),+I(Hello-2,200),+I(2.02,true,Welt-2))", 83 | "3: +I(3,+I(30),+I(Hello-3,300),+I(3.03,false,Welt-3))", 84 | "3: +I(3,+I(30),+I(Hello-3,300),+I(3.03,false,Welt-3))", 85 | "4: +I(4,+I(40),+I(null,400),+I(4.04,true,Welt-4))"); 86 | } 87 | 88 | private HBaseRowDataAsyncLookupFunction buildRowDataAsyncLookupFunction() { 89 | DataType dataType = 90 | ROW( 91 | FIELD(ROW_KEY, INT()), 92 | FIELD(FAMILY1, ROW(FIELD(F1COL1, INT()))), 93 | FIELD(FAMILY2, ROW(FIELD(F2COL1, STRING()), FIELD(F2COL2, BIGINT()))), 94 | FIELD( 95 | FAMILY3, 96 | ROW( 97 | FIELD(F3COL1, DOUBLE()), 98 | FIELD(F3COL2, DataTypes.BOOLEAN()), 99 | FIELD(F3COL3, STRING())))); 100 | HBaseTableSchema hbaseSchema = HBaseTableSchema.fromDataType(dataType); 101 | return new HBaseRowDataAsyncLookupFunction( 102 | getConf(), 103 | TEST_TABLE_1, 104 | hbaseSchema, 105 | "null", 106 | LookupOptions.MAX_RETRIES.defaultValue()); 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /flink-connector-hbase-2.2/src/test/java/org/apache/flink/connector/hbase2/util/HBaseTestingClusterAutoStarter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright The Apache Software Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | package org.apache.flink.connector.hbase2.util; 22 | 23 | import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; 24 | import org.apache.flink.test.junit5.MiniClusterExtension; 25 | 26 | import org.apache.commons.lang3.Range; 27 | import org.apache.commons.logging.Log; 28 | import org.apache.commons.logging.LogFactory; 29 | import org.apache.hadoop.conf.Configuration; 30 | import org.apache.hadoop.hbase.HBaseConfiguration; 31 | import org.apache.hadoop.hbase.HBaseTestingUtility; 32 | import org.apache.hadoop.hbase.HColumnDescriptor; 33 | import org.apache.hadoop.hbase.HConstants; 34 | import org.apache.hadoop.hbase.HTableDescriptor; 35 | import org.apache.hadoop.hbase.MasterNotRunningException; 36 | import org.apache.hadoop.hbase.TableName; 37 | import org.apache.hadoop.hbase.ZooKeeperConnectionException; 38 | import org.apache.hadoop.hbase.client.Admin; 39 | import org.apache.hadoop.hbase.client.Table; 40 | import org.apache.hadoop.util.VersionUtil; 41 | import org.junit.jupiter.api.AfterAll; 42 | import org.junit.jupiter.api.BeforeAll; 43 | import org.junit.jupiter.api.extension.RegisterExtension; 44 | 45 | import java.io.IOException; 46 | import java.util.ArrayList; 47 | import java.util.List; 48 | 49 | import static org.assertj.core.api.Assertions.assertThat; 50 | import static org.assertj.core.api.Assertions.fail; 51 | import static org.assertj.core.api.Assumptions.assumeThat; 52 | 53 | /** 54 | * By using this class as the super class of a set of tests you will have a HBase testing cluster 55 | * available that is very suitable for writing tests for scanning and filtering against. 56 | */ 57 | public class HBaseTestingClusterAutoStarter { 58 | private static final Log LOG = LogFactory.getLog(HBaseTestingClusterAutoStarter.class); 59 | 60 | @RegisterExtension 61 | private static final MiniClusterExtension MINI_CLUSTER_EXTENSION = 62 | new MiniClusterExtension( 63 | new MiniClusterResourceConfiguration.Builder() 64 | .setNumberTaskManagers(1) 65 | .setNumberSlotsPerTaskManager(4) 66 | .build()); 67 | 68 | private static final Range HADOOP_VERSION_RANGE = 69 | Range.between("2.8.0", "3.0.3", VersionUtil::compareVersions); 70 | 71 | private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); 72 | private static Admin admin = null; 73 | private static List createdTables = new ArrayList<>(); 74 | 75 | private static Configuration conf; 76 | 77 | protected static void createTable( 78 | TableName tableName, byte[][] columnFamilyName, byte[][] splitKeys) { 79 | assertThat(admin).as("HBaseAdmin is not initialized successfully.").isNotNull(); 80 | HTableDescriptor desc = new HTableDescriptor(tableName); 81 | for (byte[] fam : columnFamilyName) { 82 | HColumnDescriptor colDef = new HColumnDescriptor(fam); 83 | desc.addFamily(colDef); 84 | } 85 | 86 | try { 87 | admin.createTable(desc, splitKeys); 88 | createdTables.add(tableName); 89 | assertThat(admin.tableExists(tableName)).as("Fail to create the table").isTrue(); 90 | } catch (IOException e) { 91 | fail("Exception found while creating table", e); 92 | } 93 | } 94 | 95 | protected static Table openTable(TableName tableName) throws IOException { 96 | Table table = TEST_UTIL.getConnection().getTable(tableName); 97 | assertThat(admin.tableExists(tableName)).as("Fail to create the table").isTrue(); 98 | return table; 99 | } 100 | 101 | private static void deleteTables() { 102 | if (admin != null) { 103 | for (TableName tableName : createdTables) { 104 | try { 105 | if (admin.tableExists(tableName)) { 106 | admin.disableTable(tableName); 107 | admin.deleteTable(tableName); 108 | } 109 | } catch (IOException e) { 110 | fail("Exception found deleting the table", e); 111 | } 112 | } 113 | } 114 | } 115 | 116 | public static Configuration getConf() { 117 | return conf; 118 | } 119 | 120 | public static String getZookeeperQuorum() { 121 | return "localhost:" + TEST_UTIL.getZkCluster().getClientPort(); 122 | } 123 | 124 | private static void initialize(Configuration c) { 125 | conf = HBaseConfiguration.create(c); 126 | // the default retry number is 15 in hbase-2.2, set 15 for test 127 | conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 15); 128 | try { 129 | admin = TEST_UTIL.getAdmin(); 130 | } catch (MasterNotRunningException e) { 131 | fail("Master is not running", e); 132 | } catch (ZooKeeperConnectionException e) { 133 | fail("Cannot connect to ZooKeeper", e); 134 | } catch (IOException e) { 135 | fail("IOException", e); 136 | } 137 | } 138 | 139 | @BeforeAll 140 | public static void setUp() throws Exception { 141 | // HBase 2.2.3 HBaseTestingUtility works with only a certain range of hadoop versions 142 | String hadoopVersion = System.getProperty("hadoop.version", "2.8.5"); 143 | assumeThat(HADOOP_VERSION_RANGE.contains(hadoopVersion)).isTrue(); 144 | TEST_UTIL.startMiniCluster(1); 145 | 146 | // https://issues.apache.org/jira/browse/HBASE-11711 147 | TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", -1); 148 | 149 | // Make sure the zookeeper quorum value contains the right port number (varies per run). 150 | LOG.info("Hbase minicluster client port: " + TEST_UTIL.getZkCluster().getClientPort()); 151 | TEST_UTIL 152 | .getConfiguration() 153 | .set( 154 | "hbase.zookeeper.quorum", 155 | "localhost:" + TEST_UTIL.getZkCluster().getClientPort()); 156 | 157 | initialize(TEST_UTIL.getConfiguration()); 158 | } 159 | 160 | @AfterAll 161 | public static void tearDown() throws Exception { 162 | if (conf == null) { 163 | LOG.info("Skipping Hbase tear down. It was never started"); 164 | return; 165 | } 166 | LOG.info("HBase minicluster: Shutting down"); 167 | deleteTables(); 168 | TEST_UTIL.shutdownMiniCluster(); 169 | LOG.info("HBase minicluster: Down"); 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /flink-connector-hbase-2.2/src/test/java/org/eclipse/jetty/util/JavaVersion.java: -------------------------------------------------------------------------------- 1 | // 2 | // ======================================================================== 3 | // Copyright (c) 1995-2018 Mort Bay Consulting Pty. Ltd. 4 | // ------------------------------------------------------------------------ 5 | // All rights reserved. This program and the accompanying materials 6 | // are made available under the terms of the Eclipse Public License v1.0 7 | // and Apache License v2.0 which accompanies this distribution. 8 | // 9 | // The Eclipse Public License is available at 10 | // http://www.eclipse.org/legal/epl-v10.html 11 | // 12 | // The Apache License v2.0 is available at 13 | // http://www.opensource.org/licenses/apache2.0.php 14 | // 15 | // You may elect to redistribute this code under either of these licenses. 16 | // ======================================================================== 17 | // 18 | 19 | package org.eclipse.jetty.util; 20 | 21 | import java.util.regex.Matcher; 22 | import java.util.regex.Pattern; 23 | 24 | /** 25 | * Java Version Utility class. 26 | * 27 | *

Parses java versions to extract a consistent set of version parts 28 | */ 29 | public class JavaVersion { 30 | /** 31 | * Context attribute that can be set to target a different version of the jvm than the current 32 | * runtime. Acceptable values should correspond to those returned by JavaVersion.getPlatform(). 33 | */ 34 | public static final String JAVA_TARGET_PLATFORM = "org.eclipse.jetty.javaTargetPlatform"; 35 | 36 | /** Regex for Java version numbers. */ 37 | private static final String VSTR_FORMAT = "(?[1-9][0-9]*(?:(?:\\.0)*\\.[0-9]+)*).*"; 38 | 39 | static final Pattern VSTR_PATTERN = Pattern.compile(VSTR_FORMAT); 40 | 41 | public static final JavaVersion VERSION = 42 | parse( 43 | System.getProperty( 44 | "java.runtime.version", System.getProperty("java.version", "1.8"))); 45 | 46 | public static JavaVersion parse(String v) { 47 | Matcher m = VSTR_PATTERN.matcher(v); 48 | if (!m.matches() || m.group("VNUM") == null) { 49 | System.err.println("ERROR: Invalid version string: '" + v + "'"); 50 | return new JavaVersion(v + "-UNKNOWN", 8, 1, 8, 0); 51 | } 52 | 53 | // $VNUM is a dot-separated list of integers of arbitrary length 54 | String[] split = m.group("VNUM").split("\\."); 55 | int[] version = new int[split.length]; 56 | for (int i = 0; i < split.length; i++) { 57 | version[i] = Integer.parseInt(split[i]); 58 | } 59 | 60 | return new JavaVersion( 61 | v, 62 | (version[0] >= 9 || version.length == 1) ? version[0] : version[1], 63 | version[0], 64 | version.length > 1 ? version[1] : 0, 65 | version.length > 2 ? version[2] : 0); 66 | } 67 | 68 | private final String version; 69 | private final int platform; 70 | private final int major; 71 | private final int minor; 72 | private final int micro; 73 | 74 | private JavaVersion(String version, int platform, int major, int minor, int micro) { 75 | this.version = version; 76 | this.platform = platform; 77 | this.major = major; 78 | this.minor = minor; 79 | this.micro = micro; 80 | } 81 | 82 | /** @return the string from which this JavaVersion was created */ 83 | public String getVersion() { 84 | return version; 85 | } 86 | 87 | /** 88 | * Returns the Java Platform version, such as {@code 8} for JDK 1.8.0_92 and {@code 9} for JDK 89 | * 9.2.4. 90 | * 91 | * @return the Java Platform version 92 | */ 93 | public int getPlatform() { 94 | return platform; 95 | } 96 | 97 | /** 98 | * Returns the major number version, such as {@code 1} for JDK 1.8.0_92 and {@code 9} for JDK 99 | * 9.2.4. 100 | * 101 | * @return the major number version 102 | */ 103 | public int getMajor() { 104 | return major; 105 | } 106 | 107 | /** 108 | * Returns the minor number version, such as {@code 8} for JDK 1.8.0_92 and {@code 2} for JDK 109 | * 9.2.4. 110 | * 111 | * @return the minor number version 112 | */ 113 | public int getMinor() { 114 | return minor; 115 | } 116 | 117 | /** 118 | * Returns the micro number version (aka security number), such as {@code 0} for JDK 1.8.0_92 119 | * and {@code 4} for JDK 9.2.4. 120 | * 121 | * @return the micro number version 122 | */ 123 | public int getMicro() { 124 | return micro; 125 | } 126 | 127 | /** 128 | * Returns the update number version, such as {@code 92} for JDK 1.8.0_92 and {@code 0} for JDK 129 | * 9.2.4. 130 | * 131 | * @return the update number version 132 | */ 133 | @Deprecated 134 | public int getUpdate() { 135 | return 0; 136 | } 137 | 138 | /** 139 | * Returns the remaining string after the version numbers, such as {@code -internal} for JDK 140 | * 1.8.0_92-internal and {@code -ea} for JDK 9-ea, or {@code +13} for JDK 9.2.4+13. 141 | * 142 | * @return the remaining string after the version numbers 143 | */ 144 | @Deprecated 145 | public String getSuffix() { 146 | return null; 147 | } 148 | 149 | @Override 150 | public String toString() { 151 | return version; 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /flink-connector-hbase-2.2/src/test/java/org/slf4j/impl/Log4jLoggerAdapter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.slf4j.impl; 20 | 21 | /** Fake appender to work around HBase referring to it directly. */ 22 | public interface Log4jLoggerAdapter {} 23 | -------------------------------------------------------------------------------- /flink-connector-hbase-2.2/src/test/resources/archunit.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | # By default we allow removing existing violations, but fail when new violations are added. 20 | freeze.store.default.allowStoreUpdate=true 21 | 22 | # Enable this if a new (frozen) rule has been added in order to create the initial store and record the existing violations. 23 | #freeze.store.default.allowStoreCreation=true 24 | 25 | # Enable this to add allow new violations to be recorded. 26 | # NOTE: Adding new violations should be avoided when possible. If the rule was correct to flag a new 27 | # violation, please try to avoid creating the violation. If the violation was created due to a 28 | # shortcoming of the rule, file a JIRA issue so the rule can be improved. 29 | #freeze.refreeze=true 30 | 31 | freeze.store.default.path=archunit-violations 32 | -------------------------------------------------------------------------------- /flink-connector-hbase-2.2/src/test/resources/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 20 | 21 | 23 | 24 | 25 | 26 | hbase_conf_key 27 | hbase_conf_value! 28 | 29 | 30 | -------------------------------------------------------------------------------- /flink-connector-hbase-2.2/src/test/resources/log4j2-test.properties: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | ################################################################################ 18 | 19 | # Set root logger level to OFF to not flood build logs 20 | # set manually to INFO for debugging purposes 21 | rootLogger.level = OFF 22 | rootLogger.appenderRef.test.ref = TestLogger 23 | 24 | appender.testlogger.name = TestLogger 25 | appender.testlogger.type = CONSOLE 26 | appender.testlogger.target = SYSTEM_ERR 27 | appender.testlogger.layout.type = PatternLayout 28 | appender.testlogger.layout.pattern = %-4r [%t] %-5p %c %x - %m%n 29 | -------------------------------------------------------------------------------- /flink-connector-hbase-2.2/src/test/resources/org/apache/flink/connector/hbase2/HBaseTablePlanTest.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 28 | 29 | 30 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/archunit-violations/f8f77b71-7087-4b3f-88c5-29588fadcc52: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/flink-connector-hbase/ca5a4aae3e953841371e9a568ad8472a5ef8c1d2/flink-connector-hbase-base/archunit-violations/f8f77b71-7087-4b3f-88c5-29588fadcc52 -------------------------------------------------------------------------------- /flink-connector-hbase-base/archunit-violations/fd028eab-0e49-4e91-9c40-477a55f378f2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/flink-connector-hbase/ca5a4aae3e953841371e9a568ad8472a5ef8c1d2/flink-connector-hbase-base/archunit-violations/fd028eab-0e49-4e91-9c40-477a55f378f2 -------------------------------------------------------------------------------- /flink-connector-hbase-base/archunit-violations/stored.rules: -------------------------------------------------------------------------------- 1 | # 2 | #Tue Feb 22 12:17:29 CET 2022 3 | Tests\ inheriting\ from\ AbstractTestBase\ should\ have\ name\ ending\ with\ ITCase=f8f77b71-7087-4b3f-88c5-29588fadcc52 4 | ITCASE\ tests\ should\ use\ a\ MiniCluster\ resource\ or\ extension=fd028eab-0e49-4e91-9c40-477a55f378f2 5 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 20 | 22 | 23 | 4.0.0 24 | 25 | 26 | org.apache.flink 27 | flink-connector-hbase-parent 28 | 4.0-SNAPSHOT 29 | 30 | 31 | flink-connector-hbase-base 32 | Flink : Connectors : HBase Base 33 | jar 34 | 35 | 36 | 37 | 38 | org.apache.flink 39 | flink-core 40 | provided 41 | 42 | 43 | 44 | org.apache.flink 45 | flink-streaming-java 46 | provided 47 | 48 | 49 | 50 | 51 | 52 | 53 | org.apache.flink 54 | flink-table-api-java-bridge 55 | provided 56 | true 57 | 58 | 59 | 60 | 61 | org.apache.hadoop 62 | hadoop-common 63 | provided 64 | 65 | 66 | com.google.guava 67 | guava 68 | 69 | 70 | log4j 71 | log4j 72 | 73 | 74 | org.slf4j 75 | slf4j-log4j12 76 | 77 | 78 | net.minidev 79 | json-smart 80 | 81 | 82 | 83 | 84 | 85 | org.apache.hbase 86 | hbase-client 87 | ${hbase2.version} 88 | 89 | 90 | 91 | org.mortbay.jetty 92 | jetty-util 93 | 94 | 95 | org.mortbay.jetty 96 | jetty 97 | 98 | 99 | org.mortbay.jetty 100 | jetty-sslengine 101 | 102 | 103 | org.mortbay.jetty 104 | jsp-2.1 105 | 106 | 107 | org.mortbay.jetty 108 | jsp-api-2.1 109 | 110 | 111 | org.mortbay.jetty 112 | servlet-api-2.5 113 | 114 | 115 | 116 | org.apache.hbase 117 | hbase-annotations 118 | 119 | 120 | com.sun.jersey 121 | jersey-core 122 | 123 | 124 | org.apache.hadoop 125 | hadoop-common 126 | 127 | 128 | org.apache.hadoop 129 | hadoop-auth 130 | 131 | 132 | org.apache.hadoop 133 | hadoop-annotations 134 | 135 | 136 | org.apache.hadoop 137 | hadoop-mapreduce-client-core 138 | 139 | 140 | org.apache.hadoop 141 | hadoop-client 142 | 143 | 144 | org.apache.hadoop 145 | hadoop-hdfs 146 | 147 | 148 | log4j 149 | log4j 150 | 151 | 152 | org.slf4j 153 | slf4j-log4j12 154 | 155 | 156 | 157 | 158 | 159 | 160 | org.apache.logging.log4j 161 | log4j-1.2-api 162 | provided 163 | 164 | 165 | 166 | 167 | com.google.guava 168 | guava 169 | 32.0.0-jre 170 | test 171 | 172 | 173 | 174 | org.apache.flink 175 | flink-architecture-tests-test 176 | test 177 | 178 | 179 | 180 | 181 | 182 | 183 | org.apache.maven.plugins 184 | maven-jar-plugin 185 | 186 | 187 | 188 | test-jar 189 | 190 | 191 | 192 | 193 | META-INF/services/org.junit.jupiter.api.extension.Extension 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | java11 205 | 206 | [11,) 207 | 208 | 209 | 210 | 211 | 212 | org.apache.maven.plugins 213 | maven-surefire-plugin 214 | 215 | 216 | true 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/src/main/java/org/apache/flink/connector/hbase/options/HBaseWriteOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase.options; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | 23 | import org.apache.hadoop.hbase.client.ConnectionConfiguration; 24 | 25 | import java.io.Serializable; 26 | import java.util.Objects; 27 | 28 | /** Options for HBase writing. */ 29 | @Internal 30 | public class HBaseWriteOptions implements Serializable { 31 | 32 | private static final long serialVersionUID = 1L; 33 | 34 | private final long bufferFlushMaxSizeInBytes; 35 | private final long bufferFlushMaxRows; 36 | private final long bufferFlushIntervalMillis; 37 | private final boolean ignoreNullValue; 38 | private final Integer parallelism; 39 | 40 | private HBaseWriteOptions( 41 | long bufferFlushMaxSizeInBytes, 42 | long bufferFlushMaxMutations, 43 | long bufferFlushIntervalMillis, 44 | boolean ignoreNullValue, 45 | Integer parallelism) { 46 | this.bufferFlushMaxSizeInBytes = bufferFlushMaxSizeInBytes; 47 | this.bufferFlushMaxRows = bufferFlushMaxMutations; 48 | this.bufferFlushIntervalMillis = bufferFlushIntervalMillis; 49 | this.ignoreNullValue = ignoreNullValue; 50 | this.parallelism = parallelism; 51 | } 52 | 53 | public long getBufferFlushMaxSizeInBytes() { 54 | return bufferFlushMaxSizeInBytes; 55 | } 56 | 57 | public long getBufferFlushMaxRows() { 58 | return bufferFlushMaxRows; 59 | } 60 | 61 | public long getBufferFlushIntervalMillis() { 62 | return bufferFlushIntervalMillis; 63 | } 64 | 65 | public boolean isIgnoreNullValue() { 66 | return ignoreNullValue; 67 | } 68 | 69 | public Integer getParallelism() { 70 | return parallelism; 71 | } 72 | 73 | @Override 74 | public String toString() { 75 | return "HBaseWriteOptions{" 76 | + "bufferFlushMaxSizeInBytes=" 77 | + bufferFlushMaxSizeInBytes 78 | + ", bufferFlushMaxRows=" 79 | + bufferFlushMaxRows 80 | + ", bufferFlushIntervalMillis=" 81 | + bufferFlushIntervalMillis 82 | + ", ignoreNullValue=" 83 | + ignoreNullValue 84 | + ", parallelism=" 85 | + parallelism 86 | + '}'; 87 | } 88 | 89 | @Override 90 | public boolean equals(Object o) { 91 | if (this == o) { 92 | return true; 93 | } 94 | if (o == null || getClass() != o.getClass()) { 95 | return false; 96 | } 97 | HBaseWriteOptions that = (HBaseWriteOptions) o; 98 | return bufferFlushMaxSizeInBytes == that.bufferFlushMaxSizeInBytes 99 | && bufferFlushMaxRows == that.bufferFlushMaxRows 100 | && bufferFlushIntervalMillis == that.bufferFlushIntervalMillis 101 | && ignoreNullValue == that.ignoreNullValue 102 | && parallelism == that.parallelism; 103 | } 104 | 105 | @Override 106 | public int hashCode() { 107 | return Objects.hash( 108 | bufferFlushMaxSizeInBytes, 109 | bufferFlushMaxRows, 110 | bufferFlushIntervalMillis, 111 | parallelism); 112 | } 113 | 114 | /** Creates a builder for {@link HBaseWriteOptions}. */ 115 | public static Builder builder() { 116 | return new Builder(); 117 | } 118 | 119 | /** Builder for {@link HBaseWriteOptions}. */ 120 | public static class Builder { 121 | 122 | private long bufferFlushMaxSizeInBytes = ConnectionConfiguration.WRITE_BUFFER_SIZE_DEFAULT; 123 | private long bufferFlushMaxRows = 0; 124 | private long bufferFlushIntervalMillis = 0; 125 | private boolean ignoreNullValue; 126 | private Integer parallelism; 127 | 128 | /** 129 | * Optional. Sets when to flush a buffered request based on the memory size of rows 130 | * currently added. Default to 2mb. 131 | */ 132 | public Builder setBufferFlushMaxSizeInBytes(long bufferFlushMaxSizeInBytes) { 133 | this.bufferFlushMaxSizeInBytes = bufferFlushMaxSizeInBytes; 134 | return this; 135 | } 136 | 137 | /** 138 | * Optional. Sets when to flush buffered request based on the number of rows currently 139 | * added. Defaults to not set, i.e. won't flush based on the number of buffered rows. 140 | */ 141 | public Builder setBufferFlushMaxRows(long bufferFlushMaxRows) { 142 | this.bufferFlushMaxRows = bufferFlushMaxRows; 143 | return this; 144 | } 145 | 146 | /** 147 | * Optional. Sets a flush interval flushing buffered requesting if the interval passes, in 148 | * milliseconds. Defaults to not set, i.e. won't flush based on flush interval. 149 | */ 150 | public Builder setBufferFlushIntervalMillis(long bufferFlushIntervalMillis) { 151 | this.bufferFlushIntervalMillis = bufferFlushIntervalMillis; 152 | return this; 153 | } 154 | 155 | /** 156 | * Optional. Sets whether ignore null value or not. By defaults, null value will be writing. 157 | */ 158 | public Builder setIgnoreNullValue(boolean ignoreNullValue) { 159 | this.ignoreNullValue = ignoreNullValue; 160 | return this; 161 | } 162 | 163 | /** 164 | * Optional. Defines the parallelism of the HBase sink operator. By default, the parallelism 165 | * is determined by the framework using the same parallelism of the upstream chained 166 | * operator. 167 | */ 168 | public Builder setParallelism(Integer parallelism) { 169 | this.parallelism = parallelism; 170 | return this; 171 | } 172 | 173 | /** Creates a new instance of {@link HBaseWriteOptions}. */ 174 | public HBaseWriteOptions build() { 175 | return new HBaseWriteOptions( 176 | bufferFlushMaxSizeInBytes, 177 | bufferFlushMaxRows, 178 | bufferFlushIntervalMillis, 179 | ignoreNullValue, 180 | parallelism); 181 | } 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/src/main/java/org/apache/flink/connector/hbase/sink/HBaseMutationConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase.sink; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | 23 | import org.apache.hadoop.hbase.client.Delete; 24 | import org.apache.hadoop.hbase.client.Mutation; 25 | import org.apache.hadoop.hbase.client.Put; 26 | 27 | import java.io.Serializable; 28 | 29 | /** 30 | * A converter used to converts the input record into HBase {@link Mutation}. 31 | * 32 | * @param type of input record. 33 | */ 34 | @Internal 35 | public interface HBaseMutationConverter extends Serializable { 36 | 37 | /** Initialization method for the function. It is called once before conversion method. */ 38 | void open(); 39 | 40 | /** 41 | * Converts the input record into HBase {@link Mutation}. A mutation can be a {@link Put} or 42 | * {@link Delete}. 43 | */ 44 | Mutation convertToMutation(T record); 45 | } 46 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/src/main/java/org/apache/flink/connector/hbase/sink/RowDataToMutationConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase.sink; 20 | 21 | import org.apache.flink.connector.hbase.sink.WritableMetadata.TimeToLiveMetadata; 22 | import org.apache.flink.connector.hbase.sink.WritableMetadata.TimestampMetadata; 23 | import org.apache.flink.connector.hbase.util.HBaseSerde; 24 | import org.apache.flink.connector.hbase.util.HBaseTableSchema; 25 | import org.apache.flink.table.data.RowData; 26 | import org.apache.flink.table.types.DataType; 27 | import org.apache.flink.types.RowKind; 28 | 29 | import org.apache.hadoop.hbase.client.Mutation; 30 | 31 | import java.util.List; 32 | 33 | /** 34 | * An implementation of {@link HBaseMutationConverter} which converts {@link RowData} into {@link 35 | * Mutation}. 36 | */ 37 | public class RowDataToMutationConverter implements HBaseMutationConverter { 38 | private static final long serialVersionUID = 1L; 39 | 40 | private final HBaseTableSchema schema; 41 | private final String nullStringLiteral; 42 | private final boolean ignoreNullValue; 43 | private final TimestampMetadata timestampMetadata; 44 | private final TimeToLiveMetadata timeToLiveMetadata; 45 | private transient HBaseSerde serde; 46 | 47 | public RowDataToMutationConverter( 48 | HBaseTableSchema schema, 49 | DataType physicalDataType, 50 | List metadataKeys, 51 | String nullStringLiteral, 52 | boolean ignoreNullValue) { 53 | this.schema = schema; 54 | this.nullStringLiteral = nullStringLiteral; 55 | this.ignoreNullValue = ignoreNullValue; 56 | this.timestampMetadata = new TimestampMetadata(metadataKeys, physicalDataType); 57 | this.timeToLiveMetadata = new TimeToLiveMetadata(metadataKeys, physicalDataType); 58 | } 59 | 60 | @Override 61 | public void open() { 62 | this.serde = new HBaseSerde(schema, nullStringLiteral, ignoreNullValue); 63 | } 64 | 65 | @Override 66 | public Mutation convertToMutation(RowData record) { 67 | Long timestamp = timestampMetadata.read(record); 68 | Long timeToLive = timeToLiveMetadata.read(record); 69 | RowKind kind = record.getRowKind(); 70 | if (kind == RowKind.INSERT || kind == RowKind.UPDATE_AFTER) { 71 | return serde.createPutMutation(record, timestamp, timeToLive); 72 | } else { 73 | return serde.createDeleteMutation(record, timestamp); 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/src/main/java/org/apache/flink/connector/hbase/sink/WritableMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase.sink; 20 | 21 | import org.apache.flink.table.api.DataTypes; 22 | import org.apache.flink.table.data.RowData; 23 | import org.apache.flink.table.types.DataType; 24 | 25 | import org.apache.hadoop.hbase.HConstants; 26 | 27 | import java.io.Serializable; 28 | import java.util.Collections; 29 | import java.util.HashMap; 30 | import java.util.List; 31 | import java.util.Map; 32 | 33 | /** Writable metadata for HBase. */ 34 | public abstract class WritableMetadata implements Serializable { 35 | 36 | private static final long serialVersionUID = 1L; 37 | 38 | public abstract T read(RowData row); 39 | 40 | /** 41 | * Returns the map of metadata keys and their corresponding data types that can be consumed by 42 | * HBase sink for writing. 43 | * 44 | *

Note: All the supported writable metadata should be manually registered in it. 45 | */ 46 | public static Map list() { 47 | Map metadataMap = new HashMap<>(); 48 | metadataMap.put(TimestampMetadata.KEY, TimestampMetadata.DATA_TYPE); 49 | metadataMap.put(TimeToLiveMetadata.KEY, TimeToLiveMetadata.DATA_TYPE); 50 | return Collections.unmodifiableMap(metadataMap); 51 | } 52 | 53 | private static void validateNotNull(RowData row, int pos, String key) { 54 | if (row.isNullAt(pos)) { 55 | throw new IllegalArgumentException( 56 | String.format("Writable metadata '%s' can not accept null value", key)); 57 | } 58 | } 59 | 60 | /** Timestamp metadata for HBase. */ 61 | public static class TimestampMetadata extends WritableMetadata { 62 | 63 | public static final String KEY = "timestamp"; 64 | public static final DataType DATA_TYPE = 65 | DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3).nullable(); 66 | 67 | private final int pos; 68 | 69 | public TimestampMetadata(List metadataKeys, DataType physicalDataType) { 70 | int idx = metadataKeys.indexOf(KEY); 71 | this.pos = idx < 0 ? -1 : idx + physicalDataType.getLogicalType().getChildren().size(); 72 | } 73 | 74 | @Override 75 | public Long read(RowData row) { 76 | if (pos < 0) { 77 | return HConstants.LATEST_TIMESTAMP; 78 | } 79 | validateNotNull(row, pos, KEY); 80 | return row.getTimestamp(pos, 3).getMillisecond(); 81 | } 82 | } 83 | 84 | /** Time-to-live metadata for HBase. */ 85 | public static class TimeToLiveMetadata extends WritableMetadata { 86 | 87 | public static final String KEY = "ttl"; 88 | public static final DataType DATA_TYPE = DataTypes.BIGINT().nullable(); 89 | 90 | private final int pos; 91 | 92 | public TimeToLiveMetadata(List metadataKeys, DataType physicalDataType) { 93 | int idx = metadataKeys.indexOf(KEY); 94 | this.pos = idx < 0 ? -1 : idx + physicalDataType.getLogicalType().getChildren().size(); 95 | } 96 | 97 | @Override 98 | public Long read(RowData row) { 99 | if (pos < 0) { 100 | return null; 101 | } 102 | validateNotNull(row, pos, KEY); 103 | return row.getLong(pos); 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/src/main/java/org/apache/flink/connector/hbase/source/AbstractHBaseDynamicTableSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase.source; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | import org.apache.flink.annotation.VisibleForTesting; 23 | import org.apache.flink.api.common.io.InputFormat; 24 | import org.apache.flink.connector.hbase.util.HBaseTableSchema; 25 | import org.apache.flink.table.connector.ChangelogMode; 26 | import org.apache.flink.table.connector.Projection; 27 | import org.apache.flink.table.connector.source.InputFormatProvider; 28 | import org.apache.flink.table.connector.source.LookupTableSource; 29 | import org.apache.flink.table.connector.source.ScanTableSource; 30 | import org.apache.flink.table.connector.source.abilities.SupportsProjectionPushDown; 31 | import org.apache.flink.table.connector.source.lookup.LookupFunctionProvider; 32 | import org.apache.flink.table.connector.source.lookup.PartialCachingLookupProvider; 33 | import org.apache.flink.table.connector.source.lookup.cache.LookupCache; 34 | import org.apache.flink.table.data.RowData; 35 | import org.apache.flink.table.types.DataType; 36 | 37 | import org.apache.hadoop.conf.Configuration; 38 | 39 | import javax.annotation.Nullable; 40 | 41 | import static org.apache.flink.util.Preconditions.checkArgument; 42 | 43 | /** HBase table source implementation. */ 44 | @Internal 45 | public abstract class AbstractHBaseDynamicTableSource 46 | implements ScanTableSource, LookupTableSource, SupportsProjectionPushDown { 47 | 48 | protected final Configuration conf; 49 | protected final String tableName; 50 | protected HBaseTableSchema hbaseSchema; 51 | protected final String nullStringLiteral; 52 | protected final int maxRetryTimes; 53 | @Nullable protected final LookupCache cache; 54 | 55 | public AbstractHBaseDynamicTableSource( 56 | Configuration conf, 57 | String tableName, 58 | HBaseTableSchema hbaseSchema, 59 | String nullStringLiteral, 60 | int maxRetryTimes, 61 | @Nullable LookupCache cache) { 62 | this.conf = conf; 63 | this.tableName = tableName; 64 | this.hbaseSchema = hbaseSchema; 65 | this.nullStringLiteral = nullStringLiteral; 66 | this.maxRetryTimes = maxRetryTimes; 67 | this.cache = cache; 68 | } 69 | 70 | @Override 71 | public ScanRuntimeProvider getScanRuntimeProvider(ScanContext runtimeProviderContext) { 72 | return InputFormatProvider.of(getInputFormat()); 73 | } 74 | 75 | protected abstract InputFormat getInputFormat(); 76 | 77 | @Override 78 | public LookupRuntimeProvider getLookupRuntimeProvider(LookupContext context) { 79 | checkArgument( 80 | context.getKeys().length == 1 && context.getKeys()[0].length == 1, 81 | "Currently, HBase table can only be lookup by single rowkey."); 82 | checkArgument( 83 | hbaseSchema.getRowKeyName().isPresent(), 84 | "HBase schema must have a row key when used in lookup mode."); 85 | checkArgument( 86 | DataType.getFieldNames(hbaseSchema.convertToDataType()) 87 | .get(context.getKeys()[0][0]) 88 | .equals(hbaseSchema.getRowKeyName().get()), 89 | "Currently, HBase table only supports lookup by rowkey field."); 90 | HBaseRowDataLookupFunction lookupFunction = 91 | new HBaseRowDataLookupFunction( 92 | conf, tableName, hbaseSchema, nullStringLiteral, maxRetryTimes); 93 | if (cache != null) { 94 | return PartialCachingLookupProvider.of(lookupFunction, cache); 95 | } else { 96 | return LookupFunctionProvider.of(lookupFunction); 97 | } 98 | } 99 | 100 | @Override 101 | public boolean supportsNestedProjection() { 102 | // planner doesn't support nested projection push down yet. 103 | return false; 104 | } 105 | 106 | @Override 107 | public void applyProjection(int[][] projectedFields, DataType producedDataType) { 108 | this.hbaseSchema = 109 | HBaseTableSchema.fromDataType( 110 | Projection.of(projectedFields).project(hbaseSchema.convertToDataType())); 111 | } 112 | 113 | @Override 114 | public ChangelogMode getChangelogMode() { 115 | return ChangelogMode.insertOnly(); 116 | } 117 | 118 | @Override 119 | public String asSummaryString() { 120 | return "HBase"; 121 | } 122 | 123 | // ------------------------------------------------------------------------------------------- 124 | 125 | @VisibleForTesting 126 | public HBaseTableSchema getHBaseTableSchema() { 127 | return this.hbaseSchema; 128 | } 129 | 130 | @VisibleForTesting 131 | public int getMaxRetryTimes() { 132 | return maxRetryTimes; 133 | } 134 | 135 | @VisibleForTesting 136 | @Nullable 137 | public LookupCache getCache() { 138 | return cache; 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/src/main/java/org/apache/flink/connector/hbase/source/HBaseRowDataLookupFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase.source; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | import org.apache.flink.annotation.VisibleForTesting; 23 | import org.apache.flink.connector.hbase.util.HBaseConfigurationUtil; 24 | import org.apache.flink.connector.hbase.util.HBaseSerde; 25 | import org.apache.flink.connector.hbase.util.HBaseTableSchema; 26 | import org.apache.flink.table.data.GenericRowData; 27 | import org.apache.flink.table.data.RowData; 28 | import org.apache.flink.table.functions.FunctionContext; 29 | import org.apache.flink.table.functions.LookupFunction; 30 | import org.apache.flink.util.StringUtils; 31 | 32 | import org.apache.hadoop.conf.Configuration; 33 | import org.apache.hadoop.hbase.HConstants; 34 | import org.apache.hadoop.hbase.TableName; 35 | import org.apache.hadoop.hbase.TableNotFoundException; 36 | import org.apache.hadoop.hbase.client.Connection; 37 | import org.apache.hadoop.hbase.client.ConnectionFactory; 38 | import org.apache.hadoop.hbase.client.Get; 39 | import org.apache.hadoop.hbase.client.HTable; 40 | import org.apache.hadoop.hbase.client.Result; 41 | import org.slf4j.Logger; 42 | import org.slf4j.LoggerFactory; 43 | 44 | import java.io.IOException; 45 | import java.util.Collection; 46 | import java.util.Collections; 47 | 48 | /** 49 | * The HBaseRowDataLookupFunction is a standard user-defined table function, it can be used in 50 | * tableAPI and also useful for temporal table join plan in SQL. It looks up the result as {@link 51 | * RowData}. 52 | */ 53 | @Internal 54 | public class HBaseRowDataLookupFunction extends LookupFunction { 55 | 56 | private static final Logger LOG = LoggerFactory.getLogger(HBaseRowDataLookupFunction.class); 57 | private static final long serialVersionUID = 1L; 58 | 59 | private final String hTableName; 60 | private final byte[] serializedConfig; 61 | private final HBaseTableSchema hbaseTableSchema; 62 | private final String nullStringLiteral; 63 | 64 | private transient Connection hConnection; 65 | private transient HTable table; 66 | private transient HBaseSerde serde; 67 | 68 | private final int maxRetryTimes; 69 | 70 | public HBaseRowDataLookupFunction( 71 | Configuration configuration, 72 | String hTableName, 73 | HBaseTableSchema hbaseTableSchema, 74 | String nullStringLiteral, 75 | int maxRetryTimes) { 76 | this.serializedConfig = HBaseConfigurationUtil.serializeConfiguration(configuration); 77 | this.hTableName = hTableName; 78 | this.hbaseTableSchema = hbaseTableSchema; 79 | this.nullStringLiteral = nullStringLiteral; 80 | this.maxRetryTimes = maxRetryTimes; 81 | } 82 | 83 | /** 84 | * The invoke entry point of lookup function. 85 | * 86 | * @param keyRow - A {@link RowData} that wraps lookup keys. Currently only support single 87 | * rowkey. 88 | */ 89 | @Override 90 | public Collection lookup(RowData keyRow) throws IOException { 91 | for (int retry = 0; retry <= maxRetryTimes; retry++) { 92 | try { 93 | // TODO: The implementation of LookupFunction will pass a GenericRowData as key row 94 | // and it's safe to cast for now. We need to update the logic once we improve the 95 | // LookupFunction in the future. 96 | Get get = serde.createGet(((GenericRowData) keyRow).getField(0)); 97 | if (get != null) { 98 | Result result = table.get(get); 99 | if (!result.isEmpty()) { 100 | return Collections.singletonList(serde.convertToNewRow(result)); 101 | } 102 | } 103 | break; 104 | } catch (IOException e) { 105 | LOG.error(String.format("HBase lookup error, retry times = %d", retry), e); 106 | if (retry >= maxRetryTimes) { 107 | throw new RuntimeException("Execution of HBase lookup failed.", e); 108 | } 109 | try { 110 | Thread.sleep(1000 * retry); 111 | } catch (InterruptedException e1) { 112 | throw new RuntimeException(e1); 113 | } 114 | } 115 | } 116 | return Collections.emptyList(); 117 | } 118 | 119 | private Configuration prepareRuntimeConfiguration() { 120 | // create default configuration from current runtime env (`hbase-site.xml` in classpath) 121 | // first, 122 | // and overwrite configuration using serialized configuration from client-side env 123 | // (`hbase-site.xml` in classpath). 124 | // user params from client-side have the highest priority 125 | Configuration runtimeConfig = 126 | HBaseConfigurationUtil.deserializeConfiguration( 127 | serializedConfig, HBaseConfigurationUtil.getHBaseConfiguration()); 128 | 129 | // do validation: check key option(s) in final runtime configuration 130 | if (StringUtils.isNullOrWhitespaceOnly(runtimeConfig.get(HConstants.ZOOKEEPER_QUORUM))) { 131 | LOG.error( 132 | "can not connect to HBase without {} configuration", 133 | HConstants.ZOOKEEPER_QUORUM); 134 | throw new IllegalArgumentException( 135 | "check HBase configuration failed, lost: '" 136 | + HConstants.ZOOKEEPER_QUORUM 137 | + "'!"); 138 | } 139 | 140 | return runtimeConfig; 141 | } 142 | 143 | @Override 144 | public void open(FunctionContext context) { 145 | LOG.info("start open ..."); 146 | Configuration config = prepareRuntimeConfiguration(); 147 | try { 148 | hConnection = ConnectionFactory.createConnection(config); 149 | table = (HTable) hConnection.getTable(TableName.valueOf(hTableName)); 150 | } catch (TableNotFoundException tnfe) { 151 | LOG.error("Table '{}' not found ", hTableName, tnfe); 152 | throw new RuntimeException("HBase table '" + hTableName + "' not found.", tnfe); 153 | } catch (IOException ioe) { 154 | LOG.error("Exception while creating connection to HBase.", ioe); 155 | throw new RuntimeException("Cannot create connection to HBase.", ioe); 156 | } 157 | this.serde = new HBaseSerde(hbaseTableSchema, nullStringLiteral); 158 | LOG.info("end open."); 159 | } 160 | 161 | @Override 162 | public void close() { 163 | LOG.info("start close ..."); 164 | if (null != table) { 165 | try { 166 | table.close(); 167 | table = null; 168 | } catch (IOException e) { 169 | // ignore exception when close. 170 | LOG.warn("exception when close table", e); 171 | } 172 | } 173 | if (null != hConnection) { 174 | try { 175 | hConnection.close(); 176 | hConnection = null; 177 | } catch (IOException e) { 178 | // ignore exception when close. 179 | LOG.warn("exception when close connection", e); 180 | } 181 | } 182 | LOG.info("end close."); 183 | } 184 | 185 | @VisibleForTesting 186 | public String getHTableName() { 187 | return hTableName; 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/src/main/java/org/apache/flink/connector/hbase/source/TableInputSplit.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase.source; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | import org.apache.flink.core.io.LocatableInputSplit; 23 | 24 | /** 25 | * This class implements a input splits for HBase. Each table input split corresponds to a key range 26 | * (low, high). All references to row below refer to the key of the row. 27 | */ 28 | @Internal 29 | public class TableInputSplit extends LocatableInputSplit { 30 | 31 | private static final long serialVersionUID = 1L; 32 | 33 | /** The name of the table to retrieve data from. */ 34 | private final byte[] tableName; 35 | 36 | /** The start row of the split. */ 37 | private final byte[] startRow; 38 | 39 | /** The end row of the split. */ 40 | private final byte[] endRow; 41 | 42 | /** 43 | * Creates a new table input split. 44 | * 45 | * @param splitNumber the number of the input split 46 | * @param hostnames the names of the hosts storing the data the input split refers to 47 | * @param tableName the name of the table to retrieve data from 48 | * @param startRow the start row of the split 49 | * @param endRow the end row of the split 50 | */ 51 | public TableInputSplit( 52 | final int splitNumber, 53 | final String[] hostnames, 54 | final byte[] tableName, 55 | final byte[] startRow, 56 | final byte[] endRow) { 57 | super(splitNumber, hostnames); 58 | 59 | this.tableName = tableName; 60 | this.startRow = startRow; 61 | this.endRow = endRow; 62 | } 63 | 64 | /** 65 | * Returns the table name. 66 | * 67 | * @return The table name. 68 | */ 69 | public byte[] getTableName() { 70 | return this.tableName; 71 | } 72 | 73 | /** 74 | * Returns the start row. 75 | * 76 | * @return The start row. 77 | */ 78 | public byte[] getStartRow() { 79 | return this.startRow; 80 | } 81 | 82 | /** 83 | * Returns the end row. 84 | * 85 | * @return The end row. 86 | */ 87 | public byte[] getEndRow() { 88 | return this.endRow; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/src/main/java/org/apache/flink/connector/hbase/table/HBaseConnectorOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase.table; 20 | 21 | import org.apache.flink.annotation.PublicEvolving; 22 | import org.apache.flink.configuration.ConfigOption; 23 | import org.apache.flink.configuration.ConfigOptions; 24 | import org.apache.flink.configuration.MemorySize; 25 | import org.apache.flink.table.connector.source.lookup.LookupOptions; 26 | import org.apache.flink.table.factories.FactoryUtil; 27 | 28 | import java.time.Duration; 29 | 30 | /** Options for the HBase connector. */ 31 | @PublicEvolving 32 | public class HBaseConnectorOptions { 33 | 34 | public static final ConfigOption TABLE_NAME = 35 | ConfigOptions.key("table-name") 36 | .stringType() 37 | .noDefaultValue() 38 | .withDescription( 39 | "The name of HBase table to connect. " 40 | + "By default, the table is in 'default' namespace. " 41 | + "To assign the table a specified namespace you need to use 'namespace:table'."); 42 | 43 | public static final ConfigOption ZOOKEEPER_QUORUM = 44 | ConfigOptions.key("zookeeper.quorum") 45 | .stringType() 46 | .noDefaultValue() 47 | .withDescription("The HBase Zookeeper quorum."); 48 | 49 | public static final ConfigOption ZOOKEEPER_ZNODE_PARENT = 50 | ConfigOptions.key("zookeeper.znode.parent") 51 | .stringType() 52 | .defaultValue("/hbase") 53 | .withDescription("The root dir in Zookeeper for HBase cluster."); 54 | 55 | public static final ConfigOption NULL_STRING_LITERAL = 56 | ConfigOptions.key("null-string-literal") 57 | .stringType() 58 | .defaultValue("null") 59 | .withDescription( 60 | "Representation for null values for string fields. HBase source and " 61 | + "sink encodes/decodes empty bytes as null values for all types except string type."); 62 | 63 | public static final ConfigOption SINK_BUFFER_FLUSH_MAX_SIZE = 64 | ConfigOptions.key("sink.buffer-flush.max-size") 65 | .memoryType() 66 | .defaultValue(MemorySize.parse("2mb")) 67 | .withDescription( 68 | "Writing option, maximum size in memory of buffered rows for each " 69 | + "writing request. This can improve performance for writing data to HBase database, " 70 | + "but may increase the latency. Can be set to '0' to disable it. "); 71 | 72 | public static final ConfigOption SINK_BUFFER_FLUSH_MAX_ROWS = 73 | ConfigOptions.key("sink.buffer-flush.max-rows") 74 | .intType() 75 | .defaultValue(1000) 76 | .withDescription( 77 | "Writing option, maximum number of rows to buffer for each writing request. " 78 | + "This can improve performance for writing data to HBase database, but may increase the latency. " 79 | + "Can be set to '0' to disable it."); 80 | 81 | public static final ConfigOption SINK_BUFFER_FLUSH_INTERVAL = 82 | ConfigOptions.key("sink.buffer-flush.interval") 83 | .durationType() 84 | .defaultValue(Duration.ofSeconds(1)) 85 | .withDescription( 86 | "Writing option, the interval to flush any buffered rows. " 87 | + "This can improve performance for writing data to HBase database, but may increase the latency. " 88 | + "Can be set to '0' to disable it. Note, both 'sink.buffer-flush.max-size' and 'sink.buffer-flush.max-rows' " 89 | + "can be set to '0' with the flush interval set allowing for complete async processing of buffered actions."); 90 | 91 | public static final ConfigOption SINK_IGNORE_NULL_VALUE = 92 | ConfigOptions.key("sink.ignore-null-value") 93 | .booleanType() 94 | .defaultValue(false) 95 | .withDescription("Writing option, whether ignore null value or not."); 96 | 97 | public static final ConfigOption LOOKUP_ASYNC = 98 | ConfigOptions.key("lookup.async") 99 | .booleanType() 100 | .defaultValue(false) 101 | .withDescription("whether to set async lookup."); 102 | 103 | /** @deprecated Please use {@link LookupOptions#PARTIAL_CACHE_MAX_ROWS} instead. */ 104 | @Deprecated 105 | public static final ConfigOption LOOKUP_CACHE_MAX_ROWS = 106 | ConfigOptions.key("lookup.cache.max-rows") 107 | .longType() 108 | .defaultValue(-1L) 109 | .withDescription( 110 | "the max number of rows of lookup cache, over this value, the oldest rows will " 111 | + "be eliminated. \"cache.max-rows\" and \"cache.ttl\" options must all be specified if any of them is " 112 | + "specified. Cache is not enabled as default."); 113 | 114 | /** @deprecated Please use {@link LookupOptions#PARTIAL_CACHE_EXPIRE_AFTER_WRITE} instead. */ 115 | @Deprecated 116 | public static final ConfigOption LOOKUP_CACHE_TTL = 117 | ConfigOptions.key("lookup.cache.ttl") 118 | .durationType() 119 | .defaultValue(Duration.ofSeconds(0)) 120 | .withDescription("the cache time to live."); 121 | 122 | /** @deprecated Please used {@link LookupOptions#MAX_RETRIES} instead. */ 123 | public static final ConfigOption LOOKUP_MAX_RETRIES = 124 | ConfigOptions.key("lookup.max-retries") 125 | .intType() 126 | .defaultValue(3) 127 | .withDescription("the max retry times if lookup database failed."); 128 | 129 | public static final ConfigOption SINK_PARALLELISM = FactoryUtil.SINK_PARALLELISM; 130 | 131 | private HBaseConnectorOptions() {} 132 | } 133 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/src/main/java/org/apache/flink/connector/hbase/table/HBaseConnectorOptionsUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase.table; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | import org.apache.flink.configuration.ReadableConfig; 23 | import org.apache.flink.connector.hbase.options.HBaseWriteOptions; 24 | import org.apache.flink.connector.hbase.util.HBaseConfigurationUtil; 25 | import org.apache.flink.connector.hbase.util.HBaseTableSchema; 26 | import org.apache.flink.table.types.DataType; 27 | 28 | import org.apache.hadoop.conf.Configuration; 29 | import org.apache.hadoop.hbase.HConstants; 30 | 31 | import java.util.Map; 32 | import java.util.Properties; 33 | 34 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.SINK_BUFFER_FLUSH_INTERVAL; 35 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.SINK_BUFFER_FLUSH_MAX_ROWS; 36 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.SINK_BUFFER_FLUSH_MAX_SIZE; 37 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.SINK_IGNORE_NULL_VALUE; 38 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.SINK_PARALLELISM; 39 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.ZOOKEEPER_QUORUM; 40 | import static org.apache.flink.connector.hbase.table.HBaseConnectorOptions.ZOOKEEPER_ZNODE_PARENT; 41 | 42 | /** Utilities for {@link HBaseConnectorOptions}. */ 43 | @Internal 44 | public class HBaseConnectorOptionsUtil { 45 | 46 | /** Prefix for HBase specific properties. */ 47 | public static final String PROPERTIES_PREFIX = "properties."; 48 | 49 | // -------------------------------------------------------------------------------------------- 50 | // Validation 51 | // -------------------------------------------------------------------------------------------- 52 | 53 | /** 54 | * Checks that the HBase table have row key defined. A row key is defined as an atomic type, and 55 | * column families and qualifiers are defined as ROW type. There shouldn't be multiple atomic 56 | * type columns in the schema. The PRIMARY KEY constraint is optional, if exist, the primary key 57 | * constraint must be defined on the single row key field. 58 | */ 59 | public static void validatePrimaryKey(DataType dataType, int[] primaryKeyIndexes) { 60 | HBaseTableSchema hbaseSchema = HBaseTableSchema.fromDataType(dataType); 61 | if (!hbaseSchema.getRowKeyName().isPresent()) { 62 | throw new IllegalArgumentException( 63 | "HBase table requires to define a row key field. " 64 | + "A row key field is defined as an atomic type, " 65 | + "column families and qualifiers are defined as ROW type."); 66 | } 67 | if (primaryKeyIndexes.length == 0) { 68 | return; 69 | } 70 | if (primaryKeyIndexes.length > 1) { 71 | throw new IllegalArgumentException( 72 | "HBase table doesn't support a primary Key on multiple columns. " 73 | + "The primary key of HBase table must be defined on row key field."); 74 | } 75 | if (!hbaseSchema 76 | .getRowKeyName() 77 | .get() 78 | .equals(DataType.getFieldNames(dataType).get(primaryKeyIndexes[0]))) { 79 | throw new IllegalArgumentException( 80 | "Primary key of HBase table must be defined on the row key field. " 81 | + "A row key field is defined as an atomic type, " 82 | + "column families and qualifiers are defined as ROW type."); 83 | } 84 | } 85 | 86 | public static HBaseWriteOptions getHBaseWriteOptions(ReadableConfig tableOptions) { 87 | HBaseWriteOptions.Builder builder = HBaseWriteOptions.builder(); 88 | builder.setBufferFlushIntervalMillis( 89 | tableOptions.get(SINK_BUFFER_FLUSH_INTERVAL).toMillis()); 90 | builder.setBufferFlushMaxRows(tableOptions.get(SINK_BUFFER_FLUSH_MAX_ROWS)); 91 | builder.setBufferFlushMaxSizeInBytes( 92 | tableOptions.get(SINK_BUFFER_FLUSH_MAX_SIZE).getBytes()); 93 | builder.setIgnoreNullValue(tableOptions.get(SINK_IGNORE_NULL_VALUE)); 94 | builder.setParallelism(tableOptions.getOptional(SINK_PARALLELISM).orElse(null)); 95 | return builder.build(); 96 | } 97 | 98 | /** config HBase Configuration. */ 99 | public static Configuration getHBaseConfiguration(ReadableConfig tableOptions) { 100 | // create default configuration from current runtime env (`hbase-site.xml` in classpath) 101 | // first, 102 | Configuration hbaseClientConf = HBaseConfigurationUtil.getHBaseConfiguration(); 103 | hbaseClientConf.set(HConstants.ZOOKEEPER_QUORUM, tableOptions.get(ZOOKEEPER_QUORUM)); 104 | hbaseClientConf.set( 105 | HConstants.ZOOKEEPER_ZNODE_PARENT, tableOptions.get(ZOOKEEPER_ZNODE_PARENT)); 106 | // add HBase properties 107 | final Properties properties = 108 | getHBaseClientProperties( 109 | ((org.apache.flink.configuration.Configuration) tableOptions).toMap()); 110 | properties.forEach((k, v) -> hbaseClientConf.set(k.toString(), v.toString())); 111 | return hbaseClientConf; 112 | } 113 | 114 | private static Properties getHBaseClientProperties(Map tableOptions) { 115 | final Properties hbaseProperties = new Properties(); 116 | 117 | if (containsHBaseClientProperties(tableOptions)) { 118 | tableOptions.keySet().stream() 119 | .filter(key -> key.startsWith(PROPERTIES_PREFIX)) 120 | .forEach( 121 | key -> { 122 | final String value = tableOptions.get(key); 123 | final String subKey = key.substring((PROPERTIES_PREFIX).length()); 124 | hbaseProperties.put(subKey, value); 125 | }); 126 | } 127 | return hbaseProperties; 128 | } 129 | 130 | /** Returns whether the table options contains HBase client properties or not. 'properties'. */ 131 | private static boolean containsHBaseClientProperties(Map tableOptions) { 132 | return tableOptions.keySet().stream().anyMatch(k -> k.startsWith(PROPERTIES_PREFIX)); 133 | } 134 | 135 | private HBaseConnectorOptionsUtil() {} 136 | } 137 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/src/main/java/org/apache/flink/connector/hbase/util/HBaseConfigurationUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase.util; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | import org.apache.flink.util.Preconditions; 23 | 24 | import org.apache.hadoop.conf.Configuration; 25 | import org.apache.hadoop.hbase.HBaseConfiguration; 26 | import org.apache.hadoop.io.Writable; 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | import java.io.ByteArrayInputStream; 31 | import java.io.ByteArrayOutputStream; 32 | import java.io.DataInputStream; 33 | import java.io.DataOutputStream; 34 | import java.io.File; 35 | import java.io.IOException; 36 | 37 | /** This class helps to do serialization for hadoop Configuration and HBase-related classes. */ 38 | @Internal 39 | public class HBaseConfigurationUtil { 40 | 41 | private static final Logger LOG = LoggerFactory.getLogger(HBaseConfigurationUtil.class); 42 | 43 | public static final String ENV_HBASE_CONF_DIR = "HBASE_CONF_DIR"; 44 | 45 | public static Configuration getHBaseConfiguration() { 46 | 47 | // Instantiate an HBaseConfiguration to load the hbase-default.xml and hbase-site.xml from 48 | // the classpath. 49 | Configuration result = HBaseConfiguration.create(); 50 | boolean foundHBaseConfiguration = false; 51 | 52 | // We need to load both hbase-default.xml and hbase-site.xml to the hbase configuration 53 | // The properties of a newly added resource will override the ones in previous resources, so 54 | // a configuration 55 | // file with higher priority should be added later. 56 | 57 | // Approach 1: HBASE_HOME environment variables 58 | String possibleHBaseConfPath = null; 59 | 60 | final String hbaseHome = System.getenv("HBASE_HOME"); 61 | if (hbaseHome != null) { 62 | LOG.debug("Searching HBase configuration files in HBASE_HOME: {}", hbaseHome); 63 | possibleHBaseConfPath = hbaseHome + "/conf"; 64 | } 65 | 66 | if (possibleHBaseConfPath != null) { 67 | foundHBaseConfiguration = addHBaseConfIfFound(result, possibleHBaseConfPath); 68 | } 69 | 70 | // Approach 2: HBASE_CONF_DIR environment variable 71 | String hbaseConfDir = System.getenv("HBASE_CONF_DIR"); 72 | if (hbaseConfDir != null) { 73 | LOG.debug("Searching HBase configuration files in HBASE_CONF_DIR: {}", hbaseConfDir); 74 | foundHBaseConfiguration = 75 | addHBaseConfIfFound(result, hbaseConfDir) || foundHBaseConfiguration; 76 | } 77 | 78 | if (!foundHBaseConfiguration) { 79 | LOG.warn( 80 | "Could not find HBase configuration via any of the supported methods " 81 | + "(Flink configuration, environment variables)."); 82 | } 83 | 84 | return result; 85 | } 86 | 87 | /** 88 | * Search HBase configuration files in the given path, and add them to the configuration if 89 | * found. 90 | */ 91 | private static boolean addHBaseConfIfFound( 92 | Configuration configuration, String possibleHBaseConfPath) { 93 | boolean foundHBaseConfiguration = false; 94 | if (new File(possibleHBaseConfPath).exists()) { 95 | if (new File(possibleHBaseConfPath + "/hbase-default.xml").exists()) { 96 | configuration.addResource( 97 | new org.apache.hadoop.fs.Path( 98 | possibleHBaseConfPath + "/hbase-default.xml")); 99 | LOG.debug( 100 | "Adding " 101 | + possibleHBaseConfPath 102 | + "/hbase-default.xml to hbase configuration"); 103 | foundHBaseConfiguration = true; 104 | } 105 | if (new File(possibleHBaseConfPath + "/hbase-site.xml").exists()) { 106 | configuration.addResource( 107 | new org.apache.hadoop.fs.Path(possibleHBaseConfPath + "/hbase-site.xml")); 108 | LOG.debug( 109 | "Adding " 110 | + possibleHBaseConfPath 111 | + "/hbase-site.xml to hbase configuration"); 112 | foundHBaseConfiguration = true; 113 | } 114 | } 115 | return foundHBaseConfiguration; 116 | } 117 | 118 | /** Serialize a Hadoop {@link Configuration} into byte[]. */ 119 | public static byte[] serializeConfiguration(Configuration conf) { 120 | try { 121 | return serializeWritable(conf); 122 | } catch (IOException e) { 123 | throw new RuntimeException( 124 | "Encounter an IOException when serialize the Configuration.", e); 125 | } 126 | } 127 | 128 | /** 129 | * Deserialize a Hadoop {@link Configuration} from byte[]. Deserialize configs to {@code 130 | * targetConfig} if it is set. 131 | */ 132 | public static Configuration deserializeConfiguration( 133 | byte[] serializedConfig, Configuration targetConfig) { 134 | if (null == targetConfig) { 135 | targetConfig = new Configuration(); 136 | } 137 | try { 138 | deserializeWritable(targetConfig, serializedConfig); 139 | } catch (IOException e) { 140 | throw new RuntimeException( 141 | "Encounter an IOException when deserialize the Configuration.", e); 142 | } 143 | return targetConfig; 144 | } 145 | 146 | /** 147 | * Serialize writable byte[]. 148 | * 149 | * @param the type parameter 150 | * @param writable the writable 151 | * @return the byte [ ] 152 | * @throws IOException the io exception 153 | */ 154 | private static byte[] serializeWritable(T writable) throws IOException { 155 | Preconditions.checkArgument(writable != null); 156 | 157 | ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); 158 | DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); 159 | writable.write(outputStream); 160 | return byteArrayOutputStream.toByteArray(); 161 | } 162 | 163 | /** 164 | * Deserialize writable. 165 | * 166 | * @param the type parameter 167 | * @param writable the writable 168 | * @param bytes the bytes 169 | * @throws IOException the io exception 170 | */ 171 | private static void deserializeWritable(T writable, byte[] bytes) 172 | throws IOException { 173 | Preconditions.checkArgument(writable != null); 174 | Preconditions.checkArgument(bytes != null); 175 | 176 | ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); 177 | DataInputStream dataInputStream = new DataInputStream(byteArrayInputStream); 178 | writable.readFields(dataInputStream); 179 | } 180 | 181 | public static org.apache.hadoop.conf.Configuration createHBaseConf() { 182 | org.apache.hadoop.conf.Configuration hbaseClientConf = HBaseConfiguration.create(); 183 | 184 | String hbaseConfDir = System.getenv(ENV_HBASE_CONF_DIR); 185 | 186 | if (hbaseConfDir != null) { 187 | if (new File(hbaseConfDir).exists()) { 188 | String coreSite = hbaseConfDir + "/core-site.xml"; 189 | String hdfsSite = hbaseConfDir + "/hdfs-site.xml"; 190 | String hbaseSite = hbaseConfDir + "/hbase-site.xml"; 191 | if (new File(coreSite).exists()) { 192 | hbaseClientConf.addResource(new org.apache.hadoop.fs.Path(coreSite)); 193 | LOG.info("Adding " + coreSite + " to hbase configuration"); 194 | } 195 | if (new File(hdfsSite).exists()) { 196 | hbaseClientConf.addResource(new org.apache.hadoop.fs.Path(hdfsSite)); 197 | LOG.info("Adding " + hdfsSite + " to hbase configuration"); 198 | } 199 | if (new File(hbaseSite).exists()) { 200 | hbaseClientConf.addResource(new org.apache.hadoop.fs.Path(hbaseSite)); 201 | LOG.info("Adding " + hbaseSite + " to hbase configuration"); 202 | } 203 | } else { 204 | LOG.warn( 205 | "HBase config directory '{}' not found, cannot load HBase configuration.", 206 | hbaseConfDir); 207 | } 208 | } else { 209 | LOG.warn( 210 | "{} env variable not found, cannot load HBase configuration.", 211 | ENV_HBASE_CONF_DIR); 212 | } 213 | return hbaseClientConf; 214 | } 215 | } 216 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/src/main/java/org/apache/flink/connector/hbase/util/HBaseTypeUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase.util; 20 | 21 | import org.apache.flink.annotation.Internal; 22 | import org.apache.flink.api.common.typeinfo.TypeInformation; 23 | import org.apache.flink.table.types.logical.LogicalType; 24 | 25 | import org.apache.hadoop.hbase.util.Bytes; 26 | 27 | import java.math.BigDecimal; 28 | import java.math.BigInteger; 29 | import java.nio.charset.Charset; 30 | import java.sql.Date; 31 | import java.sql.Time; 32 | import java.sql.Timestamp; 33 | import java.util.Arrays; 34 | 35 | import static org.apache.flink.table.types.logical.utils.LogicalTypeChecks.getPrecision; 36 | 37 | /** A utility class to process data exchange with HBase type system. */ 38 | @Internal 39 | public class HBaseTypeUtils { 40 | 41 | private static final byte[] EMPTY_BYTES = new byte[] {}; 42 | 43 | private static final int MIN_TIMESTAMP_PRECISION = 0; 44 | private static final int MAX_TIMESTAMP_PRECISION = 3; 45 | private static final int MIN_TIME_PRECISION = 0; 46 | private static final int MAX_TIME_PRECISION = 3; 47 | 48 | /** Deserialize byte array to Java Object with the given type. */ 49 | public static Object deserializeToObject(byte[] value, int typeIdx, Charset stringCharset) { 50 | switch (typeIdx) { 51 | case 0: // byte[] 52 | return value; 53 | case 1: // String 54 | return Arrays.equals(EMPTY_BYTES, value) ? null : new String(value, stringCharset); 55 | case 2: // byte 56 | return value[0]; 57 | case 3: 58 | return Bytes.toShort(value); 59 | case 4: 60 | return Bytes.toInt(value); 61 | case 5: 62 | return Bytes.toLong(value); 63 | case 6: 64 | return Bytes.toFloat(value); 65 | case 7: 66 | return Bytes.toDouble(value); 67 | case 8: 68 | return Bytes.toBoolean(value); 69 | case 9: // sql.Timestamp encoded as long 70 | return new Timestamp(Bytes.toLong(value)); 71 | case 10: // sql.Date encoded as long 72 | return new Date(Bytes.toLong(value)); 73 | case 11: // sql.Time encoded as long 74 | return new Time(Bytes.toLong(value)); 75 | case 12: 76 | return Bytes.toBigDecimal(value); 77 | case 13: 78 | return new BigInteger(value); 79 | 80 | default: 81 | throw new IllegalArgumentException("unsupported type index:" + typeIdx); 82 | } 83 | } 84 | 85 | /** Serialize the Java Object to byte array with the given type. */ 86 | public static byte[] serializeFromObject(Object value, int typeIdx, Charset stringCharset) { 87 | switch (typeIdx) { 88 | case 0: // byte[] 89 | return (byte[]) value; 90 | case 1: // external String 91 | return value == null ? EMPTY_BYTES : ((String) value).getBytes(stringCharset); 92 | case 2: // byte 93 | return value == null ? EMPTY_BYTES : new byte[] {(byte) value}; 94 | case 3: 95 | return Bytes.toBytes((short) value); 96 | case 4: 97 | return Bytes.toBytes((int) value); 98 | case 5: 99 | return Bytes.toBytes((long) value); 100 | case 6: 101 | return Bytes.toBytes((float) value); 102 | case 7: 103 | return Bytes.toBytes((double) value); 104 | case 8: 105 | return Bytes.toBytes((boolean) value); 106 | case 9: // sql.Timestamp encoded to Long 107 | return Bytes.toBytes(((Timestamp) value).getTime()); 108 | case 10: // sql.Date encoded as long 109 | return Bytes.toBytes(((Date) value).getTime()); 110 | case 11: // sql.Time encoded as long 111 | return Bytes.toBytes(((Time) value).getTime()); 112 | case 12: 113 | return Bytes.toBytes((BigDecimal) value); 114 | case 13: 115 | return ((BigInteger) value).toByteArray(); 116 | 117 | default: 118 | throw new IllegalArgumentException("unsupported type index:" + typeIdx); 119 | } 120 | } 121 | 122 | /** 123 | * Gets the type index (type representation in HBase connector) from the {@link 124 | * TypeInformation}. 125 | */ 126 | public static int getTypeIndex(TypeInformation typeInfo) { 127 | return getTypeIndex(typeInfo.getTypeClass()); 128 | } 129 | 130 | /** Checks whether the given Class is a supported type in HBase connector. */ 131 | public static boolean isSupportedType(Class clazz) { 132 | return getTypeIndex(clazz) != -1; 133 | } 134 | 135 | private static int getTypeIndex(Class clazz) { 136 | if (byte[].class.equals(clazz)) { 137 | return 0; 138 | } else if (String.class.equals(clazz)) { 139 | return 1; 140 | } else if (Byte.class.equals(clazz)) { 141 | return 2; 142 | } else if (Short.class.equals(clazz)) { 143 | return 3; 144 | } else if (Integer.class.equals(clazz)) { 145 | return 4; 146 | } else if (Long.class.equals(clazz)) { 147 | return 5; 148 | } else if (Float.class.equals(clazz)) { 149 | return 6; 150 | } else if (Double.class.equals(clazz)) { 151 | return 7; 152 | } else if (Boolean.class.equals(clazz)) { 153 | return 8; 154 | } else if (Timestamp.class.equals(clazz)) { 155 | return 9; 156 | } else if (Date.class.equals(clazz)) { 157 | return 10; 158 | } else if (Time.class.equals(clazz)) { 159 | return 11; 160 | } else if (BigDecimal.class.equals(clazz)) { 161 | return 12; 162 | } else if (BigInteger.class.equals(clazz)) { 163 | return 13; 164 | } else { 165 | return -1; 166 | } 167 | } 168 | 169 | /** Checks whether the given {@link LogicalType} is supported in HBase connector. */ 170 | public static boolean isSupportedType(LogicalType type) { 171 | // ordered by type root definition 172 | switch (type.getTypeRoot()) { 173 | case CHAR: 174 | case VARCHAR: 175 | case BOOLEAN: 176 | case BINARY: 177 | case VARBINARY: 178 | case DECIMAL: 179 | case TINYINT: 180 | case SMALLINT: 181 | case INTEGER: 182 | case DATE: 183 | case INTERVAL_YEAR_MONTH: 184 | case BIGINT: 185 | case INTERVAL_DAY_TIME: 186 | case FLOAT: 187 | case DOUBLE: 188 | return true; 189 | case TIME_WITHOUT_TIME_ZONE: 190 | final int timePrecision = getPrecision(type); 191 | if (timePrecision < MIN_TIME_PRECISION || timePrecision > MAX_TIME_PRECISION) { 192 | throw new UnsupportedOperationException( 193 | String.format( 194 | "The precision %s of TIME type is out of the range [%s, %s] supported by " 195 | + "HBase connector", 196 | timePrecision, MIN_TIME_PRECISION, MAX_TIME_PRECISION)); 197 | } 198 | return true; 199 | case TIMESTAMP_WITHOUT_TIME_ZONE: 200 | case TIMESTAMP_WITH_LOCAL_TIME_ZONE: 201 | final int timestampPrecision = getPrecision(type); 202 | if (timestampPrecision < MIN_TIMESTAMP_PRECISION 203 | || timestampPrecision > MAX_TIMESTAMP_PRECISION) { 204 | throw new UnsupportedOperationException( 205 | String.format( 206 | "The precision %s of TIMESTAMP type is out of the range [%s, %s] supported by " 207 | + "HBase connector", 208 | timestampPrecision, 209 | MIN_TIMESTAMP_PRECISION, 210 | MAX_TIMESTAMP_PRECISION)); 211 | } 212 | return true; 213 | case TIMESTAMP_WITH_TIME_ZONE: 214 | case ARRAY: 215 | case MULTISET: 216 | case MAP: 217 | case ROW: 218 | case STRUCTURED_TYPE: 219 | case DISTINCT_TYPE: 220 | case RAW: 221 | case NULL: 222 | case SYMBOL: 223 | case UNRESOLVED: 224 | return false; 225 | default: 226 | throw new IllegalArgumentException(); 227 | } 228 | } 229 | } 230 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/src/test/java/org/apache/flink/architecture/TestCodeArchitectureTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.architecture; 20 | 21 | import org.apache.flink.architecture.common.ImportOptions; 22 | 23 | import com.tngtech.archunit.core.importer.ImportOption; 24 | import com.tngtech.archunit.junit.AnalyzeClasses; 25 | import com.tngtech.archunit.junit.ArchTest; 26 | import com.tngtech.archunit.junit.ArchTests; 27 | 28 | /** Architecture tests for test code. */ 29 | @AnalyzeClasses( 30 | packages = "org.apache.flink.connector.hbase.util", 31 | importOptions = { 32 | ImportOption.OnlyIncludeTests.class, 33 | ImportOptions.ExcludeScalaImportOption.class, 34 | ImportOptions.ExcludeShadedImportOption.class 35 | }) 36 | public class TestCodeArchitectureTest { 37 | 38 | @ArchTest 39 | public static final ArchTests COMMON_TESTS = ArchTests.in(TestCodeArchitectureTestBase.class); 40 | } 41 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/src/test/java/org/apache/flink/connector/hbase/util/HBaseConfigLoadingTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase.util; 20 | 21 | import org.apache.flink.core.testutils.CommonTestUtils; 22 | 23 | import org.junit.jupiter.api.Test; 24 | import org.junit.jupiter.api.io.TempDir; 25 | 26 | import java.io.File; 27 | import java.io.FileOutputStream; 28 | import java.io.IOException; 29 | import java.io.PrintStream; 30 | import java.nio.file.Files; 31 | import java.nio.file.Path; 32 | import java.util.HashMap; 33 | import java.util.Map; 34 | 35 | import static org.assertj.core.api.Assertions.assertThat; 36 | 37 | /** 38 | * Tests that validate the loading of the HBase configuration, relative to entries in the Flink 39 | * configuration and the environment variables. 40 | */ 41 | class HBaseConfigLoadingTest { 42 | 43 | private static final String IN_HBASE_CONFIG_KEY = "hbase_conf_key"; 44 | private static final String IN_HBASE_CONFIG_VALUE = "hbase_conf_value!"; 45 | 46 | @TempDir Path tmpDir; 47 | 48 | @Test 49 | void loadFromClasspathByDefault() { 50 | org.apache.hadoop.conf.Configuration hbaseConf = 51 | HBaseConfigurationUtil.getHBaseConfiguration(); 52 | 53 | assertThat(hbaseConf.get(IN_HBASE_CONFIG_KEY, null)).isEqualTo(IN_HBASE_CONFIG_VALUE); 54 | } 55 | 56 | @Test 57 | void loadFromEnvVariables() throws Exception { 58 | final String k1 = "where?"; 59 | final String v1 = "I'm on a boat"; 60 | final String k2 = "when?"; 61 | final String v2 = "midnight"; 62 | final String k3 = "why?"; 63 | final String v3 = "what do you think?"; 64 | final String k4 = "which way?"; 65 | final String v4 = "south, always south..."; 66 | 67 | final File hbaseConfDir = tmpDir.toFile(); 68 | 69 | final File hbaseHome = Files.createTempDirectory(tmpDir, "junit_hbaseHome_").toFile(); 70 | 71 | final File hbaseHomeConf = new File(hbaseHome, "conf"); 72 | 73 | assertThat(hbaseHomeConf.mkdirs()).isTrue(); 74 | 75 | final File file1 = new File(hbaseConfDir, "hbase-default.xml"); 76 | final File file2 = new File(hbaseConfDir, "hbase-site.xml"); 77 | final File file3 = new File(hbaseHomeConf, "hbase-default.xml"); 78 | final File file4 = new File(hbaseHomeConf, "hbase-site.xml"); 79 | 80 | printConfig(file1, k1, v1); 81 | printConfig(file2, k2, v2); 82 | printConfig(file3, k3, v3); 83 | printConfig(file4, k4, v4); 84 | 85 | final org.apache.hadoop.conf.Configuration hbaseConf; 86 | 87 | final Map originalEnv = System.getenv(); 88 | final Map newEnv = new HashMap<>(originalEnv); 89 | newEnv.put("HBASE_CONF_DIR", hbaseConfDir.getAbsolutePath()); 90 | newEnv.put("HBASE_HOME", hbaseHome.getAbsolutePath()); 91 | try { 92 | CommonTestUtils.setEnv(newEnv); 93 | hbaseConf = HBaseConfigurationUtil.getHBaseConfiguration(); 94 | } finally { 95 | CommonTestUtils.setEnv(originalEnv); 96 | } 97 | 98 | // contains extra entries 99 | assertThat(hbaseConf.get(k1, null)).isEqualTo(v1); 100 | assertThat(hbaseConf.get(k2, null)).isEqualTo(v2); 101 | assertThat(hbaseConf.get(k3, null)).isEqualTo(v3); 102 | assertThat(hbaseConf.get(k4, null)).isEqualTo(v4); 103 | 104 | // also contains classpath defaults 105 | assertThat(hbaseConf.get(IN_HBASE_CONFIG_KEY, null)).isEqualTo(IN_HBASE_CONFIG_VALUE); 106 | } 107 | 108 | @Test 109 | void loadOverlappingConfig() throws Exception { 110 | final String k1 = "key1"; 111 | 112 | final String v1 = "from HBASE_HOME/conf"; 113 | final String v2 = "from HBASE_CONF_DIR"; 114 | 115 | final File hbaseHome = tmpDir.resolve("hbaseHome").toFile(); 116 | final File hbaseHomeConf = new File(hbaseHome, "conf"); 117 | 118 | final File hbaseConfDir = tmpDir.resolve("hbaseConfDir").toFile(); 119 | 120 | assertThat(hbaseHomeConf.mkdirs()).isTrue(); 121 | final File file1 = new File(hbaseHomeConf, "hbase-site.xml"); 122 | 123 | Map properties1 = new HashMap<>(); 124 | properties1.put(k1, v1); 125 | printConfigs(file1, properties1); 126 | 127 | // HBASE_CONF_DIR conf will override k1 with v2 128 | assertThat(hbaseConfDir.mkdirs()).isTrue(); 129 | final File file2 = new File(hbaseConfDir, "hbase-site.xml"); 130 | Map properties2 = new HashMap<>(); 131 | properties2.put(k1, v2); 132 | printConfigs(file2, properties2); 133 | 134 | final org.apache.hadoop.conf.Configuration hbaseConf; 135 | 136 | final Map originalEnv = System.getenv(); 137 | final Map newEnv = new HashMap<>(originalEnv); 138 | newEnv.put("HBASE_CONF_DIR", hbaseConfDir.getAbsolutePath()); 139 | newEnv.put("HBASE_HOME", hbaseHome.getAbsolutePath()); 140 | try { 141 | CommonTestUtils.setEnv(newEnv); 142 | hbaseConf = HBaseConfigurationUtil.getHBaseConfiguration(); 143 | } finally { 144 | CommonTestUtils.setEnv(originalEnv); 145 | } 146 | 147 | // contains extra entries 148 | assertThat(hbaseConf.get(k1, null)).isEqualTo(v2); 149 | 150 | // also contains classpath defaults 151 | assertThat(hbaseConf.get(IN_HBASE_CONFIG_KEY, null)).isEqualTo(IN_HBASE_CONFIG_VALUE); 152 | } 153 | 154 | private static void printConfig(File file, String key, String value) throws IOException { 155 | Map map = new HashMap<>(1); 156 | map.put(key, value); 157 | printConfigs(file, map); 158 | } 159 | 160 | private static void printConfigs(File file, Map properties) throws IOException { 161 | try (PrintStream out = new PrintStream(new FileOutputStream(file))) { 162 | out.println(""); 163 | out.println(""); 164 | out.println(""); 165 | for (Map.Entry entry : properties.entrySet()) { 166 | out.println("\t"); 167 | out.println("\t\t" + entry.getKey() + ""); 168 | out.println("\t\t" + entry.getValue() + ""); 169 | out.println("\t"); 170 | } 171 | out.println(""); 172 | } 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/src/test/java/org/apache/flink/connector/hbase/util/HBaseSerdeTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.connector.hbase.util; 20 | 21 | import org.apache.flink.table.api.DataTypes; 22 | import org.apache.flink.table.data.GenericRowData; 23 | import org.apache.flink.table.data.RowData; 24 | import org.apache.flink.table.data.StringData; 25 | import org.apache.flink.table.types.DataType; 26 | 27 | import org.apache.hadoop.hbase.Cell; 28 | import org.apache.hadoop.hbase.HConstants; 29 | import org.apache.hadoop.hbase.KeyValue; 30 | import org.apache.hadoop.hbase.client.Put; 31 | import org.apache.hadoop.hbase.client.Result; 32 | import org.apache.hadoop.hbase.util.Bytes; 33 | import org.junit.jupiter.api.Test; 34 | 35 | import java.util.ArrayList; 36 | import java.util.List; 37 | 38 | import static org.apache.flink.table.api.DataTypes.BIGINT; 39 | import static org.apache.flink.table.api.DataTypes.DOUBLE; 40 | import static org.apache.flink.table.api.DataTypes.FIELD; 41 | import static org.apache.flink.table.api.DataTypes.INT; 42 | import static org.apache.flink.table.api.DataTypes.ROW; 43 | import static org.apache.flink.table.api.DataTypes.STRING; 44 | import static org.assertj.core.api.Assertions.assertThat; 45 | 46 | /** Test for {@link HBaseSerde}. */ 47 | class HBaseSerdeTest { 48 | 49 | private static final String ROW_KEY = "rowkey"; 50 | 51 | private static final String FAMILY1 = "family1"; 52 | private static final String F1COL1 = "col1"; 53 | 54 | private static final String FAMILY2 = "family2"; 55 | private static final String F2COL1 = "col1"; 56 | private static final String F2COL2 = "col2"; 57 | 58 | private static final String FAMILY3 = "family3"; 59 | private static final String F3COL1 = "col1"; 60 | private static final String F3COL2 = "col2"; 61 | private static final String F3COL3 = "col3"; 62 | 63 | @Test 64 | void convertToNewRowTest() { 65 | HBaseSerde serde = createHBaseSerde(false); 66 | List> cellsList = prepareCells(); 67 | List resultRowDatas = new ArrayList<>(); 68 | List resultRowDataStr = new ArrayList<>(); 69 | for (List cells : cellsList) { 70 | RowData row = serde.convertToNewRow(Result.create(cells)); 71 | resultRowDatas.add(row); 72 | resultRowDataStr.add(row.toString()); 73 | } 74 | 75 | assertThat(resultRowDatas.get(0)) 76 | .as("RowData should not be reused") 77 | .isNotSameAs(resultRowDatas.get(1)); 78 | assertThat(resultRowDataStr) 79 | .containsExactly( 80 | "+I(1,+I(10),+I(Hello-1,100),+I(1.01,false,Welt-1))", 81 | "+I(2,+I(20),+I(Hello-2,200),+I(2.02,true,Welt-2))"); 82 | } 83 | 84 | @Test 85 | void convertToReusedRowTest() { 86 | HBaseSerde serde = createHBaseSerde(false); 87 | List> cellsList = prepareCells(); 88 | List resultRowDatas = new ArrayList<>(); 89 | List resultRowDataStr = new ArrayList<>(); 90 | for (List cells : cellsList) { 91 | RowData row = serde.convertToReusedRow(Result.create(cells)); 92 | resultRowDatas.add(row); 93 | resultRowDataStr.add(row.toString()); 94 | } 95 | 96 | assertThat(resultRowDatas.get(0)) 97 | .as("RowData should be reused") 98 | .isSameAs(resultRowDatas.get(1)); 99 | 100 | assertThat(resultRowDataStr) 101 | .containsExactly( 102 | "+I(1,+I(10),+I(Hello-1,100),+I(1.01,false,Welt-1))", 103 | "+I(2,+I(20),+I(Hello-2,200),+I(2.02,true,Welt-2))"); 104 | } 105 | 106 | @Test 107 | public void writeIgnoreNullValueTest() { 108 | HBaseSerde serde = createHBaseSerde(false); 109 | Put m1 = serde.createPutMutation(prepareRowData(), HConstants.LATEST_TIMESTAMP, null); 110 | assert m1 != null; 111 | assertThat(m1.getRow()).isNotEmpty(); 112 | assertThat(m1.get(FAMILY1.getBytes(), F1COL1.getBytes())).isNotEmpty(); 113 | assertThat(m1.get(FAMILY2.getBytes(), F2COL1.getBytes())).isNotEmpty(); 114 | assertThat(m1.get(FAMILY2.getBytes(), F2COL2.getBytes())).isNotEmpty(); 115 | assertThat(m1.get(FAMILY3.getBytes(), F3COL1.getBytes())).isNotEmpty(); 116 | assertThat(m1.get(FAMILY3.getBytes(), F3COL2.getBytes())).isNotEmpty(); 117 | assertThat(m1.get(FAMILY3.getBytes(), F3COL3.getBytes())).isNotEmpty(); 118 | 119 | HBaseSerde writeIgnoreNullValueSerde = createHBaseSerde(true); 120 | Put m2 = 121 | writeIgnoreNullValueSerde.createPutMutation( 122 | prepareRowData(), HConstants.LATEST_TIMESTAMP, null); 123 | assert m2 != null; 124 | assertThat(m2.getRow()).isNotEmpty(); 125 | assertThat(m2.get(FAMILY1.getBytes(), F1COL1.getBytes())).isEmpty(); 126 | assertThat(m2.get(FAMILY2.getBytes(), F2COL1.getBytes())).isNotEmpty(); 127 | assertThat(m2.get(FAMILY2.getBytes(), F2COL2.getBytes())).isEmpty(); 128 | assertThat(m2.get(FAMILY3.getBytes(), F2COL1.getBytes())).isNotEmpty(); 129 | assertThat(m2.get(FAMILY3.getBytes(), F3COL2.getBytes())).isNotEmpty(); 130 | assertThat(m2.get(FAMILY3.getBytes(), F3COL3.getBytes())).isEmpty(); 131 | } 132 | 133 | private HBaseTableSchema createHBaseTableSchema() { 134 | DataType dataType = 135 | ROW( 136 | FIELD(ROW_KEY, INT()), 137 | FIELD(FAMILY1, ROW(FIELD(F1COL1, INT()))), 138 | FIELD(FAMILY2, ROW(FIELD(F2COL1, STRING()), FIELD(F2COL2, BIGINT()))), 139 | FIELD( 140 | FAMILY3, 141 | ROW( 142 | FIELD(F3COL1, DOUBLE()), 143 | FIELD(F3COL2, DataTypes.BOOLEAN()), 144 | FIELD(F3COL3, STRING())))); 145 | return HBaseTableSchema.fromDataType(dataType); 146 | } 147 | 148 | private HBaseSerde createHBaseSerde(boolean writeIgnoreNullValue) { 149 | return new HBaseSerde(createHBaseTableSchema(), "null", writeIgnoreNullValue); 150 | } 151 | 152 | private List> prepareCells() { 153 | List> cellList = new ArrayList<>(); 154 | byte[] fam1 = Bytes.toBytes(FAMILY1); 155 | byte[] f1c1 = Bytes.toBytes(F1COL1); 156 | 157 | byte[] fam2 = Bytes.toBytes(FAMILY2); 158 | byte[] f2c1 = Bytes.toBytes(F2COL1); 159 | byte[] f2c2 = Bytes.toBytes(F2COL2); 160 | 161 | byte[] fam3 = Bytes.toBytes(FAMILY3); 162 | byte[] f3c1 = Bytes.toBytes(F3COL1); 163 | byte[] f3c2 = Bytes.toBytes(F3COL2); 164 | byte[] f3c3 = Bytes.toBytes(F3COL3); 165 | 166 | byte[] row1 = Bytes.toBytes(1); 167 | byte[] row2 = Bytes.toBytes(2); 168 | 169 | Cell kv111 = new KeyValue(row1, fam1, f1c1, Bytes.toBytes(10)); 170 | Cell kv121 = new KeyValue(row1, fam2, f2c1, Bytes.toBytes("Hello-1")); 171 | Cell kv122 = new KeyValue(row1, fam2, f2c2, Bytes.toBytes(100L)); 172 | Cell kv131 = new KeyValue(row1, fam3, f3c1, Bytes.toBytes(1.01)); 173 | Cell kv132 = new KeyValue(row1, fam3, f3c2, Bytes.toBytes(false)); 174 | Cell kv133 = new KeyValue(row1, fam3, f3c3, Bytes.toBytes("Welt-1")); 175 | 176 | Cell kv211 = new KeyValue(row2, fam1, f1c1, Bytes.toBytes(20)); 177 | Cell kv221 = new KeyValue(row2, fam2, f2c1, Bytes.toBytes("Hello-2")); 178 | Cell kv222 = new KeyValue(row2, fam2, f2c2, Bytes.toBytes(200L)); 179 | Cell kv231 = new KeyValue(row2, fam3, f3c1, Bytes.toBytes(2.02)); 180 | Cell kv232 = new KeyValue(row2, fam3, f3c2, Bytes.toBytes(true)); 181 | Cell kv233 = new KeyValue(row2, fam3, f3c3, Bytes.toBytes("Welt-2")); 182 | List cells1 = new ArrayList<>(); 183 | cells1.add(kv111); 184 | cells1.add(kv121); 185 | cells1.add(kv122); 186 | cells1.add(kv131); 187 | cells1.add(kv132); 188 | cells1.add(kv133); 189 | List cells2 = new ArrayList<>(); 190 | cells2.add(kv211); 191 | cells2.add(kv221); 192 | cells2.add(kv222); 193 | cells2.add(kv231); 194 | cells2.add(kv232); 195 | cells2.add(kv233); 196 | cellList.add(cells1); 197 | cellList.add(cells2); 198 | return cellList; 199 | } 200 | 201 | private RowData prepareRowData() { 202 | GenericRowData fam1Row = new GenericRowData(1); 203 | fam1Row.setField(0, null); 204 | 205 | GenericRowData fam2Row = new GenericRowData(2); 206 | fam2Row.setField(0, StringData.fromString("Hello-1")); 207 | fam2Row.setField(1, null); 208 | 209 | GenericRowData fam3Row = new GenericRowData(3); 210 | fam3Row.setField(0, 2.02); 211 | fam3Row.setField(1, true); 212 | fam3Row.setField(2, null); 213 | 214 | GenericRowData row = new GenericRowData(4); 215 | row.setField(0, 10); 216 | row.setField(1, fam1Row); 217 | row.setField(2, fam2Row); 218 | row.setField(3, fam3Row); 219 | return row; 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | org.apache.flink.util.TestLoggerExtension -------------------------------------------------------------------------------- /flink-connector-hbase-base/src/test/resources/archunit.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | # By default we allow removing existing violations, but fail when new violations are added. 20 | freeze.store.default.allowStoreUpdate=true 21 | 22 | # Enable this if a new (frozen) rule has been added in order to create the initial store and record the existing violations. 23 | #freeze.store.default.allowStoreCreation=true 24 | 25 | # Enable this to add allow new violations to be recorded. 26 | # NOTE: Adding new violations should be avoided when possible. If the rule was correct to flag a new 27 | # violation, please try to avoid creating the violation. If the violation was created due to a 28 | # shortcoming of the rule, file a JIRA issue so the rule can be improved. 29 | #freeze.refreeze=true 30 | 31 | freeze.store.default.path=archunit-violations 32 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/src/test/resources/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 20 | 21 | 23 | 24 | 25 | 26 | hbase_conf_key 27 | hbase_conf_value! 28 | 29 | 30 | -------------------------------------------------------------------------------- /flink-connector-hbase-base/src/test/resources/log4j2-test.properties: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | ################################################################################ 18 | 19 | # Set root logger level to OFF to not flood build logs 20 | # set manually to INFO for debugging purposes 21 | rootLogger.level = OFF 22 | rootLogger.appenderRef.test.ref = TestLogger 23 | 24 | appender.testlogger.name = TestLogger 25 | appender.testlogger.type = CONSOLE 26 | appender.testlogger.target = SYSTEM_ERR 27 | appender.testlogger.layout.type = PatternLayout 28 | appender.testlogger.layout.pattern = %-4r [%t] %-5p %c %x - %m%n 29 | -------------------------------------------------------------------------------- /flink-connector-hbase-e2e-tests/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 20 | 23 | 24 | 4.0.0 25 | 26 | 27 | org.apache.flink 28 | flink-connector-hbase-parent 29 | 4.0-SNAPSHOT 30 | 31 | 32 | flink-connector-hbase-e2e-tests 33 | Flink : Connectors : HBase : E2E Tests 34 | jar 35 | 36 | 37 | 38 | org.apache.flink 39 | flink-connector-test-utils 40 | ${flink.version} 41 | 42 | 43 | 44 | org.apache.flink 45 | flink-test-utils 46 | ${flink.version} 47 | 48 | 49 | 50 | org.apache.flink 51 | flink-test-utils-junit 52 | ${flink.version} 53 | 54 | 55 | 56 | org.junit.jupiter 57 | junit-jupiter 58 | 59 | 60 | 61 | org.apache.flink 62 | flink-connector-hbase-2.2 63 | ${project.version} 64 | test 65 | 66 | 67 | 68 | 69 | org.apache.hadoop 70 | hadoop-common 71 | test 72 | 73 | 74 | net.minidev 75 | json-smart 76 | 77 | 78 | 79 | 80 | 81 | org.apache.hadoop 82 | hadoop-yarn-client 83 | ${hadoop.version} 84 | test 85 | 86 | 87 | log4j 88 | log4j 89 | 90 | 91 | 92 | jdk.tools 93 | jdk.tools 94 | 95 | 96 | ch.qos.reload4j 97 | reload4j 98 | 99 | 100 | org.slf4j 101 | slf4j-reload4j 102 | 103 | 104 | 105 | 106 | 107 | org.apache.hadoop 108 | hadoop-yarn-api 109 | ${hadoop.version} 110 | test 111 | 112 | 113 | 114 | jdk.tools 115 | jdk.tools 116 | 117 | 118 | ch.qos.reload4j 119 | reload4j 120 | 121 | 122 | org.slf4j 123 | slf4j-reload4j 124 | 125 | 126 | 127 | 128 | 129 | org.apache.hadoop 130 | hadoop-minicluster 131 | ${hadoop.version} 132 | test 133 | 134 | 135 | net.minidev 136 | json-smart 137 | 138 | 139 | 140 | 141 | 142 | org.apache.hadoop 143 | hadoop-minikdc 144 | ${minikdc.version} 145 | test 146 | 147 | 148 | org.slf4j 149 | slf4j-log4j12 150 | 151 | 152 | org.slf4j 153 | slf4j-reload4j 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | org.apache.maven.plugins 163 | maven-enforcer-plugin 164 | 165 | 166 | dependency-convergence 167 | 168 | enforce 169 | 170 | 171 | true 172 | 173 | 174 | 175 | 176 | 177 | org.apache.maven.plugins 178 | maven-dependency-plugin 179 | 180 | 181 | copy 182 | package 183 | 184 | copy 185 | 186 | 187 | 188 | 189 | org.apache.flink 190 | flink-sql-connector-hbase-2.2 191 | ${project.version} 192 | sql-hbase-2.2.jar 193 | jar 194 | ${project.build.directory}/dependencies 195 | 196 | 197 | 198 | 199 | 200 | store-classpath-in-target-for-tests 201 | package 202 | 203 | build-classpath 204 | 205 | 206 | ${project.build.directory}/hadoop.classpath 207 | org.apache.flink 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | -------------------------------------------------------------------------------- /flink-connector-hbase-e2e-tests/src/main/java/org/apache/flink/streaming/tests/HBaseContainer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.streaming.tests; 20 | 21 | import com.github.dockerjava.api.command.InspectContainerResponse; 22 | import org.testcontainers.containers.Container; 23 | import org.testcontainers.containers.GenericContainer; 24 | import org.testcontainers.images.builder.ImageFromDockerfile; 25 | 26 | import java.util.Arrays; 27 | import java.util.stream.Collectors; 28 | 29 | /** Standalone containerized HBase instance that builds the image on the fly. */ 30 | public class HBaseContainer extends GenericContainer { 31 | 32 | private static final String HBASE_BIN = "/opt/hbase/bin"; 33 | private static final int MAX_RETRIES = 3; 34 | 35 | public HBaseContainer(String hbaseVersion) { 36 | super(getImageFromDockerfile(hbaseVersion)); 37 | } 38 | 39 | private static ImageFromDockerfile getImageFromDockerfile(String hbaseVersion) { 40 | return new ImageFromDockerfile() 41 | .withDockerfileFromBuilder( 42 | builder -> 43 | builder.from("adoptopenjdk/openjdk8") 44 | .env("HBASE_VERSION", hbaseVersion) 45 | .run( 46 | "export INITRD=no" 47 | + " && export HBASE_DIST=\"http://archive.apache.org/dist/hbase\"" 48 | + " && apt-get update -y" 49 | + " && apt-get install -y --no-install-recommends curl" 50 | + " && cd /opt" 51 | + " && curl -SL $HBASE_DIST/$HBASE_VERSION/hbase-$HBASE_VERSION-bin.tar.gz" 52 | + " | tar -x -z && mv hbase-${HBASE_VERSION} hbase") 53 | .expose(2181) 54 | .cmd( 55 | "/bin/sh", 56 | "-c", 57 | String.format( 58 | "nohup %s/start-hbase.sh & sleep infinity", 59 | HBASE_BIN))); 60 | } 61 | 62 | @Override 63 | protected void containerIsStarted(InspectContainerResponse containerInfo) { 64 | ExecResult res = null; 65 | for (int i = 0; i < MAX_RETRIES; i++) { 66 | try { 67 | res = execCmd("scan 'hbase:meta'"); 68 | if (res.getStdout().contains("hbase:namespace")) { 69 | return; 70 | } 71 | Thread.sleep(5000L); 72 | } catch (Exception e) { 73 | throw new RuntimeException("Failed to verify if container is started.", e); 74 | } 75 | } 76 | throw new IllegalStateException("Failed to start HBase properly:\n" + res); 77 | } 78 | 79 | public Container.ExecResult createTable(String table, String... colFamilies) throws Exception { 80 | String createCmd = 81 | String.format("create '%s',", table) 82 | + Arrays.stream(colFamilies) 83 | .map(cf -> String.format("{NAME=>'%s'}", cf)) 84 | .collect(Collectors.joining(",")); 85 | 86 | return execCmd(createCmd); 87 | } 88 | 89 | public Container.ExecResult putData( 90 | String table, String rowKey, String colFamily, String colQualifier, String val) 91 | throws Exception { 92 | String putCmd = 93 | String.format( 94 | "put '%s','%s','%s:%s','%s'", table, rowKey, colFamily, colQualifier, val); 95 | 96 | return execCmd(putCmd); 97 | } 98 | 99 | public Container.ExecResult scanTable(String table) throws Exception { 100 | String scanCmd = String.format("scan '%s'", table); 101 | 102 | return execCmd(scanCmd); 103 | } 104 | 105 | private Container.ExecResult execCmd(String cmd) throws Exception { 106 | String hbaseShellCmd = String.format("echo \"%s\" | %s/hbase shell", cmd, HBASE_BIN); 107 | 108 | return execInContainer("sh", "-c", hbaseShellCmd); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /flink-connector-hbase-e2e-tests/src/test/java/org/apache/flink/streaming/tests/HBaseITCase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flink.streaming.tests; 20 | 21 | import org.apache.flink.connector.testframe.container.FlinkContainers; 22 | import org.apache.flink.connector.testframe.container.TestcontainersSettings; 23 | import org.apache.flink.test.resources.ResourceTestUtils; 24 | import org.apache.flink.test.util.SQLJobSubmission; 25 | import org.apache.flink.util.FileUtils; 26 | 27 | import org.apache.commons.io.IOUtils; 28 | import org.junit.jupiter.api.AfterEach; 29 | import org.junit.jupiter.api.BeforeEach; 30 | import org.junit.jupiter.api.Test; 31 | import org.testcontainers.containers.Container; 32 | import org.testcontainers.containers.Network; 33 | 34 | import java.io.File; 35 | import java.io.FileNotFoundException; 36 | import java.io.IOException; 37 | import java.io.InputStream; 38 | import java.nio.charset.StandardCharsets; 39 | import java.nio.file.Path; 40 | import java.nio.file.Paths; 41 | import java.util.Arrays; 42 | import java.util.List; 43 | import java.util.stream.Collectors; 44 | 45 | import static org.junit.jupiter.api.Assertions.assertEquals; 46 | import static org.testcontainers.shaded.org.hamcrest.MatcherAssert.assertThat; 47 | import static org.testcontainers.shaded.org.hamcrest.Matchers.allOf; 48 | import static org.testcontainers.shaded.org.hamcrest.Matchers.containsInAnyOrder; 49 | import static org.testcontainers.shaded.org.hamcrest.Matchers.containsString; 50 | 51 | /** End to end HBase connector tests. */ 52 | class HBaseITCase { 53 | 54 | private static final String HBASE_VERSION = "2.2.3"; 55 | private static final String CONNECTOR_VERSION = "hbase-2.2"; 56 | private static final String HBASE_E2E_SQL = "hbase_e2e.sql"; 57 | private static final Path HADOOP_CP = ResourceTestUtils.getResource(".*hadoop.classpath"); 58 | private static final Network NETWORK = Network.newNetwork(); 59 | 60 | private HBaseContainer hbase; 61 | private FlinkContainers flink; 62 | private List hadoopCpJars; 63 | private Path connectorJar; 64 | 65 | @BeforeEach 66 | void start() throws Exception { 67 | // Prepare all hadoop jars to mock HADOOP_CLASSPATH, use hadoop.classpath which contains all 68 | // hadoop jars 69 | File hadoopClasspathFile = new File(HADOOP_CP.toAbsolutePath().toString()); 70 | 71 | if (!hadoopClasspathFile.exists()) { 72 | throw new FileNotFoundException( 73 | "File that contains hadoop classpath " + HADOOP_CP + " does not exist."); 74 | } 75 | 76 | String classPathContent = FileUtils.readFileUtf8(hadoopClasspathFile); 77 | hadoopCpJars = 78 | Arrays.stream(classPathContent.split(":")) 79 | .map(Paths::get) 80 | .collect(Collectors.toList()); 81 | } 82 | 83 | @AfterEach 84 | void stop() { 85 | flink.stop(); 86 | hbase.stop(); 87 | } 88 | 89 | @Test 90 | void test() throws Exception { 91 | hbase = new HBaseContainer(HBASE_VERSION).withNetwork(NETWORK).withNetworkAliases("hbase"); 92 | 93 | flink = 94 | FlinkContainers.builder() 95 | .withTestcontainersSettings( 96 | TestcontainersSettings.builder() 97 | .network(NETWORK) 98 | .dependsOn(hbase) 99 | .build()) 100 | .build(); 101 | 102 | connectorJar = ResourceTestUtils.getResource("sql-" + CONNECTOR_VERSION + ".jar"); 103 | 104 | hbase.start(); 105 | flink.start(); 106 | 107 | hbase.createTable("source", "family1", "family2"); 108 | hbase.createTable("sink", "family1", "family2"); 109 | 110 | hbase.putData("source", "row1", "family1", "f1c1", "v1"); 111 | hbase.putData("source", "row1", "family2", "f2c1", "v2"); 112 | hbase.putData("source", "row1", "family2", "f2c2", "v3"); 113 | hbase.putData("source", "row2", "family1", "f1c1", "v4"); 114 | hbase.putData("source", "row2", "family2", "f2c1", "v5"); 115 | hbase.putData("source", "row2", "family2", "f2c2", "v6"); 116 | 117 | SQLJobSubmission jobSubmission = initSqlJobSubmission(); 118 | flink.submitSQLJob(jobSubmission); 119 | List valueLines = getSinkResult(); 120 | 121 | assertEquals(6, valueLines.size()); 122 | 123 | assertThat( 124 | valueLines, 125 | containsInAnyOrder( 126 | allOf( 127 | containsString("row1"), 128 | containsString("family1"), 129 | containsString("f1c1"), 130 | containsString("value1")), 131 | allOf( 132 | containsString("row1"), 133 | containsString("family2"), 134 | containsString("f2c1"), 135 | containsString("v2")), 136 | allOf( 137 | containsString("row1"), 138 | containsString("family2"), 139 | containsString("f2c2"), 140 | containsString("v3")), 141 | allOf( 142 | containsString("row2"), 143 | containsString("family1"), 144 | containsString("f1c1"), 145 | containsString("value4")), 146 | allOf( 147 | containsString("row2"), 148 | containsString("family2"), 149 | containsString("f2c1"), 150 | containsString("v5")), 151 | allOf( 152 | containsString("row2"), 153 | containsString("family2"), 154 | containsString("f2c2"), 155 | containsString("v6")))); 156 | } 157 | 158 | private SQLJobSubmission initSqlJobSubmission() throws IOException { 159 | List sqlLines = loadSqlStatements(); 160 | return new SQLJobSubmission.SQLJobSubmissionBuilder(sqlLines) 161 | .addJar(connectorJar) 162 | .addJars(hadoopCpJars) 163 | .build(); 164 | } 165 | 166 | private List getSinkResult() throws Exception { 167 | Container.ExecResult res = hbase.scanTable("sink"); 168 | assertEquals(0, res.getExitCode()); 169 | 170 | return Arrays.stream(res.getStdout().split("\n")) 171 | .filter(line -> line.contains("value=")) 172 | .collect(Collectors.toList()); 173 | } 174 | 175 | private static List loadSqlStatements() throws IOException { 176 | try (InputStream is = 177 | HBaseITCase.class.getClassLoader().getResourceAsStream(HBASE_E2E_SQL)) { 178 | if (is == null) { 179 | throw new FileNotFoundException(HBASE_E2E_SQL); 180 | } 181 | 182 | List lines = IOUtils.readLines(is, StandardCharsets.UTF_8); 183 | 184 | return lines.stream() 185 | .map(line -> line.replace("$HBASE_CONNECTOR", CONNECTOR_VERSION)) 186 | .collect(Collectors.toList()); 187 | } 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /flink-connector-hbase-e2e-tests/src/test/resources/hbase_e2e.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | CREATE TABLE MyHBaseSource ( 18 | rowkey STRING, 19 | family1 ROW, 20 | family2 ROW 21 | ) WITH ( 22 | 'connector' = '$HBASE_CONNECTOR', 23 | 'table-name' = 'source', 24 | 'zookeeper.quorum' = 'hbase:2181' 25 | ); 26 | 27 | CREATE TABLE MyHBaseSink ( 28 | rowkey STRING, 29 | family1 ROW, 30 | family2 ROW 31 | ) WITH ( 32 | 'connector' = '$HBASE_CONNECTOR', 33 | 'table-name' = 'sink', 34 | 'zookeeper.quorum' = 'hbase:2181', 35 | 'sink.buffer-flush.max-rows' = '1', 36 | 'sink.buffer-flush.interval' = '2s' 37 | ); 38 | 39 | INSERT INTO MyHBaseSink 40 | SELECT 41 | rowkey, 42 | ROW(a), 43 | ROW(b, c) 44 | FROM ( 45 | SELECT 46 | rowkey, 47 | REGEXP_REPLACE(family1.f1c1, 'v', 'value') as a, 48 | family2.f2c1 as b, 49 | family2.f2c2 as c 50 | FROM MyHBaseSource) 51 | source; 52 | -------------------------------------------------------------------------------- /flink-connector-hbase-e2e-tests/src/test/resources/log4j2-test.properties: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | ################################################################################ 18 | 19 | # Set root logger level to OFF to not flood build logs 20 | # set manually to INFO for debugging purposes 21 | rootLogger.level = OFF 22 | rootLogger.appenderRef.test.ref = TestLogger 23 | 24 | appender.testlogger.name = TestLogger 25 | appender.testlogger.type = CONSOLE 26 | appender.testlogger.target = SYSTEM_ERR 27 | appender.testlogger.layout.type = PatternLayout 28 | appender.testlogger.layout.pattern = %-4r [%t] %-5p %c %x - %m%n 29 | -------------------------------------------------------------------------------- /flink-sql-connector-hbase-2.2/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 20 | 23 | 24 | 4.0.0 25 | 26 | 27 | org.apache.flink 28 | flink-connector-hbase-parent 29 | 4.0-SNAPSHOT 30 | 31 | 32 | flink-sql-connector-hbase-2.2 33 | Flink : Connectors : SQL : HBase 2.2 34 | 35 | jar 36 | 37 | 38 | 39 | org.apache.flink 40 | flink-connector-hbase-2.2 41 | ${project.version} 42 | 43 | 44 | 45 | 46 | 47 | 48 | org.apache.maven.plugins 49 | maven-shade-plugin 50 | 51 | 52 | shade-flink 53 | package 54 | 55 | shade 56 | 57 | 58 | 63 | 64 | 66 | hbase-default.xml 67 | hbase-default.xml 68 | 69 | 70 | 71 | 72 | org.apache.flink:flink-connector-base 73 | org.apache.flink:flink-connector-hbase-base 74 | org.apache.flink:flink-connector-hbase-2.2 75 | org.apache.hbase:hbase-* 76 | org.apache.hbase.thirdparty:hbase-shaded-* 77 | org.apache.zookeeper:zookeeper 78 | org.apache.htrace:htrace-core4 79 | com.google.protobuf:protobuf-java 80 | commons-codec:commons-codec 81 | org.apache.commons:commons-crypto 82 | org.apache.commons:commons-lang3 83 | io.netty:* 84 | io.dropwizard.metrics:metrics-core 85 | 86 | 87 | org.apache.hbase:hbase-metrics* 88 | org.apache.hbase:hbase-server* 89 | org.apache.hbase:hbase-hadoop*-compat 90 | 91 | 92 | 93 | 94 | *:* 95 | 96 | 97 | META-INF/services/com.fasterxml.** 98 | META-INF/services/org.apache.hadoop.** 99 | META-INF/services/javax.** 100 | digesterRules.xml 101 | properties.dtd 102 | PropertyList-1.0.dtd 103 | LICENSE.txt 104 | *.proto 105 | protobuf/* 106 | 107 | 108 | 109 | 110 | 111 | 112 | org.apache.zookeeper 113 | org.apache.flink.hbase.shaded.org.apache.zookeeper 114 | 115 | 116 | org.apache.htrace 117 | org.apache.flink.hbase.shaded.org.apache.htrace 118 | 119 | 120 | com.google 121 | org.apache.flink.hbase.shaded.com.google 122 | 123 | 124 | com.yammer.metrics 125 | org.apache.flink.hbase.shaded.com.yammer.metrics 126 | 127 | 128 | org.apache.commons 129 | org.apache.flink.hbase.shaded.org.apache.commons 130 | 131 | 132 | org.apache.jute 133 | org.apache.flink.hbase.shaded.org.apache.jute 134 | 135 | 136 | io.netty 137 | org.apache.flink.hbase.shaded.io.netty 138 | 139 | 140 | org.apache.hadoop.hbase 141 | org.apache.flink.hbase.shaded.org.apache.hadoop.hbase 142 | 147 | 148 | org.apache.hadoop.hbase.codec.* 149 | org.apache.hadoop.hbase.NotServingRegionException 150 | org.apache.hadoop.hbase.exceptions.RegionMovedException 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | -------------------------------------------------------------------------------- /flink-sql-connector-hbase-2.2/src/main/resources/META-INF/NOTICE: -------------------------------------------------------------------------------- 1 | flink-sql-connector-hbase-2.2 2 | Copyright 2014-2024 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | 7 | This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) 8 | 9 | - commons-codec:commons-codec:1.15 10 | - io.netty:netty-all:4.1.70.Final 11 | - io.netty:netty-buffer:4.1.70.Final 12 | - io.netty:netty-codec:4.1.70.Final 13 | - io.netty:netty-codec-dns:4.1.70.Final 14 | - io.netty:netty-codec-haproxy:4.1.70.Final 15 | - io.netty:netty-codec-http:4.1.70.Final 16 | - io.netty:netty-codec-http2:4.1.70.Final 17 | - io.netty:netty-codec-memcache:4.1.70.Final 18 | - io.netty:netty-codec-mqtt:4.1.70.Final 19 | - io.netty:netty-codec-redis:4.1.70.Final 20 | - io.netty:netty-codec-smtp:4.1.70.Final 21 | - io.netty:netty-codec-socks:4.1.70.Final 22 | - io.netty:netty-codec-stomp:4.1.70.Final 23 | - io.netty:netty-codec-xml:4.1.70.Final 24 | - io.netty:netty-common:4.1.70.Final 25 | - io.netty:netty-handler:4.1.70.Final 26 | - io.netty:netty-handler-proxy:4.1.70.Final 27 | - io.netty:netty-resolver:4.1.70.Final 28 | - io.netty:netty-resolver-dns:4.1.70.Final 29 | - io.netty:netty-resolver-dns-classes-macos:4.1.70.Final 30 | - io.netty:netty-resolver-dns-native-macos:osx-aarch_64:4.1.70.Final 31 | - io.netty:netty-resolver-dns-native-macos:osx-x86_64:4.1.70.Final 32 | - io.netty:netty-transport:4.1.70.Final 33 | - io.netty:netty-transport-classes-epoll:4.1.70.Final 34 | - io.netty:netty-transport-classes-kqueue:4.1.70.Final 35 | - io.netty:netty-transport-native-unix-common:4.1.70.Final 36 | - io.netty:netty-transport-native-epoll:linux-aarch_64:4.1.70.Final 37 | - io.netty:netty-transport-native-epoll:linux-x86_64:4.1.70.Final 38 | - io.netty:netty-transport-native-kqueue:osx-aarch_64:4.1.70.Final 39 | - io.netty:netty-transport-native-kqueue:osx-x86_64:4.1.70.Final 40 | - io.netty:netty-transport-rxtx:4.1.70.Final 41 | - io.netty:netty-transport-sctp:4.1.70.Final 42 | - io.netty:netty-transport-udt:4.1.70.Final 43 | - io.dropwizard.metrics:metrics-core:3.2.6 44 | - org.apache.commons:commons-crypto:1.0.0 45 | - org.apache.commons:commons-lang3:3.3.2 46 | - org.apache.hbase:hbase-client:2.2.3 47 | - org.apache.hbase:hbase-common:2.2.3 48 | - org.apache.hbase:hbase-protocol:2.2.3 49 | - org.apache.hbase:hbase-protocol-shaded:2.2.3 50 | - org.apache.hbase.thirdparty:hbase-shaded-protobuf:2.2.1 51 | - org.apache.hbase.thirdparty:hbase-shaded-miscellaneous:2.2.1 52 | - org.apache.hbase.thirdparty:hbase-shaded-netty:2.2.1 53 | - org.apache.htrace:htrace-core4:4.2.0-incubating 54 | - org.apache.zookeeper:zookeeper:3.4.14 55 | 56 | This project bundles the following dependencies under the BSD license. 57 | See bundled license files for details. 58 | 59 | - com.google.protobuf:protobuf-java:2.5.0 60 | 61 | The bundled Apache HTrace org.apache.htrace:htrace-core4 dependency bundles the following dependencies under 62 | the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) 63 | 64 | - com.fasterxml.jackson.core:jackson-annotations:2.4.0 65 | - com.fasterxml.jackson.core:jackson-core:2.4.0 66 | - com.fasterxml.jackson.core:jackson-databind:2.4.0 67 | - commons-logging:commons-logging:1.1.1 68 | 69 | The bundled Apache HBase Patched & Relocated (Shaded) Protobuf org.apache.hbase.thirdparty:hbase-shaded-protobuf 70 | dependency bundles the following dependencies under the BSD license. See bundled license files for details. 71 | 72 | - com.google.protobuf:protobuf-java:3.7.1 73 | 74 | The bundled Apache HBase Relocated (Shaded) Third-party Miscellaneous Libs 75 | org.apache.hbase.thirdparty:hbase-shaded-miscellaneous dependency bundles the following dependencies under 76 | the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) 77 | 78 | - com.google.code.gson:gson:2.8.5 79 | - com.google.guava:failureaccess:1.0.1 80 | - com.google.guava:guava:27.1-jre 81 | - commons-cli:commons-cli:1.4 82 | - org.apache.commons:commons-collections4:4.3 83 | 84 | The bundled Apache HBase Relocated (Shaded) Third-party Miscellaneous Libs 85 | org.apache.hbase.thirdparty:hbase-shaded-miscellaneous dependency bundles the following dependencies under 86 | the BSD license. See bundled license files for details. 87 | 88 | - com.google.protobuf:protobuf-java-util:3.7.1 89 | 90 | The bundled Apache HBase Relocated (Shaded) Netty Libs org.apache.hbase.thirdparty:hbase-shaded-netty 91 | dependency bundles the following dependencies under 92 | the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) 93 | 94 | - io.netty:netty-all:4.1.34.Final 95 | -------------------------------------------------------------------------------- /flink-sql-connector-hbase-2.2/src/main/resources/META-INF/licenses/LICENSE.protobuf: -------------------------------------------------------------------------------- 1 | Copyright 2008 Google Inc. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Code generated by the Protocol Buffer compiler is owned by the owner 30 | of the input file used when generating it. This code is not 31 | standalone and requires a support library to be linked with it. This 32 | support library is itself covered by the above license. 33 | -------------------------------------------------------------------------------- /tools/ci/log4j.properties: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | ################################################################################ 18 | 19 | rootLogger.level = INFO 20 | rootLogger.appenderRef.out.ref = ConsoleAppender 21 | 22 | # ----------------------------------------------------------------------------- 23 | # Console (use 'console') 24 | # ----------------------------------------------------------------------------- 25 | 26 | appender.console.name = ConsoleAppender 27 | appender.console.type = CONSOLE 28 | appender.console.layout.type = PatternLayout 29 | appender.console.layout.pattern = %d{HH:mm:ss,SSS} [%20t] %-5p %-60c %x - %m%n 30 | 31 | # ----------------------------------------------------------------------------- 32 | # File (use 'file') 33 | # ----------------------------------------------------------------------------- 34 | appender.file.name = FileAppender 35 | appender.file.type = FILE 36 | appender.file.fileName = ${sys:log.dir}/mvn-${sys:mvn.forkNumber:-output}.log 37 | appender.file.layout.type = PatternLayout 38 | appender.file.layout.pattern = %d{HH:mm:ss,SSS} [%20t] %-5p %-60c %x - %m%n 39 | appender.file.createOnDemand = true 40 | 41 | # suppress the irrelevant (wrong) warnings from the netty channel handler 42 | logger.netty.name = org.jboss.netty.channel.DefaultChannelPipeline 43 | logger.netty.level = ERROR 44 | -------------------------------------------------------------------------------- /tools/maven/suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 20 | 21 | 24 | 25 | 26 | 27 | --------------------------------------------------------------------------------