├── .github ├── pull_request_template.md └── workflows │ ├── build.yml │ ├── prepare_release_branch.yml │ └── publish.yml ├── .gitignore ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── dev ├── README.md ├── checkstyle-suppressions.xml └── checkstyle.xml ├── docs └── JoinTable.PNG ├── lombok.config ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── getindata │ │ └── connectors │ │ └── http │ │ ├── HttpPostRequestCallback.java │ │ ├── HttpPostRequestCallbackFactory.java │ │ ├── HttpSink.java │ │ ├── HttpSinkBuilder.java │ │ ├── HttpStatusCodeValidationFailedException.java │ │ ├── LookupArg.java │ │ ├── LookupQueryCreator.java │ │ ├── LookupQueryCreatorFactory.java │ │ ├── SchemaLifecycleAwareElementConverter.java │ │ └── internal │ │ ├── BasicAuthHeaderValuePreprocessor.java │ │ ├── ComposeHeaderPreprocessor.java │ │ ├── HeaderPreprocessor.java │ │ ├── HeaderValuePreprocessor.java │ │ ├── OIDCAuthHeaderValuePreprocessor.java │ │ ├── PollingClient.java │ │ ├── PollingClientFactory.java │ │ ├── SinkHttpClient.java │ │ ├── SinkHttpClientBuilder.java │ │ ├── SinkHttpClientResponse.java │ │ ├── auth │ │ └── OidcAccessTokenManager.java │ │ ├── config │ │ ├── ConfigException.java │ │ ├── HttpConnectorConfigConstants.java │ │ └── SinkRequestSubmitMode.java │ │ ├── retry │ │ ├── HttpClientWithRetry.java │ │ ├── RetryConfigProvider.java │ │ └── RetryStrategyType.java │ │ ├── security │ │ ├── SecurityContext.java │ │ └── SelfSignedTrustManager.java │ │ ├── sink │ │ ├── HttpSinkInternal.java │ │ ├── HttpSinkRequestEntry.java │ │ ├── HttpSinkWriter.java │ │ ├── HttpSinkWriterStateSerializer.java │ │ └── httpclient │ │ │ ├── AbstractRequestSubmitter.java │ │ │ ├── BatchRequestSubmitter.java │ │ │ ├── BatchRequestSubmitterFactory.java │ │ │ ├── HttpRequest.java │ │ │ ├── JavaNetHttpResponseWrapper.java │ │ │ ├── JavaNetSinkHttpClient.java │ │ │ ├── PerRequestRequestSubmitterFactory.java │ │ │ ├── PerRequestSubmitter.java │ │ │ ├── RequestSubmitter.java │ │ │ └── RequestSubmitterFactory.java │ │ ├── status │ │ ├── ComposeHttpStatusCodeChecker.java │ │ ├── HttpCodesParser.java │ │ ├── HttpResponseChecker.java │ │ ├── HttpResponseCodeType.java │ │ ├── HttpStatusCodeChecker.java │ │ ├── SingleValueHttpStatusCodeChecker.java │ │ ├── TypeStatusCodeChecker.java │ │ └── WhiteListHttpStatusCodeChecker.java │ │ ├── table │ │ ├── SerializationSchemaElementConverter.java │ │ ├── lookup │ │ │ ├── AsyncHttpTableLookupFunction.java │ │ │ ├── BodyBasedRequestFactory.java │ │ │ ├── GetRequestFactory.java │ │ │ ├── HttpLookupConfig.java │ │ │ ├── HttpLookupConnectorOptions.java │ │ │ ├── HttpLookupSourceRequestEntry.java │ │ │ ├── HttpLookupTableSource.java │ │ │ ├── HttpLookupTableSourceFactory.java │ │ │ ├── HttpRequestFactory.java │ │ │ ├── HttpTableLookupFunction.java │ │ │ ├── JavaNetHttpPollingClient.java │ │ │ ├── JavaNetHttpPollingClientFactory.java │ │ │ ├── LookupQueryInfo.java │ │ │ ├── LookupRow.java │ │ │ ├── LookupSchemaEntry.java │ │ │ ├── RequestFactoryBase.java │ │ │ ├── RowDataLookupSchemaEntryBase.java │ │ │ ├── RowDataSingleValueLookupSchemaEntry.java │ │ │ ├── RowTypeLookupSchemaEntry.java │ │ │ ├── Slf4JHttpLookupPostRequestCallback.java │ │ │ ├── Slf4jHttpLookupPostRequestCallbackFactory.java │ │ │ ├── TableSourceHelper.java │ │ │ └── querycreators │ │ │ │ ├── ElasticSearchLiteQueryCreator.java │ │ │ │ ├── ElasticSearchLiteQueryCreatorFactory.java │ │ │ │ ├── GenericGetQueryCreator.java │ │ │ │ ├── GenericGetQueryCreatorFactory.java │ │ │ │ ├── GenericJsonAndUrlQueryCreator.java │ │ │ │ ├── GenericJsonAndUrlQueryCreatorFactory.java │ │ │ │ ├── GenericJsonQueryCreator.java │ │ │ │ ├── GenericJsonQueryCreatorFactory.java │ │ │ │ ├── ObjectMapperAdapter.java │ │ │ │ ├── PrefixedConfigOption.java │ │ │ │ └── QueryFormatAwareConfiguration.java │ │ └── sink │ │ │ ├── HttpDynamicSink.java │ │ │ ├── HttpDynamicSinkConnectorOptions.java │ │ │ ├── HttpDynamicTableSinkFactory.java │ │ │ ├── Slf4jHttpPostRequestCallback.java │ │ │ └── Slf4jHttpPostRequestCallbackFactory.java │ │ └── utils │ │ ├── ConfigUtils.java │ │ ├── ExceptionUtils.java │ │ ├── HttpHeaderUtils.java │ │ ├── JavaNetHttpClientFactory.java │ │ ├── ProxyConfig.java │ │ ├── SerializationSchemaUtils.java │ │ ├── SynchronizedSerializationSchema.java │ │ ├── ThreadUtils.java │ │ └── uri │ │ ├── CharArrayBuffer.java │ │ ├── NameValuePair.java │ │ ├── ParserCursor.java │ │ ├── TokenParser.java │ │ ├── URIBuilder.java │ │ └── URLEncodedUtils.java └── resources │ └── META-INF │ └── services │ └── org.apache.flink.table.factories.Factory └── test ├── java └── com │ └── getindata │ ├── StreamTableJob.java │ └── connectors │ └── http │ ├── ExceptionUtilsTest.java │ ├── HttpPostRequestCallbackFactoryTest.java │ ├── TestHelper.java │ ├── TestLookupPostRequestCallbackFactory.java │ ├── TestPostRequestCallbackFactory.java │ ├── app │ ├── HttpStubApp.java │ └── JsonTransform.java │ └── internal │ ├── BasicAuthHeaderValuePreprocessorTest.java │ ├── ComposeHeaderPreprocessorTest.java │ ├── HttpsConnectionTestBase.java │ ├── auth │ └── OidcAccessTokenManagerTest.java │ ├── config │ └── ConfigExceptionTest.java │ ├── retry │ ├── HttpClientWithRetryTest.java │ ├── RetryConfigProviderTest.java │ └── RetryStrategyTypeTest.java │ ├── sink │ ├── HttpSinkBuilderTest.java │ ├── HttpSinkConnectionTest.java │ ├── HttpSinkWriterStateSerializerTest.java │ ├── HttpSinkWriterTest.java │ └── httpclient │ │ ├── BatchRequestSubmitterFactoryTest.java │ │ ├── BatchRequestSubmitterTest.java │ │ ├── JavaNetSinkHttpClientConnectionTest.java │ │ ├── JavaNetSinkHttpClientTest.java │ │ └── status │ │ └── ComposeHttpStatusCodeCheckerTest.java │ ├── status │ ├── HttpCodesParserTest.java │ └── HttpResponseCheckerTest.java │ ├── table │ ├── lookup │ │ ├── AsyncHttpTableLookupFunctionTest.java │ │ ├── BodyBasedRequestFactoryTest.java │ │ ├── HttpLookupTableSourceFactoryTest.java │ │ ├── HttpLookupTableSourceITCaseTest.java │ │ ├── HttpLookupTableSourceTest.java │ │ ├── JavaNetHttpPollingClientConnectionTest.java │ │ ├── JavaNetHttpPollingClientFactoryTest.java │ │ ├── JavaNetHttpPollingClientHttpsConnectionTest.java │ │ ├── JavaNetHttpPollingClientTest.java │ │ ├── JavaNetHttpPollingClientWithWireTest.java │ │ ├── JsonTransform.java │ │ ├── LookupQueryInfoTest.java │ │ ├── RowDataSingleValueLookupSchemaEntryTest.java │ │ ├── RowTypeLookupSchemaEntryTest.java │ │ ├── TableSourceHelperTest.java │ │ └── querycreators │ │ │ ├── CustomFormatFactory.java │ │ │ ├── CustomJsonFormatFactory.java │ │ │ ├── ElasticSearchLiteQueryCreatorTest.java │ │ │ ├── GenericGetQueryCreatorTest.java │ │ │ ├── GenericJsonAndUrlQueryCreatorFactoryTest.java │ │ │ ├── GenericJsonAndUrlQueryCreatorTest.java │ │ │ ├── GenericJsonQueryCreatorFactoryTest.java │ │ │ ├── GenericJsonQueryCreatorTest.java │ │ │ ├── PathBean.java │ │ │ ├── PersonBean.java │ │ │ ├── QueryCreatorUtils.java │ │ │ └── QueryFormatAwareConfigurationTest.java │ └── sink │ │ ├── BatchRequestHttpDynamicSinkInsertTest.java │ │ ├── HttpDynamicSinkTest.java │ │ ├── HttpDynamicTableSinkFactoryTest.java │ │ └── PerRequestHttpDynamicSinkInsertTest.java │ └── utils │ ├── ConfigUtilsTest.java │ ├── HttpHeaderUtilsTest.java │ ├── JavaNetHttpClientFactoryTest.java │ └── uri │ ├── CharArrayBufferTest.java │ ├── ParserCursorTest.java │ ├── TokenParserTest.java │ ├── URIBuilderTest.java │ └── URLEncodedUtilsTest.java └── resources ├── META-INF └── services │ └── org.apache.flink.table.factories.Factory ├── auth └── AuthResult.json ├── http-array-result-with-nulls └── HttpResult.json ├── http-array-result └── HttpResult.json ├── http └── HttpResult.json ├── json └── sink │ ├── allInOneBatch.txt │ ├── fourSingleEventBatches.txt │ ├── threeBatches.txt │ └── twoBatches.txt └── security └── certs ├── ca.crt ├── ca.key ├── ca.pass.key ├── ca.srl ├── ca_server_bundle.cert.pem ├── client.crt ├── client.csr ├── client.key ├── client.pass.key ├── clientPrivateKey.der ├── clientPrivateKey.pem ├── client_keyStore.p12 ├── server.crt ├── server.csr ├── server.key ├── server.pass.key ├── serverKeyStore.jks └── serverTrustStore.jks /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | #### Description 2 | 3 | `describe the purpose of the change here` 4 | 5 | Resolves `` 6 | 7 | ##### PR Checklist 8 | - [ ] Tests added 9 | - [ ] [Changelog](CHANGELOG.md) updated 10 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'main' 7 | - 'release/**' 8 | pull_request: 9 | 10 | env: 11 | MAVEN_CLI_OPTS: "--batch-mode" 12 | MAVEN_OPTS: "-Dmaven.repo.local=.m2/repository" 13 | JAVA_ADDITIONAL_OPTS: "-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss,SSS" 14 | FF_USE_FASTZIP: "true" 15 | 16 | jobs: 17 | build: 18 | runs-on: ubuntu-latest 19 | strategy: 20 | matrix: 21 | flink: ["1.18.1", "1.19.1", "1.20.0"] 22 | steps: 23 | - uses: actions/checkout@v3 24 | 25 | - name: Set up JDK 11 26 | uses: actions/setup-java@v3 27 | with: 28 | java-version: '11' 29 | distribution: 'adopt' 30 | cache: maven 31 | 32 | - name: Build for Flink ${{ matrix.flink }} 33 | run: mvn $MAVEN_CLI_OPTS $JAVA_ADDITIONAL_OPTS -Dflink.version=${{ matrix.flink }} compile 34 | 35 | - name: Tests for Flink ${{ matrix.flink }} 36 | run: | 37 | mvn $MAVEN_CLI_OPTS $JAVA_ADDITIONAL_OPTS -Dflink.version=${{ matrix.flink }} test integration-test 38 | 39 | - name: Test JavaDoc 40 | run: mvn $MAVEN_CLI_OPTS $JAVA_ADDITIONAL_OPTS javadoc:javadoc 41 | if: startsWith(matrix.flink, '1.20') 42 | 43 | - name: Add coverage to PR 44 | id: jacoco 45 | uses: madrapps/jacoco-report@v1.7.1 46 | with: 47 | paths: ${{ github.workspace }}/target/site/jacoco/jacoco.xml 48 | token: ${{ secrets.GITHUB_TOKEN }} 49 | min-coverage-overall: 40 50 | min-coverage-changed-files: 60 51 | if: startsWith(matrix.flink, '1.20') && github.event.pull_request.head.repo.fork == false 52 | -------------------------------------------------------------------------------- /.github/workflows/prepare_release_branch.yml: -------------------------------------------------------------------------------- 1 | name: Prepare release branch 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | version_part: 7 | description: The part of the version to update (Patch, Minor or Major) 8 | required: true 9 | type: choice 10 | options: 11 | - Patch 12 | - Minor 13 | - Major 14 | default: 'Minor' 15 | 16 | jobs: 17 | prepare-branch: 18 | runs-on: ubuntu-latest 19 | 20 | permissions: # Permissions to create a new branch and a new PR. 21 | contents: write 22 | pull-requests: write 23 | 24 | steps: 25 | - uses: actions/checkout@v3 26 | 27 | - name: Set up JDK 11 28 | uses: actions/setup-java@v3 29 | with: 30 | java-version: '11' 31 | distribution: 'adopt' 32 | cache: maven 33 | 34 | - name: Validate inputs 35 | run: | 36 | echo "INPUT_VERSION_PART: ${{ github.event.inputs.version_part }}" 37 | python -c "if '${{ github.event.inputs.version_part }}' not in ['Patch', 'Minor', 'Major']: raise ValueError(\"'${{ github.event.inputs.version_part }}' must be one of ['Patch', 'Minor', 'Major'])\")" 38 | 39 | - name: Save current version 40 | id: save_current_version 41 | run: | 42 | mvn versions:set -DremoveSnapshot -DgenerateBackupPoms=false 43 | echo "::set-output name=current_version::$(mvn -B help:evaluate -Dexpression=project.version -q -DforceStdout)" 44 | 45 | - name: Update the CHANGELOG according to 'Keep a Changelog' guidelines 46 | uses: thomaseizinger/keep-a-changelog-new-release@v1 47 | with: 48 | version: ${{ steps.save_current_version.outputs.current_version }} 49 | 50 | - name: Create a new release branch 51 | run: | 52 | git config user.name github-actions 53 | git config user.email github-actions@github.com 54 | git commit -am "Bump CHANGELOG for release ${{ steps.save_current_version.outputs.current_version }}" 55 | git checkout -b release/${{ steps.save_current_version.outputs.current_version }} 56 | git push -u origin release/${{ steps.save_current_version.outputs.current_version }} 57 | 58 | - name: Bump development version 59 | run: | 60 | git checkout -b bump-version-after-${{ steps.save_current_version.outputs.current_version }} 61 | mvn validate -D 'bump${{ github.event.inputs.version_part }}' -DgenerateBackupPoms=false 62 | git commit -am "Bump development version to $(mvn -B help:evaluate -Dexpression=project.version -q -DforceStdout)" 63 | git push -u origin bump-version-after-${{ steps.save_current_version.outputs.current_version }} 64 | 65 | - name: Open a PR to bump development version to main 66 | id: open_pr 67 | uses: vsoch/pull-request-action@1.1.0 68 | env: 69 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 70 | PULL_REQUEST_BRANCH: main 71 | PULL_REQUEST_FROM_BRANCH: bump-version-after-${{ steps.save_current_version.outputs.current_version }} 72 | PULL_REQUEST_TITLE: "Bump development version after release of ${{ steps.save_current_version.outputs.current_version }}" 73 | PULL_REQUEST_BODY: "Bump SNAPSHOT version and CHANGELOG for subsequent development." 74 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | env: 8 | MAVEN_CLI_OPTS: "--batch-mode" 9 | MAVEN_OPTS: "-Dmaven.repo.local=.m2/repository" 10 | JAVA_ADDITIONAL_OPTS: "-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss,SSS" 11 | FF_USE_FASTZIP: "true" 12 | 13 | jobs: 14 | build: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v4 18 | 19 | - name: Check release tag match # ... and fail fast if they do not 20 | run: diff <(echo "${{ github.ref_name }}") <(echo "$(mvn -B help:evaluate -Dexpression=project.version -q -DforceStdout)") 21 | 22 | - name: Set up JDK 11 23 | uses: actions/setup-java@v4 24 | with: 25 | java-version: '11' 26 | distribution: 'temurin' 27 | cache: maven 28 | 29 | - name: Build 30 | run: mvn $MAVEN_CLI_OPTS $JAVA_ADDITIONAL_OPTS compile 31 | 32 | - name: Tests 33 | run: | 34 | mvn $MAVEN_CLI_OPTS $JAVA_ADDITIONAL_OPTS test integration-test 35 | 36 | - name: Set up Apache Maven Central 37 | uses: actions/setup-java@v4 38 | with: 39 | java-version: '11' 40 | distribution: 'temurin' 41 | server-id: ossrh 42 | server-username: MAVEN_USERNAME 43 | server-password: MAVEN_PASSWORD 44 | gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} 45 | gpg-passphrase: GPG_PRIVATE_KEY_PASSWORD 46 | 47 | - name: Publish to Apache Maven Central 48 | if: github.event.release 49 | run: mvn deploy -P release 50 | env: 51 | MAVEN_USERNAME: ${{ secrets.SONATYPE_TOKEN_USERNAME }} 52 | MAVEN_PASSWORD: ${{ secrets.SONATYPE_TOKEN_PASSWORD }} 53 | GPG_PRIVATE_KEY_PASSWORD: ${{ secrets.GPG_PRIVATE_KEY_PASSWORD }} 54 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea/ 2 | .idea 3 | .vscode 4 | .classpath 5 | .gitignore.swp 6 | .project 7 | .settings 8 | target 9 | bin 10 | /flink.http.connector.iml 11 | /src/main/flink-http-connector.iml 12 | /src/main/main.iml 13 | /src/test/test.iml 14 | /flink-http-connector.iml 15 | /dependency-reduced-pom.xml 16 | /.java-version 17 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## PR Guidelines 2 | 1. Fork branch from `main`. 3 | 1. Ensure to provide unit tests for new functionality. 4 | 1. Update documentation accordingly. 5 | 1. Update [changelog](CHANGELOG.md) according to ["Keep a changelog"](https://keepachangelog.com/en/1.0.0/) guidelines. 6 | 1. Squash changes with a single commit as much as possible and ensure verbose PR name. 7 | 1. Open a PR against `main` 8 | 9 | *We reserve the right to take over and modify or abandon PRs that do not match the workflow or are abandoned.* 10 | -------------------------------------------------------------------------------- /dev/README.md: -------------------------------------------------------------------------------- 1 | # Dev README 2 | Below are some helpful IntelliJ configurations you can set to match our coding style and standards. 3 | 4 | ## Checkstyle 5 | This project uses checkstyle to format Java code. If developing locally, please setup checkstyle using the following steps. 6 | 7 | 1. Add the CheckStyle-IDEA plugin to IntelliJ. 8 | - `Settings > Plugins > Marketplace > CheckStyle-IDEA > INSTALL`. 9 | - Restart your IDE if prompted. 10 | 11 | 2. Configure IntelliJ to use the `checkstyle.xml` file provided in this directory. 12 | - Go to `Settings > Tools > Checkstyle` (this tool location may differ based on your version of IntelliJ). 13 | - Set the version to 8.29. 14 | - Under the `Configuration File` heading, click the `+` symbol to add our specific configuration file. 15 | - Give our file a useful description, such as `GID Java Checks`, and provide the `connectors/dev/checkstyle.xml` path. 16 | - Click `Next` to add the checkstyle file 17 | - Check `Active` next to it once it has been added 18 | - In the top right, set the Scan Scope to `Only Java sources (including tests)` 19 | 20 | 3. Now, on the bottom tab bar, there should be a `CheckStyle` tab that lets you run Java style checks against using the `Check Project` button. 21 | 22 | ## Java Import Order 23 | We use the following import order in our Java files. Please update this in `Settings > Editor > Code Style > Java > Imports > Import Layout`: 24 | 25 | ``` 26 | import java.* 27 | import javax.* 28 | 29 | import scala.* 30 | 31 | import all other imports 32 | 33 | import com.getindata.connectors.* 34 | import com.getindata.connectors.internal.* 35 | ``` 36 | -------------------------------------------------------------------------------- /dev/checkstyle-suppressions.xml: -------------------------------------------------------------------------------- 1 | 17 | 18 | 21 | 22 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /docs/JoinTable.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/flink-http-connector/bc340abe08c54a79f7153e554008b061724505dd/docs/JoinTable.PNG -------------------------------------------------------------------------------- /lombok.config: -------------------------------------------------------------------------------- 1 | lombok.addLombokGeneratedAnnotation = true -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/HttpPostRequestCallback.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http; 2 | 3 | import java.io.Serializable; 4 | import java.net.http.HttpResponse; 5 | import java.util.Map; 6 | 7 | /** 8 | * An interface for post request callback action, processing a response and its respective request. 9 | * 10 | *

One can customize the behaviour of such a callback by implementing both 11 | * {@link HttpPostRequestCallback} and {@link HttpPostRequestCallbackFactory}. 12 | * 13 | * @param type of the HTTP request wrapper 14 | */ 15 | public interface HttpPostRequestCallback extends Serializable { 16 | /** 17 | * Process HTTP request and the matching response. 18 | * @param response HTTP response 19 | * @param requestEntry request's payload 20 | * @param endpointUrl the URL of the endpoint 21 | * @param headerMap mapping of header names to header values 22 | */ 23 | void call( 24 | HttpResponse response, 25 | RequestT requestEntry, 26 | String endpointUrl, 27 | Map headerMap 28 | ); 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/HttpPostRequestCallbackFactory.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http; 2 | 3 | import org.apache.flink.table.factories.Factory; 4 | 5 | import com.getindata.connectors.http.internal.table.lookup.HttpLookupTableSource; 6 | import com.getindata.connectors.http.internal.table.sink.HttpDynamicSink; 7 | 8 | /** 9 | * The {@link Factory} that dynamically creates and injects {@link HttpPostRequestCallback} to 10 | * {@link HttpDynamicSink} and {@link HttpLookupTableSource}. 11 | * 12 | *

Custom implementations of {@link HttpPostRequestCallbackFactory} can be registered along 13 | * other factories in 14 | * 

resources/META-INF/services/org.apache.flink.table.factories.Factory
15 | * file and then referenced by their identifiers in: 16 | *
    17 | *
  • 18 | * The HttpSink DDL property field gid.connector.http.sink.request-callback 19 | * for HTTP sink. 20 | *
    • 21 | *
  • 22 | * The Http lookup DDL property field gid.connector.http.source.lookup.request-callback 23 | * for HTTP lookup. 24 | *
    • 25 | *
26 | *
27 | * 28 | *

The following example shows the minimum Table API example to create a {@link HttpDynamicSink} 29 | * that uses a custom callback created by a factory that returns my-callback as its 30 | * identifier. 31 | * 32 | * 

{@code
33 |  * CREATE TABLE http (
34 |  *   id bigint,
35 |  *   some_field string
36 |  * ) with (
37 |  *   'connector' = 'http-sink'
38 |  *   'url' = 'http://example.com/myendpoint'
39 |  *   'format' = 'json',
40 |  *   'gid.connector.http.sink.request-callback' = 'my-callback'
41 |  * )
42 |  * }
43 | * 44 | *

The following example shows the minimum Table API example to create a 45 | * {@link HttpLookupTableSource} that uses a custom callback created by a factory that 46 | * returns my-callback as its identifier. 47 | * 48 | * 

{@code
49 |  * CREATE TABLE httplookup (
50 |  *   id bigint
51 |  * ) with (
52 |  *   'connector' = 'rest-lookup',
53 |  *   'url' = 'http://example.com/myendpoint',
54 |  *   'format' = 'json',
55 |  *   'gid.connector.http.source.lookup.request-callback' = 'my-callback'
56 |  * )
57 |  * }
58 | * 59 | * @param type of the HTTP request wrapper 60 | */ 61 | 62 | public interface HttpPostRequestCallbackFactory extends Factory { 63 | /** 64 | * @return {@link HttpPostRequestCallback} custom request callback instance 65 | */ 66 | HttpPostRequestCallback createHttpPostRequestCallback(); 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/HttpSink.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http; 2 | 3 | import java.util.Properties; 4 | 5 | import org.apache.flink.annotation.PublicEvolving; 6 | import org.apache.flink.connector.base.sink.writer.ElementConverter; 7 | 8 | import com.getindata.connectors.http.internal.HeaderPreprocessor; 9 | import com.getindata.connectors.http.internal.SinkHttpClientBuilder; 10 | import com.getindata.connectors.http.internal.sink.HttpSinkInternal; 11 | import com.getindata.connectors.http.internal.sink.HttpSinkRequestEntry; 12 | import com.getindata.connectors.http.internal.sink.httpclient.HttpRequest; 13 | 14 | /** 15 | * A public implementation for {@code HttpSink} that performs async requests against a specified 16 | * HTTP endpoint using the buffering protocol specified in 17 | * {@link org.apache.flink.connector.base.sink.AsyncSinkBase}. 18 | * 19 | *

20 | * To create a new instance of this class use {@link HttpSinkBuilder}. An example would be: 21 | * 

{@code
22 |  * HttpSink httpSink =
23 |  *     HttpSink.builder()
24 |  *             .setEndpointUrl("http://example.com/myendpoint")
25 |  *             .setElementConverter(
26 |  *                 (s, _context) -> new HttpSinkRequestEntry("POST", "text/plain",
27 |  *                 s.getBytes(StandardCharsets.UTF_8)))
28 |  *             .build();
29 |  * }
30 | * 31 | * @param type of the elements that should be sent through HTTP request. 32 | */ 33 | @PublicEvolving 34 | public class HttpSink extends HttpSinkInternal { 35 | 36 | HttpSink( 37 | ElementConverter elementConverter, 38 | int maxBatchSize, 39 | int maxInFlightRequests, 40 | int maxBufferedRequests, 41 | long maxBatchSizeInBytes, 42 | long maxTimeInBufferMS, 43 | long maxRecordSizeInBytes, 44 | String endpointUrl, 45 | HttpPostRequestCallback httpPostRequestCallback, 46 | HeaderPreprocessor headerPreprocessor, 47 | SinkHttpClientBuilder sinkHttpClientBuilder, 48 | Properties properties) { 49 | 50 | super(elementConverter, 51 | maxBatchSize, 52 | maxInFlightRequests, 53 | maxBufferedRequests, 54 | maxBatchSizeInBytes, 55 | maxTimeInBufferMS, 56 | maxRecordSizeInBytes, 57 | endpointUrl, 58 | httpPostRequestCallback, 59 | headerPreprocessor, 60 | sinkHttpClientBuilder, 61 | properties 62 | ); 63 | } 64 | 65 | /** 66 | * Create a {@link HttpSinkBuilder} constructing a new {@link HttpSink}. 67 | * 68 | * @param type of the elements that should be sent through HTTP request 69 | * @return {@link HttpSinkBuilder} 70 | */ 71 | public static HttpSinkBuilder builder() { 72 | return new HttpSinkBuilder<>(); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/HttpStatusCodeValidationFailedException.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http; 2 | 3 | import java.net.http.HttpResponse; 4 | 5 | import lombok.Getter; 6 | 7 | @Getter 8 | public class HttpStatusCodeValidationFailedException extends Exception { 9 | private final HttpResponse response; 10 | 11 | public HttpStatusCodeValidationFailedException(String message, HttpResponse response) { 12 | super(message); 13 | this.response = response; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/LookupArg.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http; 2 | 3 | import lombok.Data; 4 | import lombok.RequiredArgsConstructor; 5 | 6 | /** 7 | * Transfer object that contains single lookup argument (column name) and its value. 8 | */ 9 | @Data 10 | @RequiredArgsConstructor 11 | public class LookupArg { 12 | 13 | /** 14 | * Lookup argument name. 15 | */ 16 | private final String argName; 17 | 18 | /** 19 | * Lookup argument value. 20 | */ 21 | private final String argValue; 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/LookupQueryCreator.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http; 2 | 3 | import java.io.Serializable; 4 | 5 | import org.apache.flink.table.data.RowData; 6 | 7 | import com.getindata.connectors.http.internal.table.lookup.LookupQueryInfo; 8 | 9 | /** 10 | * An interface for a creator of a lookup query in the Http Lookup Source (e.g., the query that 11 | * gets appended as query parameters to the URI in GET request or supplied as the payload of a 12 | * body-based request along with optional query parameters). 13 | * 14 | *

One can customize how those queries are built by implementing {@link LookupQueryCreator} and 15 | * {@link LookupQueryCreatorFactory}. 16 | */ 17 | public interface LookupQueryCreator extends Serializable { 18 | 19 | /** 20 | * Create a lookup query (like the query appended to path in GET request) 21 | * out of the provided arguments. 22 | * 23 | * @param lookupDataRow a {@link RowData} containing request parameters. 24 | * @return a lookup query. 25 | */ 26 | LookupQueryInfo createLookupQuery(RowData lookupDataRow); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/LookupQueryCreatorFactory.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http; 2 | 3 | import java.io.Serializable; 4 | 5 | import org.apache.flink.configuration.ReadableConfig; 6 | import org.apache.flink.table.factories.DynamicTableFactory; 7 | import org.apache.flink.table.factories.Factory; 8 | 9 | import com.getindata.connectors.http.internal.table.lookup.HttpLookupTableSource; 10 | import com.getindata.connectors.http.internal.table.lookup.LookupRow; 11 | 12 | /** 13 | * The {@link Factory} that dynamically creates and injects {@link LookupQueryCreator} to 14 | * {@link HttpLookupTableSource}. 15 | * 16 | *

Custom implementations of {@link LookupQueryCreatorFactory} can be registered along other 17 | * factories in 

resources/META-INF.services/org.apache.flink.table.factories.Factory
18 | * file and then referenced by their identifiers in the HttpLookupSource DDL property field 19 | * gid.connector.http.source.lookup.query-creator. 20 | * 21 | *

The following example shows the minimum Table API example to create a 22 | * {@link HttpLookupTableSource} that uses a custom query creator created by a factory that returns 23 | * my-query-creator as its identifier. 24 | * 25 | * 

{@code
26 |  * CREATE TABLE http (
27 |  *   id bigint,
28 |  *   some_field string
29 |  * ) WITH (
30 |  *   'connector' = 'rest-lookup',
31 |  *   'format' = 'json',
32 |  *   'url' = 'http://example.com/myendpoint',
33 |  *   'gid.connector.http.source.lookup.query-creator' = 'my-query-creator'
34 |  * )
35 |  * }
36 | */ 37 | public interface LookupQueryCreatorFactory extends Factory, Serializable { 38 | 39 | /** 40 | * @param readableConfig readable config 41 | * @param lookupRow lookup row 42 | * @param dynamicTableFactoryContext context 43 | * @return {@link LookupQueryCreator} custom lookup query creator instance 44 | */ 45 | LookupQueryCreator createLookupQueryCreator( 46 | ReadableConfig readableConfig, 47 | LookupRow lookupRow, 48 | DynamicTableFactory.Context dynamicTableFactoryContext); 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/SchemaLifecycleAwareElementConverter.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http; 2 | 3 | import org.apache.flink.api.common.serialization.SerializationSchema.InitializationContext; 4 | import org.apache.flink.api.connector.sink2.Sink.InitContext; 5 | import org.apache.flink.connector.base.sink.writer.ElementConverter; 6 | 7 | /** 8 | * An enhancement for Flink's {@link ElementConverter} that expose {@link #open(InitContext)} method 9 | * that will be called by HTTP connect code to ensure that element converter is initialized 10 | * properly. This is required for cases when Flink's SerializationSchema and DeserializationSchema 11 | * objects like JsonRowDataSerializationSchema are used. 12 | *

13 | * This interface specifies the mapping between elements of a stream to request entries that can be 14 | * sent to the destination. The mapping is provided by the end-user of a sink, not the sink 15 | * creator. 16 | * 17 | *

The request entries contain all relevant information required to create and sent the actual 18 | * request. Eg, for Kinesis Data Streams, the request entry includes the payload and the partition 19 | * key. 20 | */ 21 | public interface SchemaLifecycleAwareElementConverter 22 | extends ElementConverter { 23 | 24 | /** 25 | * Initialization element converter for the schema. 26 | * 27 | *

The provided {@link InitializationContext} can be used to access additional features such 28 | * as e.g. registering user metrics. 29 | * 30 | * @param context Contextual information that can be used during initialization. 31 | */ 32 | void open(InitContext context); 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/BasicAuthHeaderValuePreprocessor.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal; 2 | 3 | import java.util.Base64; 4 | import java.util.Objects; 5 | 6 | /** 7 | * Header processor for HTTP Basic Authentication mechanism. 8 | * Only "Basic" authentication is supported currently. 9 | */ 10 | public class BasicAuthHeaderValuePreprocessor implements HeaderValuePreprocessor { 11 | 12 | public static final String BASIC = "Basic "; 13 | 14 | private boolean useRawAuthHeader = false; 15 | 16 | /** 17 | * Creates a new instance of BasicAuthHeaderValuePreprocessor that uses 18 | * the default processing of the Authorization header. 19 | */ 20 | public BasicAuthHeaderValuePreprocessor() { 21 | this(false); 22 | } 23 | 24 | /** 25 | * Creates a new instance of BasicAuthHeaderValuePreprocessor. 26 | * 27 | * @param useRawAuthHeader If set to true, the Authorization header is kept as-is, 28 | * untransformed. Otherwise, uses the default processing of the 29 | * Authorization header. 30 | */ 31 | public BasicAuthHeaderValuePreprocessor(boolean useRawAuthHeader) { 32 | this.useRawAuthHeader = useRawAuthHeader; 33 | } 34 | 35 | /** 36 | * Calculates {@link Base64} value of provided header value. For Basic authentication mechanism, 37 | * the raw value is expected to match user:password pattern. 38 | *

39 | * If rawValue starts with "Basic " prefix, or useRawAuthHeader has been set to true, it is 40 | * assumed that this value is already converted to the expected "Authorization" header value. 41 | * 42 | * @param rawValue header original value to modify. 43 | * @return value of "Authorization" header with format "Basic " + Base64 from rawValue or 44 | * rawValue without any changes if it starts with "Basic " prefix or useRawAuthHeader is 45 | * set to true. 46 | */ 47 | @Override 48 | public String preprocessHeaderValue(String rawValue) { 49 | Objects.requireNonNull(rawValue); 50 | if (useRawAuthHeader || rawValue.startsWith(BASIC)) { 51 | return rawValue; 52 | } else { 53 | return BASIC + Base64.getEncoder().encodeToString(rawValue.getBytes()); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/ComposeHeaderPreprocessor.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal; 2 | 3 | import java.util.Collections; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | /** 8 | * This implementation of {@link HeaderPreprocessor} acts as a registry for all {@link 9 | * HeaderValuePreprocessor} that should be applied on HTTP request. 10 | */ 11 | public class ComposeHeaderPreprocessor implements HeaderPreprocessor { 12 | 13 | /** 14 | * Default, pass through header value preprocessor used whenever dedicated preprocessor for a 15 | * given header does not exist. 16 | */ 17 | private static final HeaderValuePreprocessor DEFAULT_VALUE_PREPROCESSOR = rawValue -> rawValue; 18 | 19 | /** 20 | * Map with {@link HeaderValuePreprocessor} to apply. 21 | */ 22 | private final Map valuePreprocessors; 23 | 24 | /** 25 | * Creates a new instance of ComposeHeaderPreprocessor for provided {@link 26 | * HeaderValuePreprocessor} map. 27 | * 28 | * @param valuePreprocessors map of {@link HeaderValuePreprocessor} that should be used for this 29 | * processor. If null, then default, pass through header value 30 | * processor will be used for every header. 31 | */ 32 | public ComposeHeaderPreprocessor(Map valuePreprocessors) { 33 | this.valuePreprocessors = (valuePreprocessors == null) 34 | ? Collections.emptyMap() 35 | : new HashMap<>(valuePreprocessors); 36 | } 37 | 38 | @Override 39 | public String preprocessValueForHeader(String headerName, String headerRawValue) { 40 | return valuePreprocessors 41 | .getOrDefault(headerName, DEFAULT_VALUE_PREPROCESSOR) 42 | .preprocessHeaderValue(headerRawValue); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/HeaderPreprocessor.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Interface for header preprocessing 7 | */ 8 | public interface HeaderPreprocessor extends Serializable { 9 | 10 | /** 11 | * Preprocess value of a header.Preprocessing can change or validate header value. 12 | * @param headerName header name which value should be preprocessed. 13 | * @param headerRawValue header value to process. 14 | * @return preprocessed header value. 15 | */ 16 | String preprocessValueForHeader(String headerName, String headerRawValue); 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/HeaderValuePreprocessor.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Processor interface which modifies header value based on implemented logic. 7 | * An example would be calculation of Value of Authorization header. 8 | */ 9 | public interface HeaderValuePreprocessor extends Serializable { 10 | 11 | /** 12 | * Modifies header rawValue according to the implemented logic. 13 | * @param rawValue header original value to modify 14 | * @return modified header value. 15 | */ 16 | String preprocessHeaderValue(String rawValue); 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/OIDCAuthHeaderValuePreprocessor.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal; 2 | 3 | import java.net.http.HttpClient; 4 | import java.time.Duration; 5 | import java.util.Optional; 6 | 7 | import lombok.extern.slf4j.Slf4j; 8 | 9 | import com.getindata.connectors.http.internal.auth.OidcAccessTokenManager; 10 | 11 | /** 12 | * Header processor for HTTP OIDC Authentication mechanism. 13 | */ 14 | @Slf4j 15 | public class OIDCAuthHeaderValuePreprocessor implements HeaderValuePreprocessor { 16 | 17 | 18 | private final String oidcAuthURL; 19 | private final String oidcTokenRequest; 20 | private Duration oidcExpiryReduction = Duration.ofSeconds(1); 21 | /** 22 | * Add the access token to the request using OidcAuth authenticate method that 23 | * gives us a valid access token. 24 | * @param oidcAuthURL OIDC token endpoint 25 | * @param oidcTokenRequest OIDC Token Request 26 | * @param oidcExpiryReduction OIDC token expiry reduction 27 | */ 28 | 29 | public OIDCAuthHeaderValuePreprocessor(String oidcAuthURL, 30 | String oidcTokenRequest, 31 | Optional oidcExpiryReduction) { 32 | this.oidcAuthURL = oidcAuthURL; 33 | this.oidcTokenRequest = oidcTokenRequest; 34 | if (oidcExpiryReduction.isPresent()) { 35 | this.oidcExpiryReduction = oidcExpiryReduction.get(); 36 | } 37 | } 38 | 39 | @Override 40 | public String preprocessHeaderValue(String rawValue) { 41 | OidcAccessTokenManager auth = new OidcAccessTokenManager( 42 | HttpClient.newBuilder().build(), 43 | oidcTokenRequest, 44 | oidcAuthURL, 45 | oidcExpiryReduction 46 | ); 47 | // apply the OIDC authentication by adding the dynamically calculated header value. 48 | return "BEARER " + auth.authenticate(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/PollingClient.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal; 2 | 3 | import java.util.Collection; 4 | 5 | import org.apache.flink.table.data.RowData; 6 | import org.apache.flink.table.functions.FunctionContext; 7 | 8 | /** 9 | * A client that is used to get enrichment data from external component. 10 | */ 11 | public interface PollingClient { 12 | 13 | /** 14 | * Gets enrichment data from external component using provided lookup arguments. 15 | * @param lookupRow A {@link RowData} containing request parameters. 16 | * @return an optional result of data lookup. 17 | */ 18 | Collection pull(RowData lookupRow); 19 | 20 | /** 21 | * Initialize the client. 22 | * @param ctx function context 23 | */ 24 | void open(FunctionContext ctx); 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/PollingClientFactory.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal; 2 | 3 | import java.io.Serializable; 4 | 5 | import org.apache.flink.api.common.serialization.DeserializationSchema; 6 | import org.apache.flink.util.ConfigurationException; 7 | 8 | import com.getindata.connectors.http.internal.table.lookup.HttpLookupConfig; 9 | 10 | public interface PollingClientFactory extends Serializable { 11 | 12 | PollingClient createPollClient( 13 | HttpLookupConfig options, 14 | DeserializationSchema schemaDecoder 15 | ) throws ConfigurationException; 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/SinkHttpClient.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal; 2 | 3 | import java.util.List; 4 | import java.util.concurrent.CompletableFuture; 5 | 6 | import com.getindata.connectors.http.internal.sink.HttpSinkInternal; 7 | import com.getindata.connectors.http.internal.sink.HttpSinkRequestEntry; 8 | import com.getindata.connectors.http.internal.sink.HttpSinkWriter; 9 | 10 | /** 11 | * An HTTP client that is used by {@link HttpSinkWriter} to send HTTP requests processed by {@link 12 | * HttpSinkInternal}. 13 | */ 14 | public interface SinkHttpClient { 15 | 16 | /** 17 | * Sends HTTP requests to an external web service. 18 | * 19 | * @param requestEntries a set of request entries that should be sent to the destination 20 | * @param endpointUrl the URL of the endpoint 21 | * @return the new {@link CompletableFuture} wrapping {@link SinkHttpClientResponse} that 22 | * completes when all requests have been sent and returned their statuses 23 | */ 24 | CompletableFuture putRequests( 25 | List requestEntries, 26 | String endpointUrl 27 | ); 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/SinkHttpClientBuilder.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal; 2 | 3 | import java.io.Serializable; 4 | import java.util.Properties; 5 | 6 | import org.apache.flink.annotation.PublicEvolving; 7 | 8 | import com.getindata.connectors.http.HttpPostRequestCallback; 9 | import com.getindata.connectors.http.internal.sink.httpclient.HttpRequest; 10 | import com.getindata.connectors.http.internal.sink.httpclient.RequestSubmitterFactory; 11 | 12 | /** 13 | * Builder building {@link SinkHttpClient}. 14 | */ 15 | @PublicEvolving 16 | public interface SinkHttpClientBuilder extends Serializable { 17 | 18 | // TODO Consider moving HttpPostRequestCallback and HeaderPreprocessor, RequestSubmitter to be a 19 | // SinkHttpClientBuilder fields. This method is getting more and more arguments. 20 | SinkHttpClient build( 21 | Properties properties, 22 | HttpPostRequestCallback httpPostRequestCallback, 23 | HeaderPreprocessor headerPreprocessor, 24 | RequestSubmitterFactory requestSubmitterFactory 25 | 26 | ); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/SinkHttpClientResponse.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal; 2 | 3 | import java.util.List; 4 | 5 | import lombok.Data; 6 | import lombok.NonNull; 7 | import lombok.ToString; 8 | 9 | import com.getindata.connectors.http.internal.sink.HttpSinkRequestEntry; 10 | import com.getindata.connectors.http.internal.sink.httpclient.HttpRequest; 11 | 12 | /** 13 | * Data class holding {@link HttpSinkRequestEntry} instances that {@link SinkHttpClient} attempted 14 | * to write, divided into two lists — successful and failed ones. 15 | */ 16 | @Data 17 | @ToString 18 | public class SinkHttpClientResponse { 19 | 20 | /** 21 | * A list of successfully written requests. 22 | */ 23 | @NonNull 24 | private final List successfulRequests; 25 | 26 | /** 27 | * A list of requests that {@link SinkHttpClient} failed to write. 28 | */ 29 | @NonNull 30 | private final List failedRequests; 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/config/ConfigException.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.config; 2 | 3 | /** 4 | * A Runtime exception throw when there is any issue with configuration properties for Http 5 | * Connector. 6 | */ 7 | public class ConfigException extends RuntimeException { 8 | 9 | private static final long serialVersionUID = 1L; 10 | 11 | public ConfigException(String message) { 12 | super(message); 13 | } 14 | 15 | public ConfigException(String message, Throwable t) { 16 | super(message, t); 17 | } 18 | 19 | /** 20 | * Creates an exception object using predefined exception message template: 21 | * {@code Invalid value + (value) + for configuration + (property name) + (additional message) } 22 | * @param name configuration property name. 23 | * @param value configuration property value. 24 | * @param message custom message appended to the end of exception message. 25 | */ 26 | public ConfigException(String name, Object value, String message) { 27 | super("Invalid value " + value + " for configuration " + name + (message == null ? "" 28 | : ": " + message)); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/config/SinkRequestSubmitMode.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.config; 2 | 3 | public enum SinkRequestSubmitMode { 4 | 5 | SINGLE("single"), 6 | BATCH("batch"); 7 | 8 | private final String mode; 9 | 10 | SinkRequestSubmitMode(String mode) { 11 | this.mode = mode; 12 | } 13 | 14 | public String getMode() { 15 | return mode; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/retry/HttpClientWithRetry.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.retry; 2 | 3 | import java.io.IOException; 4 | import java.net.http.HttpClient; 5 | import java.net.http.HttpRequest; 6 | import java.net.http.HttpResponse; 7 | import java.util.function.Supplier; 8 | 9 | import io.github.resilience4j.retry.Retry; 10 | import io.github.resilience4j.retry.RetryConfig; 11 | import lombok.Builder; 12 | import lombok.Getter; 13 | import lombok.extern.slf4j.Slf4j; 14 | import org.apache.flink.metrics.MetricGroup; 15 | 16 | import com.getindata.connectors.http.HttpStatusCodeValidationFailedException; 17 | import com.getindata.connectors.http.internal.status.HttpResponseChecker; 18 | 19 | @Slf4j 20 | public class HttpClientWithRetry { 21 | 22 | private final HttpClient httpClient; 23 | @Getter 24 | private final HttpResponseChecker responseChecker; 25 | private final Retry retry; 26 | 27 | @Builder 28 | HttpClientWithRetry(HttpClient httpClient, 29 | RetryConfig retryConfig, 30 | HttpResponseChecker responseChecker) { 31 | this.httpClient = httpClient; 32 | this.responseChecker = responseChecker; 33 | var adjustedRetryConfig = RetryConfig.from(retryConfig) 34 | .retryExceptions(IOException.class) 35 | .retryOnResult(this::isTemporalError) 36 | .build(); 37 | this.retry = Retry.of("http-lookup-connector", adjustedRetryConfig); 38 | } 39 | 40 | public void registerMetrics(MetricGroup metrics){ 41 | var group = metrics.addGroup("http_lookup_connector"); 42 | group.gauge("successfulCallsWithRetryAttempt", 43 | () -> retry.getMetrics().getNumberOfSuccessfulCallsWithRetryAttempt()); 44 | group.gauge("successfulCallsWithoutRetryAttempt", 45 | () -> retry.getMetrics().getNumberOfSuccessfulCallsWithoutRetryAttempt()); 46 | } 47 | 48 | public HttpResponse send( 49 | Supplier requestSupplier, 50 | HttpResponse.BodyHandler responseBodyHandler 51 | ) throws IOException, InterruptedException, HttpStatusCodeValidationFailedException { 52 | try { 53 | var response = Retry.decorateCheckedSupplier(retry, 54 | () -> httpClient.send(requestSupplier.get(), responseBodyHandler)).apply(); 55 | if (!responseChecker.isSuccessful(response)) { 56 | throw new HttpStatusCodeValidationFailedException( 57 | "Incorrect response code: " + response.statusCode(), response); 58 | } 59 | return response; 60 | } catch (IOException | InterruptedException | HttpStatusCodeValidationFailedException e) { 61 | throw e; //re-throw without wrapping 62 | } catch (Throwable t) { 63 | throw new RuntimeException("Unexpected exception", t); 64 | } 65 | } 66 | 67 | private boolean isTemporalError(Object response) { 68 | return responseChecker.isTemporalError((HttpResponse) response); 69 | } 70 | } 71 | 72 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/retry/RetryConfigProvider.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.retry; 2 | 3 | import io.github.resilience4j.core.IntervalFunction; 4 | import io.github.resilience4j.retry.RetryConfig; 5 | import lombok.AccessLevel; 6 | import lombok.RequiredArgsConstructor; 7 | import org.apache.flink.configuration.ReadableConfig; 8 | import org.apache.flink.table.connector.source.lookup.LookupOptions; 9 | import static io.github.resilience4j.core.IntervalFunction.ofExponentialBackoff; 10 | 11 | import static com.getindata.connectors.http.internal.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_INITIAL_BACKOFF; 12 | import static com.getindata.connectors.http.internal.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_MAX_BACKOFF; 13 | import static com.getindata.connectors.http.internal.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_MULTIPLIER; 14 | import static com.getindata.connectors.http.internal.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_FIXED_DELAY_DELAY; 15 | import static com.getindata.connectors.http.internal.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_STRATEGY; 16 | 17 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 18 | public class RetryConfigProvider { 19 | 20 | private final ReadableConfig config; 21 | 22 | public static RetryConfig create(ReadableConfig config) { 23 | return new RetryConfigProvider(config).create(); 24 | } 25 | 26 | private RetryConfig create() { 27 | return createBuilder() 28 | .maxAttempts(config.get(LookupOptions.MAX_RETRIES) + 1) 29 | .build(); 30 | } 31 | 32 | private RetryConfig.Builder createBuilder() { 33 | var retryStrategy = getRetryStrategy(); 34 | if (retryStrategy == RetryStrategyType.FIXED_DELAY) { 35 | return configureFixedDelay(); 36 | } else if (retryStrategy == RetryStrategyType.EXPONENTIAL_DELAY) { 37 | return configureExponentialDelay(); 38 | } 39 | throw new IllegalArgumentException("Unsupported retry strategy: " + retryStrategy); 40 | } 41 | 42 | private RetryStrategyType getRetryStrategy() { 43 | return RetryStrategyType.fromCode(config.get(SOURCE_LOOKUP_RETRY_STRATEGY)); 44 | } 45 | 46 | private RetryConfig.Builder configureFixedDelay() { 47 | return RetryConfig.custom() 48 | .intervalFunction(IntervalFunction.of(config.get(SOURCE_LOOKUP_RETRY_FIXED_DELAY_DELAY))); 49 | } 50 | 51 | private RetryConfig.Builder configureExponentialDelay() { 52 | var initialDelay = config.get(SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_INITIAL_BACKOFF); 53 | var maxDelay = config.get(SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_MAX_BACKOFF); 54 | var multiplier = config.get(SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_MULTIPLIER); 55 | return RetryConfig.custom() 56 | .intervalFunction(ofExponentialBackoff(initialDelay, multiplier, maxDelay)); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/retry/RetryStrategyType.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.retry; 2 | 3 | import lombok.AccessLevel; 4 | import lombok.Getter; 5 | import lombok.RequiredArgsConstructor; 6 | 7 | @Getter 8 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 9 | public enum RetryStrategyType { 10 | FIXED_DELAY("fixed-delay"), 11 | EXPONENTIAL_DELAY("exponential-delay"), 12 | ; 13 | 14 | private final String code; 15 | 16 | public static RetryStrategyType fromCode(String code) { 17 | if (code == null) { 18 | throw new NullPointerException("Code is null"); 19 | } 20 | for (var strategy : RetryStrategyType.values()) { 21 | if (strategy.getCode().equalsIgnoreCase(code)) { 22 | return strategy; 23 | } 24 | } 25 | throw new IllegalArgumentException("No enum constant for " + code); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/security/SelfSignedTrustManager.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.security; 2 | 3 | import java.net.Socket; 4 | import java.security.cert.CertificateException; 5 | import java.security.cert.X509Certificate; 6 | import javax.net.ssl.SSLEngine; 7 | import javax.net.ssl.X509ExtendedTrustManager; 8 | import javax.net.ssl.X509TrustManager; 9 | 10 | import lombok.extern.slf4j.Slf4j; 11 | 12 | @Slf4j 13 | public class SelfSignedTrustManager extends X509ExtendedTrustManager { 14 | 15 | private final X509TrustManager delegate; 16 | 17 | public SelfSignedTrustManager(X509TrustManager delegate) { 18 | this.delegate = delegate; 19 | } 20 | 21 | public void checkClientTrusted(X509Certificate[] chain, String s) throws CertificateException { 22 | this.delegate.checkClientTrusted(chain, s); 23 | } 24 | 25 | public void checkClientTrusted(X509Certificate[] chain, String s, Socket socket) 26 | throws CertificateException { 27 | this.delegate.checkClientTrusted(chain, s); 28 | } 29 | 30 | public void checkClientTrusted(X509Certificate[] chain, String s, SSLEngine sslEngine) 31 | throws CertificateException { 32 | this.delegate.checkClientTrusted(chain, s); 33 | } 34 | 35 | public void checkServerTrusted(X509Certificate[] chain, String s) throws CertificateException { 36 | log.info("Allowing self signed server certificates."); 37 | } 38 | 39 | public void checkServerTrusted(X509Certificate[] chain, String s, Socket socket) 40 | throws CertificateException { 41 | log.info("Allowing self signed server certificates."); 42 | } 43 | 44 | public void checkServerTrusted(X509Certificate[] chain, String s, SSLEngine sslEngine) 45 | throws CertificateException { 46 | log.info("Allowing self signed server certificates."); 47 | } 48 | 49 | public X509Certificate[] getAcceptedIssuers() { 50 | return this.delegate.getAcceptedIssuers(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/sink/HttpSinkRequestEntry.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.sink; 2 | 3 | import java.io.Serializable; 4 | 5 | import lombok.EqualsAndHashCode; 6 | import lombok.NonNull; 7 | import lombok.RequiredArgsConstructor; 8 | import lombok.ToString; 9 | 10 | /** 11 | * Represents a single {@link com.getindata.connectors.http.HttpSink} request. Contains the HTTP 12 | * method name, Content-Type header value, and byte representation of the body of the request. 13 | */ 14 | @RequiredArgsConstructor 15 | @EqualsAndHashCode 16 | @ToString 17 | public final class HttpSinkRequestEntry implements Serializable { 18 | 19 | /** 20 | * HTTP method name to use when sending the request. 21 | */ 22 | @NonNull 23 | public final String method; 24 | 25 | /** 26 | * Body of the request, encoded as byte array. 27 | */ 28 | public final byte[] element; 29 | 30 | /** 31 | * @return the size of the {@link HttpSinkRequestEntry#element} 32 | */ 33 | public long getSizeInBytes() { 34 | return element.length; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/sink/HttpSinkWriterStateSerializer.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.sink; 2 | 3 | import java.io.DataInputStream; 4 | import java.io.DataOutputStream; 5 | import java.io.IOException; 6 | 7 | import org.apache.flink.connector.base.sink.writer.AsyncSinkWriterStateSerializer; 8 | 9 | /** 10 | * An implementation of {@link AsyncSinkWriterStateSerializer} for {@link HttpSinkInternal} and its 11 | * {@link HttpSinkWriter}. 12 | */ 13 | public class HttpSinkWriterStateSerializer 14 | extends AsyncSinkWriterStateSerializer { 15 | 16 | @Override 17 | protected void serializeRequestToStream(HttpSinkRequestEntry s, DataOutputStream out) 18 | throws IOException { 19 | out.writeUTF(s.method); 20 | out.write(s.element); 21 | } 22 | 23 | @Override 24 | protected HttpSinkRequestEntry deserializeRequestFromStream(long requestSize, 25 | DataInputStream in) throws IOException { 26 | var method = in.readUTF(); 27 | var bytes = new byte[(int) requestSize]; 28 | in.read(bytes); 29 | return new HttpSinkRequestEntry(method, bytes); 30 | } 31 | 32 | @Override 33 | public int getVersion() { 34 | return 1; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/sink/httpclient/AbstractRequestSubmitter.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.sink.httpclient; 2 | 3 | import java.net.http.HttpClient; 4 | import java.util.Properties; 5 | import java.util.concurrent.ExecutorService; 6 | import java.util.concurrent.Executors; 7 | 8 | import org.apache.flink.util.concurrent.ExecutorThreadFactory; 9 | 10 | import com.getindata.connectors.http.internal.config.HttpConnectorConfigConstants; 11 | import com.getindata.connectors.http.internal.utils.ThreadUtils; 12 | 13 | public abstract class AbstractRequestSubmitter implements RequestSubmitter { 14 | 15 | protected static final int HTTP_CLIENT_PUBLISHING_THREAD_POOL_SIZE = 1; 16 | 17 | protected static final String DEFAULT_REQUEST_TIMEOUT_SECONDS = "30"; 18 | 19 | /** 20 | * Thread pool to handle HTTP response from HTTP client. 21 | */ 22 | protected final ExecutorService publishingThreadPool; 23 | 24 | protected final int httpRequestTimeOutSeconds; 25 | 26 | protected final String[] headersAndValues; 27 | 28 | protected final HttpClient httpClient; 29 | 30 | public AbstractRequestSubmitter( 31 | Properties properties, 32 | String[] headersAndValues, 33 | HttpClient httpClient) { 34 | 35 | this.headersAndValues = headersAndValues; 36 | this.publishingThreadPool = 37 | Executors.newFixedThreadPool( 38 | HTTP_CLIENT_PUBLISHING_THREAD_POOL_SIZE, 39 | new ExecutorThreadFactory( 40 | "http-sink-client-response-worker", ThreadUtils.LOGGING_EXCEPTION_HANDLER)); 41 | 42 | this.httpRequestTimeOutSeconds = Integer.parseInt( 43 | properties.getProperty(HttpConnectorConfigConstants.SINK_HTTP_TIMEOUT_SECONDS, 44 | DEFAULT_REQUEST_TIMEOUT_SECONDS) 45 | ); 46 | 47 | this.httpClient = httpClient; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/sink/httpclient/HttpRequest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.sink.httpclient; 2 | 3 | import java.util.List; 4 | 5 | import lombok.Data; 6 | 7 | @Data 8 | public class HttpRequest { 9 | 10 | public final java.net.http.HttpRequest httpRequest; 11 | 12 | public final List elements; 13 | 14 | public final String method; 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/sink/httpclient/JavaNetHttpResponseWrapper.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.sink.httpclient; 2 | 3 | import java.net.http.HttpResponse; 4 | import java.util.Optional; 5 | 6 | import lombok.Data; 7 | import lombok.NonNull; 8 | 9 | import com.getindata.connectors.http.internal.sink.HttpSinkRequestEntry; 10 | 11 | /** 12 | * A wrapper structure around an HTTP response, keeping a reference to a particular {@link 13 | * HttpSinkRequestEntry}. Used internally by the {@code HttpSinkWriter} to pass {@code 14 | * HttpSinkRequestEntry} along some other element that it is logically connected with. 15 | */ 16 | @Data 17 | final class JavaNetHttpResponseWrapper { 18 | 19 | /** 20 | * A representation of a single {@link com.getindata.connectors.http.HttpSink} request. 21 | */ 22 | @NonNull 23 | private final HttpRequest httpRequest; 24 | 25 | /** 26 | * A response to an HTTP request based on {@link HttpSinkRequestEntry}. 27 | */ 28 | private final HttpResponse response; 29 | 30 | public Optional> getResponse() { 31 | return Optional.ofNullable(response); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/sink/httpclient/PerRequestRequestSubmitterFactory.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.sink.httpclient; 2 | 3 | import java.util.Properties; 4 | import java.util.concurrent.ExecutorService; 5 | import java.util.concurrent.Executors; 6 | 7 | import org.apache.flink.util.concurrent.ExecutorThreadFactory; 8 | 9 | import com.getindata.connectors.http.internal.utils.JavaNetHttpClientFactory; 10 | import com.getindata.connectors.http.internal.utils.ThreadUtils; 11 | 12 | public class PerRequestRequestSubmitterFactory implements RequestSubmitterFactory { 13 | 14 | // TODO Add this property to config. Make sure to add note in README.md that will describe that 15 | // any value greater than one will break order of messages. 16 | int HTTP_CLIENT_THREAD_POOL_SIZE = 1; 17 | 18 | @Override 19 | public RequestSubmitter createSubmitter(Properties properties, String[] headersAndValues) { 20 | 21 | ExecutorService httpClientExecutor = 22 | Executors.newFixedThreadPool( 23 | HTTP_CLIENT_THREAD_POOL_SIZE, 24 | new ExecutorThreadFactory( 25 | "http-sink-client-per-request-worker", ThreadUtils.LOGGING_EXCEPTION_HANDLER)); 26 | 27 | return new PerRequestSubmitter( 28 | properties, 29 | headersAndValues, 30 | JavaNetHttpClientFactory.createClient(properties, httpClientExecutor) 31 | ); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/sink/httpclient/PerRequestSubmitter.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.sink.httpclient; 2 | 3 | import java.net.URI; 4 | import java.net.http.HttpClient; 5 | import java.net.http.HttpClient.Version; 6 | import java.net.http.HttpRequest.BodyPublishers; 7 | import java.net.http.HttpRequest.Builder; 8 | import java.net.http.HttpResponse; 9 | import java.time.Duration; 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | import java.util.Properties; 13 | import java.util.concurrent.CompletableFuture; 14 | 15 | import lombok.extern.slf4j.Slf4j; 16 | 17 | import com.getindata.connectors.http.internal.sink.HttpSinkRequestEntry; 18 | 19 | /** 20 | * This implementation creates HTTP requests for every processed event. 21 | */ 22 | @Slf4j 23 | public class PerRequestSubmitter extends AbstractRequestSubmitter { 24 | 25 | public PerRequestSubmitter( 26 | Properties properties, 27 | String[] headersAndValues, 28 | HttpClient httpClient) { 29 | 30 | super(properties, headersAndValues, httpClient); 31 | } 32 | 33 | @Override 34 | public List> submit( 35 | String endpointUrl, 36 | List requestToSubmit) { 37 | 38 | var endpointUri = URI.create(endpointUrl); 39 | var responseFutures = new ArrayList>(); 40 | 41 | for (var entry : requestToSubmit) { 42 | HttpRequest httpRequest = buildHttpRequest(entry, endpointUri); 43 | var response = httpClient 44 | .sendAsync( 45 | httpRequest.getHttpRequest(), 46 | HttpResponse.BodyHandlers.ofString()) 47 | .exceptionally(ex -> { 48 | // TODO This will be executed on a ForkJoinPool Thread... refactor this someday. 49 | log.error("Request fatally failed because of an exception", ex); 50 | return null; 51 | }) 52 | .thenApplyAsync( 53 | res -> new JavaNetHttpResponseWrapper(httpRequest, res), 54 | publishingThreadPool 55 | ); 56 | responseFutures.add(response); 57 | } 58 | return responseFutures; 59 | } 60 | 61 | private HttpRequest buildHttpRequest(HttpSinkRequestEntry requestEntry, URI endpointUri) { 62 | Builder requestBuilder = java.net.http.HttpRequest 63 | .newBuilder() 64 | .uri(endpointUri) 65 | .version(Version.HTTP_1_1) 66 | .timeout(Duration.ofSeconds(httpRequestTimeOutSeconds)) 67 | .method(requestEntry.method, 68 | BodyPublishers.ofByteArray(requestEntry.element)); 69 | 70 | if (headersAndValues.length != 0) { 71 | requestBuilder.headers(headersAndValues); 72 | } 73 | 74 | return new HttpRequest( 75 | requestBuilder.build(), 76 | List.of(requestEntry.element), 77 | requestEntry.method 78 | ); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/sink/httpclient/RequestSubmitter.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.sink.httpclient; 2 | 3 | import java.util.List; 4 | import java.util.concurrent.CompletableFuture; 5 | 6 | import com.getindata.connectors.http.internal.sink.HttpSinkRequestEntry; 7 | 8 | /** 9 | * Submits request via HTTP. 10 | */ 11 | public interface RequestSubmitter { 12 | 13 | List> submit( 14 | String endpointUrl, 15 | List requestToSubmit); 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/sink/httpclient/RequestSubmitterFactory.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.sink.httpclient; 2 | 3 | import java.util.Properties; 4 | 5 | public interface RequestSubmitterFactory { 6 | 7 | RequestSubmitter createSubmitter(Properties properties, String[] headersAndValues); 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/status/HttpCodesParser.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.status; 2 | 3 | import java.util.Collections; 4 | import java.util.HashSet; 5 | import java.util.Set; 6 | import java.util.regex.Pattern; 7 | import java.util.stream.Collectors; 8 | import java.util.stream.IntStream; 9 | import static java.lang.String.format; 10 | 11 | import lombok.experimental.UtilityClass; 12 | import org.apache.flink.util.ConfigurationException; 13 | 14 | @UtilityClass 15 | public class HttpCodesParser { 16 | 17 | private final Pattern CODE_GROUP_EXPRESSION = Pattern.compile("[1-5][xX]{2}"); 18 | private final String DELIMITER = Pattern.quote(","); 19 | private final int HTTP_CODE_MIN = 100; 20 | private final int HTTP_CODE_MAX = 599; 21 | 22 | public Set parse(String codesExpression) throws ConfigurationException { 23 | var whitelist = new HashSet(); 24 | var blacklist = new HashSet(); 25 | for (var rawCode : codesExpression.split(DELIMITER)) { 26 | var code = rawCode.trim(); 27 | if (code.isEmpty()) { 28 | continue; 29 | } 30 | if (code.startsWith("!")) { 31 | try { 32 | blacklist.add(parseHttpCode(code.substring(1))); 33 | continue; 34 | } catch (NumberFormatException e) { 35 | throw new ConfigurationException("Can not parse code " + code); 36 | } 37 | } 38 | try { 39 | whitelist.add(parseHttpCode(code)); 40 | } catch (NumberFormatException e) { 41 | if (CODE_GROUP_EXPRESSION.matcher(code).matches()) { 42 | var firstGroupCode = Integer.parseInt(code.substring(0, 1)) * 100; 43 | var groupCodes = IntStream.range(firstGroupCode, firstGroupCode + 100) 44 | .boxed().collect(Collectors.toList()); 45 | whitelist.addAll(groupCodes); 46 | } else { 47 | throw new ConfigurationException("Can not parse code " + code); 48 | } 49 | } 50 | } 51 | 52 | whitelist.removeAll(blacklist); 53 | return Collections.unmodifiableSet(whitelist); 54 | } 55 | 56 | private Integer parseHttpCode(String str) throws ConfigurationException { 57 | var parsed = Integer.parseInt(str); 58 | if (parsed < HTTP_CODE_MIN || parsed > HTTP_CODE_MAX) { 59 | throw new ConfigurationException(format("Http code out of the range [%s]", parsed)); 60 | } 61 | return parsed; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/status/HttpResponseChecker.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.status; 2 | 3 | import java.net.http.HttpResponse; 4 | import java.util.HashSet; 5 | import java.util.Set; 6 | 7 | import lombok.Getter; 8 | import lombok.NonNull; 9 | import org.apache.flink.util.ConfigurationException; 10 | 11 | @Getter 12 | public class HttpResponseChecker { 13 | 14 | private final Set successCodes; 15 | private final Set temporalErrorCodes; 16 | 17 | HttpResponseChecker(@NonNull String successCodeExpr, @NonNull String temporalErrorCodeExpr) 18 | throws ConfigurationException { 19 | this(HttpCodesParser.parse(successCodeExpr), HttpCodesParser.parse(temporalErrorCodeExpr)); 20 | } 21 | 22 | public HttpResponseChecker(@NonNull Set successCodes, @NonNull Set temporalErrorCodes) 23 | throws ConfigurationException { 24 | this.successCodes = successCodes; 25 | this.temporalErrorCodes = temporalErrorCodes; 26 | validate(); 27 | } 28 | 29 | public boolean isSuccessful(HttpResponse response) { 30 | return isSuccessful(response.statusCode()); 31 | } 32 | 33 | public boolean isSuccessful(int httpStatusCode) { 34 | return successCodes.contains(httpStatusCode); 35 | } 36 | 37 | public boolean isTemporalError(HttpResponse response) { 38 | return isTemporalError(response.statusCode()); 39 | } 40 | 41 | public boolean isTemporalError(int httpStatusCode) { 42 | return temporalErrorCodes.contains(httpStatusCode); 43 | } 44 | 45 | private void validate() throws ConfigurationException { 46 | if (successCodes.isEmpty()) { 47 | throw new ConfigurationException("Success code list can not be empty"); 48 | } 49 | var intersection = new HashSet<>(successCodes); 50 | intersection.retainAll(temporalErrorCodes); 51 | if (!intersection.isEmpty()) { 52 | throw new ConfigurationException("Http codes " + intersection + 53 | " can not be used as both success and retry codes"); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/status/HttpResponseCodeType.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.status; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | /** 7 | * This enum represents HTTP response code types, grouped by "hundreds" digit. 8 | */ 9 | public enum HttpResponseCodeType { 10 | 11 | INFO(1), 12 | SUCCESS(2), 13 | REDIRECTION(3), 14 | CLIENT_ERROR(4), 15 | SERVER_ERROR(5); 16 | 17 | private static final Map map; 18 | 19 | static { 20 | map = new HashMap<>(); 21 | for (HttpResponseCodeType httpResponseCodeType : HttpResponseCodeType.values()) { 22 | map.put(httpResponseCodeType.httpTypeCode, httpResponseCodeType); 23 | } 24 | } 25 | 26 | private final int httpTypeCode; 27 | 28 | HttpResponseCodeType(int httpTypeCode) { 29 | this.httpTypeCode = httpTypeCode; 30 | } 31 | 32 | /** 33 | * @param statusCode Http status code to get the {@link HttpResponseCodeType} instance for. 34 | * @return a {@link HttpResponseCodeType} instance based on http type code, for example {@code 35 | * HttpResponseCodeType.getByCode(1)} will return {@link HttpResponseCodeType#INFO} type. 36 | */ 37 | public static HttpResponseCodeType getByCode(int statusCode) { 38 | return map.get(statusCode); 39 | } 40 | 41 | /** 42 | * @return a "hundreds" digit that represents given {@link HttpResponseCodeType} instance. 43 | * For example {@code HttpResponseCodeType.INFO.getHttpTypeCode()} will return 1 since HTTP 44 | * information repossess have status codes in range 100 - 199. 45 | */ 46 | public int getHttpTypeCode() { 47 | return this.httpTypeCode; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/status/HttpStatusCodeChecker.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.status; 2 | 3 | /** 4 | * Base interface for all classes that would validate HTTP status 5 | * code whether it is an error or not. 6 | */ 7 | public interface HttpStatusCodeChecker { 8 | 9 | /** 10 | * Validates http status code wheter it is considered as error code. The logic for 11 | * what status codes are considered as "errors" depends on the concreted implementation 12 | * @param statusCode http status code to assess. 13 | * @return true if statusCode is considered as Error and false if not. 14 | */ 15 | boolean isErrorCode(int statusCode); 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/status/SingleValueHttpStatusCodeChecker.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.status; 2 | 3 | import lombok.EqualsAndHashCode; 4 | import lombok.RequiredArgsConstructor; 5 | 6 | /** 7 | * An implementation of {@link HttpStatusCodeChecker} that validates status code against 8 | * constant value. 9 | */ 10 | @RequiredArgsConstructor 11 | @EqualsAndHashCode 12 | public class SingleValueHttpStatusCodeChecker implements HttpStatusCodeChecker { 13 | 14 | /** 15 | * A reference http status code to compare with. 16 | */ 17 | private final int errorCode; 18 | 19 | /** 20 | * Validates given statusCode against constant value. 21 | * @param statusCode http status code to assess. 22 | * @return true if status code is considered as error or false if not. 23 | */ 24 | @Override 25 | public boolean isErrorCode(int statusCode) { 26 | return errorCode == statusCode; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/status/TypeStatusCodeChecker.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.status; 2 | 3 | import lombok.EqualsAndHashCode; 4 | 5 | /** 6 | * Implementation of {@link HttpStatusCodeChecker} that verifies if given Http status code 7 | * belongs to specific HTTP code type family. For example if it any of 100's 200's or 500's code. 8 | */ 9 | @EqualsAndHashCode 10 | public class TypeStatusCodeChecker implements HttpStatusCodeChecker { 11 | 12 | /** 13 | * First digit from HTTP status code that describes a type of code, 14 | * for example 1 for all 100's, 5 for all 500's. 15 | */ 16 | private final int httpTypeCode; 17 | 18 | /** 19 | * Creates TypeStatusCodeChecker for given {@link HttpResponseCodeType} 20 | * 21 | * @param httpResponseCodeType {@link HttpResponseCodeType} for this {@link 22 | * TypeStatusCodeChecker} instance. 23 | */ 24 | public TypeStatusCodeChecker(HttpResponseCodeType httpResponseCodeType) { 25 | this.httpTypeCode = httpResponseCodeType.getHttpTypeCode(); 26 | } 27 | 28 | /** 29 | * Checks whether given status code belongs to Http code status type. 30 | * For example: 31 | * 

{@code
32 |      *    TypeStatusCodeChecker checker =  new TypeStatusCodeChecker(5);
33 |      *    checker.isErrorCode(505); <- will return true.
34 |      *    }
35 |      *
36 | * @param statusCode http status code to assess. 37 | * @return true if status code is considered as error or false if not. 38 | */ 39 | @Override 40 | public boolean isErrorCode(int statusCode) { 41 | return statusCode / 100 == httpTypeCode; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/status/WhiteListHttpStatusCodeChecker.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.status; 2 | 3 | import lombok.EqualsAndHashCode; 4 | import lombok.RequiredArgsConstructor; 5 | 6 | /** 7 | * Class that implements logic of a "white list" against single constant value. 8 | */ 9 | @RequiredArgsConstructor 10 | @EqualsAndHashCode 11 | public class WhiteListHttpStatusCodeChecker { 12 | 13 | private final int whiteListCode; 14 | 15 | /** 16 | * Checks if given statusCode is considered as "white listed" 17 | * @param statusCode status code to check. 18 | * @return true if given statusCode is white listed and false if not. 19 | */ 20 | public boolean isWhiteListed(int statusCode) { 21 | return whiteListCode == statusCode; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/SerializationSchemaElementConverter.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table; 2 | 3 | import org.apache.flink.api.common.serialization.SerializationSchema; 4 | import org.apache.flink.api.connector.sink2.Sink.InitContext; 5 | import org.apache.flink.api.connector.sink2.SinkWriter.Context; 6 | import org.apache.flink.table.data.RowData; 7 | import org.apache.flink.util.FlinkRuntimeException; 8 | 9 | import com.getindata.connectors.http.SchemaLifecycleAwareElementConverter; 10 | import com.getindata.connectors.http.internal.sink.HttpSinkRequestEntry; 11 | 12 | public class SerializationSchemaElementConverter 13 | implements SchemaLifecycleAwareElementConverter { 14 | 15 | private final String insertMethod; 16 | 17 | private final SerializationSchema serializationSchema; 18 | 19 | private boolean schemaOpened = false; 20 | 21 | public SerializationSchemaElementConverter( 22 | String insertMethod, 23 | SerializationSchema serializationSchema) { 24 | 25 | this.insertMethod = insertMethod; 26 | this.serializationSchema = serializationSchema; 27 | } 28 | 29 | @Override 30 | public void open(InitContext context) { 31 | if (!schemaOpened) { 32 | try { 33 | serializationSchema.open(context.asSerializationSchemaInitializationContext()); 34 | schemaOpened = true; 35 | } catch (Exception e) { 36 | throw new FlinkRuntimeException("Failed to initialize serialization schema.", e); 37 | } 38 | } 39 | } 40 | 41 | @Override 42 | public HttpSinkRequestEntry apply(RowData rowData, Context context) { 43 | return new HttpSinkRequestEntry( 44 | insertMethod, 45 | serializationSchema.serialize(rowData)); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/BodyBasedRequestFactory.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import java.net.URI; 4 | import java.net.URISyntaxException; 5 | import java.net.http.HttpRequest; 6 | import java.net.http.HttpRequest.BodyPublishers; 7 | import java.net.http.HttpRequest.Builder; 8 | import java.time.Duration; 9 | 10 | import lombok.extern.slf4j.Slf4j; 11 | import org.slf4j.Logger; 12 | 13 | import com.getindata.connectors.http.LookupQueryCreator; 14 | import com.getindata.connectors.http.internal.HeaderPreprocessor; 15 | import com.getindata.connectors.http.internal.utils.uri.URIBuilder; 16 | 17 | /** 18 | * Implementation of {@link HttpRequestFactory} for REST calls that sends their parameters using 19 | * request body or in the path. 20 | */ 21 | @Slf4j 22 | public class BodyBasedRequestFactory extends RequestFactoryBase { 23 | 24 | private final String methodName; 25 | 26 | public BodyBasedRequestFactory( 27 | String methodName, 28 | LookupQueryCreator lookupQueryCreator, 29 | HeaderPreprocessor headerPreprocessor, 30 | HttpLookupConfig options) { 31 | 32 | super(lookupQueryCreator, headerPreprocessor, options); 33 | this.methodName = methodName.toUpperCase(); 34 | } 35 | 36 | /** 37 | * Method for preparing {@link HttpRequest.Builder} for REST request that sends their parameters 38 | * in request body, for example PUT or POST methods 39 | * 40 | * @param lookupQueryInfo lookup query info used for request body. 41 | * @return {@link HttpRequest.Builder} for given lookupQuery. 42 | */ 43 | @Override 44 | protected Builder setUpRequestMethod(LookupQueryInfo lookupQueryInfo) { 45 | return HttpRequest.newBuilder() 46 | .uri(constructUri(lookupQueryInfo)) 47 | .method(methodName, BodyPublishers.ofString(lookupQueryInfo.getLookupQuery())) 48 | .timeout(Duration.ofSeconds(this.httpRequestTimeOutSeconds)); 49 | } 50 | 51 | @Override 52 | protected Logger getLogger() { 53 | return log; 54 | } 55 | 56 | URI constructUri(LookupQueryInfo lookupQueryInfo) { 57 | StringBuilder resolvedUrl = new StringBuilder(baseUrl); 58 | if (lookupQueryInfo.hasBodyBasedUrlQueryParameters()) { 59 | resolvedUrl.append(baseUrl.contains("?") ? "&" : "?") 60 | .append(lookupQueryInfo.getBodyBasedUrlQueryParameters()); 61 | } 62 | resolvedUrl = resolvePathParameters(lookupQueryInfo, resolvedUrl); 63 | 64 | try { 65 | return new URIBuilder(resolvedUrl.toString()).build(); 66 | } catch (URISyntaxException e) { 67 | throw new RuntimeException(e); 68 | } 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/GetRequestFactory.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import java.net.URI; 4 | import java.net.URISyntaxException; 5 | import java.net.http.HttpRequest; 6 | import java.net.http.HttpRequest.Builder; 7 | import java.time.Duration; 8 | 9 | import lombok.extern.slf4j.Slf4j; 10 | import org.slf4j.Logger; 11 | 12 | import com.getindata.connectors.http.LookupQueryCreator; 13 | import com.getindata.connectors.http.internal.HeaderPreprocessor; 14 | import com.getindata.connectors.http.internal.utils.uri.URIBuilder; 15 | 16 | /** 17 | * Implementation of {@link HttpRequestFactory} for GET REST calls. 18 | */ 19 | @Slf4j 20 | public class GetRequestFactory extends RequestFactoryBase { 21 | 22 | public GetRequestFactory( 23 | LookupQueryCreator lookupQueryCreator, 24 | HeaderPreprocessor headerPreprocessor, 25 | HttpLookupConfig options) { 26 | 27 | super(lookupQueryCreator, headerPreprocessor, options); 28 | } 29 | 30 | @Override 31 | protected Logger getLogger() { 32 | return log; 33 | } 34 | 35 | /** 36 | * Method for preparing {@link HttpRequest.Builder} for REST GET request, where lookupQueryInfo 37 | * is used as query parameters for GET requests, for example: 38 | * 
39 |      *     http:localhost:8080/service?id=1
40 |      *
41 | * or as payload for body-based requests with optional parameters, for example: 42 | * 
43 |      *     http:localhost:8080/service?id=1
44 |      *     body payload: { "uid": 2 }
45 |      *
46 | * @param lookupQueryInfo lookup query info used for request query parameters. 47 | * @return {@link HttpRequest.Builder} for given GET lookupQuery 48 | */ 49 | @Override 50 | protected Builder setUpRequestMethod(LookupQueryInfo lookupQueryInfo) { 51 | return HttpRequest.newBuilder() 52 | .uri(constructGetUri(lookupQueryInfo)) 53 | .GET() 54 | .timeout(Duration.ofSeconds(this.httpRequestTimeOutSeconds)); 55 | } 56 | 57 | URI constructGetUri(LookupQueryInfo lookupQueryInfo) { 58 | StringBuilder resolvedUrl = new StringBuilder(baseUrl); 59 | if (lookupQueryInfo.hasLookupQuery()) { 60 | resolvedUrl.append(baseUrl.contains("?") ? "&" : "?") 61 | .append(lookupQueryInfo.getLookupQuery()); 62 | } 63 | resolvedUrl = resolvePathParameters(lookupQueryInfo, resolvedUrl); 64 | try { 65 | return new URIBuilder(resolvedUrl.toString()).build(); 66 | } catch (URISyntaxException e) { 67 | throw new RuntimeException(e); 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/HttpLookupConfig.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import java.io.Serializable; 4 | import java.util.Properties; 5 | 6 | import lombok.Builder; 7 | import lombok.Data; 8 | import lombok.RequiredArgsConstructor; 9 | import org.apache.flink.configuration.Configuration; 10 | import org.apache.flink.configuration.ReadableConfig; 11 | 12 | import com.getindata.connectors.http.HttpPostRequestCallback; 13 | 14 | @Builder 15 | @Data 16 | @RequiredArgsConstructor 17 | public class HttpLookupConfig implements Serializable { 18 | 19 | private final String lookupMethod; 20 | 21 | private final String url; 22 | 23 | @Builder.Default 24 | private final boolean useAsync = false; 25 | 26 | @Builder.Default 27 | private final Properties properties = new Properties(); 28 | 29 | @Builder.Default 30 | private final ReadableConfig readableConfig = new Configuration(); 31 | 32 | private final HttpPostRequestCallback httpPostRequestCallback; 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/HttpLookupSourceRequestEntry.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import java.net.http.HttpRequest; 4 | 5 | import lombok.Data; 6 | import lombok.ToString; 7 | 8 | /** 9 | * Wrapper class around {@link HttpRequest} that contains information about an actual lookup request 10 | * body or request parameters. 11 | */ 12 | @Data 13 | @ToString 14 | public class HttpLookupSourceRequestEntry { 15 | 16 | /** 17 | * Wrapped {@link HttpRequest} object. 18 | */ 19 | private final HttpRequest httpRequest; 20 | 21 | /** 22 | * This field represents lookup query. Depending on used REST request method, this field can 23 | * represent a request body, for example a Json string when PUT/POST requests method was used, 24 | * or it can represent a query parameters if GET method was used. 25 | */ 26 | private final LookupQueryInfo lookupQueryInfo; 27 | 28 | public HttpLookupSourceRequestEntry(HttpRequest httpRequest, LookupQueryInfo lookupQueryInfo) { 29 | this.httpRequest = httpRequest; 30 | this.lookupQueryInfo = lookupQueryInfo; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/HttpRequestFactory.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import java.io.Serializable; 4 | import java.net.http.HttpRequest; 5 | 6 | import org.apache.flink.table.data.RowData; 7 | 8 | /** 9 | * Factory for creating {@link HttpRequest} objects for Rest clients. 10 | */ 11 | public interface HttpRequestFactory extends Serializable { 12 | 13 | /** 14 | * Creates a {@link HttpRequest} from given {@link RowData}. 15 | * 16 | * @param lookupRow {@link RowData} object used for building http request. 17 | * @return {@link HttpRequest} created from {@link RowData} 18 | */ 19 | HttpLookupSourceRequestEntry buildLookupRequest(RowData lookupRow); 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/HttpTableLookupFunction.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import java.util.Collection; 4 | import java.util.concurrent.atomic.AtomicInteger; 5 | 6 | import lombok.AccessLevel; 7 | import lombok.Getter; 8 | import lombok.extern.slf4j.Slf4j; 9 | import org.apache.flink.annotation.VisibleForTesting; 10 | import org.apache.flink.api.common.serialization.DeserializationSchema; 11 | import org.apache.flink.table.data.RowData; 12 | import org.apache.flink.table.functions.FunctionContext; 13 | import org.apache.flink.table.functions.LookupFunction; 14 | 15 | import com.getindata.connectors.http.internal.PollingClient; 16 | import com.getindata.connectors.http.internal.PollingClientFactory; 17 | import com.getindata.connectors.http.internal.utils.SerializationSchemaUtils; 18 | 19 | @Slf4j 20 | public class HttpTableLookupFunction extends LookupFunction { 21 | 22 | private final PollingClientFactory pollingClientFactory; 23 | 24 | private final DeserializationSchema responseSchemaDecoder; 25 | 26 | @VisibleForTesting 27 | @Getter(AccessLevel.PACKAGE) 28 | private final LookupRow lookupRow; 29 | 30 | @VisibleForTesting 31 | @Getter(AccessLevel.PACKAGE) 32 | private final HttpLookupConfig options; 33 | 34 | private transient AtomicInteger localHttpCallCounter; 35 | 36 | private transient PollingClient client; 37 | 38 | public HttpTableLookupFunction( 39 | PollingClientFactory pollingClientFactory, 40 | DeserializationSchema responseSchemaDecoder, 41 | LookupRow lookupRow, 42 | HttpLookupConfig options) { 43 | 44 | this.pollingClientFactory = pollingClientFactory; 45 | this.responseSchemaDecoder = responseSchemaDecoder; 46 | this.lookupRow = lookupRow; 47 | this.options = options; 48 | } 49 | 50 | @Override 51 | public void open(FunctionContext context) throws Exception { 52 | this.responseSchemaDecoder.open( 53 | SerializationSchemaUtils 54 | .createDeserializationInitContext(HttpTableLookupFunction.class)); 55 | 56 | this.localHttpCallCounter = new AtomicInteger(0); 57 | this.client = pollingClientFactory 58 | .createPollClient(options, responseSchemaDecoder); 59 | 60 | context.getMetricGroup() 61 | .gauge("http-table-lookup-call-counter", () -> localHttpCallCounter.intValue()); 62 | 63 | client.open(context); 64 | } 65 | 66 | @Override 67 | public Collection lookup(RowData keyRow) { 68 | localHttpCallCounter.incrementAndGet(); 69 | return client.pull(keyRow); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClientFactory.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import java.net.http.HttpClient; 4 | 5 | import org.apache.flink.api.common.serialization.DeserializationSchema; 6 | import org.apache.flink.table.data.RowData; 7 | import org.apache.flink.util.ConfigurationException; 8 | 9 | import com.getindata.connectors.http.internal.PollingClientFactory; 10 | import com.getindata.connectors.http.internal.utils.JavaNetHttpClientFactory; 11 | 12 | public class JavaNetHttpPollingClientFactory implements PollingClientFactory { 13 | 14 | private final HttpRequestFactory requestFactory; 15 | 16 | public JavaNetHttpPollingClientFactory(HttpRequestFactory requestFactory) { 17 | this.requestFactory = requestFactory; 18 | } 19 | 20 | @Override 21 | public JavaNetHttpPollingClient createPollClient( 22 | HttpLookupConfig options, 23 | DeserializationSchema schemaDecoder) throws ConfigurationException { 24 | 25 | HttpClient httpClient = JavaNetHttpClientFactory.createClient(options); 26 | 27 | return new JavaNetHttpPollingClient( 28 | httpClient, 29 | schemaDecoder, 30 | options, 31 | requestFactory 32 | ); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/LookupQueryInfo.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import java.io.Serializable; 4 | import java.nio.charset.StandardCharsets; 5 | import java.util.Collections; 6 | import java.util.Map; 7 | import java.util.stream.Collectors; 8 | 9 | import lombok.Getter; 10 | import lombok.ToString; 11 | 12 | import com.getindata.connectors.http.internal.utils.uri.NameValuePair; 13 | import com.getindata.connectors.http.internal.utils.uri.URLEncodedUtils; 14 | 15 | 16 | /** 17 | * Holds the lookup query for an HTTP request. 18 | * The {@code lookupQuery} either contain the query parameters for a GET operation 19 | * or the payload of a body-based request. 20 | * The {@code bodyBasedUrlQueryParams} contains the optional query parameters of a 21 | * body-based request in addition to its payload supplied with {@code lookupQuery}. 22 | */ 23 | @ToString 24 | public class LookupQueryInfo implements Serializable { 25 | @Getter 26 | private final String lookupQuery; 27 | 28 | private final Map bodyBasedUrlQueryParams; 29 | 30 | private final Map pathBasedUrlParams; 31 | 32 | public LookupQueryInfo(String lookupQuery) { 33 | this(lookupQuery, null, null); 34 | } 35 | 36 | public LookupQueryInfo(String lookupQuery, Map bodyBasedUrlQueryParams, 37 | Map pathBasedUrlParams) { 38 | this.lookupQuery = 39 | lookupQuery == null ? "" : lookupQuery; 40 | this.bodyBasedUrlQueryParams = 41 | bodyBasedUrlQueryParams == null ? Collections.emptyMap() : bodyBasedUrlQueryParams; 42 | this.pathBasedUrlParams = 43 | pathBasedUrlParams == null ? Collections.emptyMap() : pathBasedUrlParams; 44 | } 45 | 46 | public String getBodyBasedUrlQueryParameters() { 47 | return URLEncodedUtils.format( 48 | bodyBasedUrlQueryParams 49 | .entrySet() 50 | .stream() 51 | // sort the map by key to ensure there is a reliable order for unit tests 52 | .sorted(Map.Entry.comparingByKey()) 53 | .map(entry -> new NameValuePair(entry.getKey(), entry.getValue())) 54 | .collect(Collectors.toList()), 55 | StandardCharsets.UTF_8); 56 | } 57 | 58 | public Map getPathBasedUrlParameters() { 59 | return pathBasedUrlParams; 60 | } 61 | 62 | public boolean hasLookupQuery() { 63 | return !lookupQuery.isBlank(); 64 | } 65 | 66 | public boolean hasBodyBasedUrlQueryParameters() { 67 | return !bodyBasedUrlQueryParams.isEmpty(); 68 | } 69 | 70 | public boolean hasPathBasedUrlParameters() { 71 | return !pathBasedUrlParams.isEmpty(); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/LookupRow.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import java.io.Serializable; 4 | import java.util.Collection; 5 | import java.util.LinkedList; 6 | import java.util.List; 7 | 8 | import lombok.Getter; 9 | import lombok.Setter; 10 | import lombok.ToString; 11 | import org.apache.flink.annotation.VisibleForTesting; 12 | import org.apache.flink.table.data.RowData; 13 | import org.apache.flink.table.types.DataType; 14 | 15 | import com.getindata.connectors.http.LookupArg; 16 | 17 | @ToString 18 | public class LookupRow implements Serializable { 19 | 20 | private final List> lookupEntries; 21 | 22 | @Getter 23 | @Setter 24 | private DataType lookupPhysicalRowDataType; 25 | 26 | public LookupRow() { 27 | this.lookupEntries = new LinkedList<>(); 28 | } 29 | 30 | /** 31 | * Creates a collection of {@link LookupArg} elements. Every column and its value from provided 32 | * {@link RowData} is converted to {@link LookupArg}. 33 | * 34 | * @param lookupDataRow A {@link RowData} to get the values from for {@code 35 | * LookupArg#getArgValue()}. 36 | * @return Collection of {@link LookupArg} objects created from lookupDataRow. 37 | */ 38 | public Collection convertToLookupArgs(RowData lookupDataRow) { 39 | List lookupArgs = new LinkedList<>(); 40 | for (LookupSchemaEntry lookupSchemaEntry : lookupEntries) { 41 | lookupArgs.addAll(lookupSchemaEntry.convertToLookupArg(lookupDataRow)); 42 | } 43 | return lookupArgs; 44 | } 45 | 46 | public LookupRow addLookupEntry(LookupSchemaEntry lookupSchemaEntry) { 47 | this.lookupEntries.add(lookupSchemaEntry); 48 | return this; 49 | } 50 | 51 | @VisibleForTesting 52 | List> getLookupEntries() { 53 | return new LinkedList<>(lookupEntries); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/LookupSchemaEntry.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import java.io.Serializable; 4 | import java.util.List; 5 | 6 | import com.getindata.connectors.http.LookupArg; 7 | 8 | /** 9 | * Represents Lookup entry with its name and provides conversion method to collection of {@link 10 | * LookupArg} elements. 11 | * 12 | * @param type of lookupKeyRow used for converting to {@link LookupArg}. 13 | */ 14 | public interface LookupSchemaEntry extends Serializable { 15 | 16 | /** 17 | * @return lookup Field name. 18 | */ 19 | String getFieldName(); 20 | 21 | /** 22 | * Creates a collection of {@link LookupArg} elements from provided T lookupKeyRow 23 | * 24 | * @param lookupKeyRow Element to get the values from for {@code LookupArg#getArgValue()}. 25 | * @return Collection of {@link LookupArg} objects created from lookupKeyRow. 26 | */ 27 | List convertToLookupArg(T lookupKeyRow); 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/RowDataLookupSchemaEntryBase.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import org.apache.flink.table.data.RowData; 4 | 5 | /** 6 | * Base implementation of {@link LookupSchemaEntry} for {@link RowData} type. 7 | */ 8 | public abstract class RowDataLookupSchemaEntryBase implements LookupSchemaEntry { 9 | 10 | /** 11 | * Lookup field name represented by this instance. 12 | */ 13 | protected final String fieldName; 14 | 15 | /** 16 | * {@link RowData.FieldGetter} matching RowData type for field represented by this instance. 17 | */ 18 | protected final RowData.FieldGetter fieldGetter; 19 | 20 | /** 21 | * Creates new instance. 22 | * 23 | * @param fieldName field name that this instance represents, matching {@link RowData} column 24 | * name. 25 | * @param fieldGetter {@link RowData.FieldGetter} for data type matching {@link RowData} column 26 | * type that this instance represents. 27 | */ 28 | public RowDataLookupSchemaEntryBase(String fieldName, RowData.FieldGetter fieldGetter) { 29 | this.fieldName = fieldName; 30 | this.fieldGetter = fieldGetter; 31 | } 32 | 33 | public String getFieldName() { 34 | return this.fieldName; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/RowDataSingleValueLookupSchemaEntry.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import java.util.Collections; 4 | import java.util.List; 5 | 6 | import lombok.extern.slf4j.Slf4j; 7 | import org.apache.flink.table.data.RowData; 8 | import org.apache.flink.table.data.RowData.FieldGetter; 9 | import org.apache.flink.table.data.binary.BinaryStringData; 10 | 11 | import com.getindata.connectors.http.LookupArg; 12 | 13 | /** 14 | * Implementation of {@link LookupSchemaEntry} for {@link RowData} type that represents single 15 | * lookup column. 16 | */ 17 | @Slf4j 18 | public class RowDataSingleValueLookupSchemaEntry extends RowDataLookupSchemaEntryBase { 19 | 20 | /** 21 | * Creates new instance. 22 | * 23 | * @param fieldName field name that this instance represents, matching {@link RowData} column 24 | * name. 25 | * @param fieldGetter {@link RowData.FieldGetter} for data type matching {@link RowData} column 26 | * type that this instance represents. 27 | */ 28 | public RowDataSingleValueLookupSchemaEntry(String fieldName, FieldGetter fieldGetter) { 29 | super(fieldName, fieldGetter); 30 | } 31 | 32 | /** 33 | * Creates single element collection that contains {@link LookupArg} for single column from 34 | * given lookupKeyRow. The column is defined by 'fieldName' and 'fieldGetter' used for creating 35 | * {@link RowDataSingleValueLookupSchemaEntry} instance 36 | * 37 | * @param lookupKeyRow Element to get the values from for {@code LookupArg#getArgValue()}. 38 | * @return single element collection of {@link LookupArg}. 39 | */ 40 | @Override 41 | public List convertToLookupArg(RowData lookupKeyRow) { 42 | 43 | Object value = tryGetValue(lookupKeyRow); 44 | 45 | if (value == null) { 46 | return Collections.emptyList(); 47 | } 48 | 49 | if (!(value instanceof BinaryStringData)) { 50 | log.debug("Unsupported Key Type {}. Trying simple toString().", 51 | value.getClass()); 52 | } 53 | 54 | return Collections.singletonList(new LookupArg(getFieldName(), value.toString())); 55 | } 56 | 57 | private Object tryGetValue(RowData lookupKeyRow) { 58 | try { 59 | return fieldGetter.getFieldOrNull(lookupKeyRow); 60 | } catch (ClassCastException e) { 61 | throw new RuntimeException( 62 | "Class cast exception on field getter for field " + getFieldName(), e); 63 | } 64 | } 65 | 66 | @lombok.Generated 67 | @Override 68 | public String toString() { 69 | return "RowDataSingleValueLookupSchemaEntry{" + 70 | "fieldName='" + fieldName + '\'' + 71 | ", fieldGetter=" + fieldGetter + 72 | '}'; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/RowTypeLookupSchemaEntry.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import java.util.Collections; 4 | import java.util.LinkedList; 5 | import java.util.List; 6 | 7 | import org.apache.flink.table.data.RowData; 8 | import org.apache.flink.table.data.RowData.FieldGetter; 9 | 10 | import com.getindata.connectors.http.LookupArg; 11 | 12 | /** 13 | * Implementation of {@link LookupSchemaEntry} for {@link RowData} type that represents multiple 14 | * columns. 15 | */ 16 | public class RowTypeLookupSchemaEntry extends RowDataLookupSchemaEntryBase { 17 | 18 | /** 19 | * {@link LookupSchemaEntry} elements for every lookup column represented by {@link 20 | * RowTypeLookupSchemaEntry} instance. 21 | */ 22 | private final List> keyColumns; 23 | 24 | /** 25 | * Creates new instance. 26 | * 27 | * @param fieldName field name that this instance represents, matching {@link RowData} column 28 | * name. 29 | * @param fieldGetter {@link RowData.FieldGetter} for data type matching {@link RowData} column 30 | * type that this instance represents. 31 | */ 32 | public RowTypeLookupSchemaEntry(String fieldName, FieldGetter fieldGetter) { 33 | super(fieldName, fieldGetter); 34 | this.keyColumns = new LinkedList<>(); 35 | } 36 | 37 | /** 38 | * Add {@link LookupSchemaEntry} to keyColumns that this {@link RowTypeLookupSchemaEntry} 39 | * represents. 40 | * 41 | * @param lookupSchemaEntry {@link LookupSchemaEntry} to add. 42 | * @return this {@link RowTypeLookupSchemaEntry} instance. 43 | */ 44 | public RowTypeLookupSchemaEntry addLookupEntry(LookupSchemaEntry lookupSchemaEntry) { 45 | this.keyColumns.add(lookupSchemaEntry); 46 | return this; 47 | } 48 | 49 | /** 50 | * Creates collection of {@link LookupArg} that represents every lookup element from {@link 51 | * LookupSchemaEntry} added to this instance. 52 | * 53 | * @param lookupKeyRow Element to get the values from for {@code LookupArg#getArgValue()}. 54 | * @return collection of {@link LookupArg} that represents entire lookup {@link RowData} 55 | */ 56 | @Override 57 | public List convertToLookupArg(RowData lookupKeyRow) { 58 | 59 | RowData nestedRow = (RowData) fieldGetter.getFieldOrNull(lookupKeyRow); 60 | 61 | if (nestedRow == null) { 62 | return Collections.emptyList(); 63 | } 64 | 65 | List lookupArgs = new LinkedList<>(); 66 | for (LookupSchemaEntry lookupSchemaEntry : keyColumns) { 67 | lookupArgs.addAll(lookupSchemaEntry.convertToLookupArg(nestedRow)); 68 | } 69 | 70 | return lookupArgs; 71 | } 72 | 73 | @lombok.Generated 74 | @Override 75 | public String toString() { 76 | return "RowTypeLookupSchemaEntry{" + 77 | "fieldName='" + fieldName + '\'' + 78 | ", fieldGetter=" + fieldGetter + 79 | ", keyColumns=" + keyColumns + 80 | '}'; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/Slf4JHttpLookupPostRequestCallback.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import java.net.http.HttpRequest; 4 | import java.net.http.HttpResponse; 5 | import java.util.List; 6 | import java.util.Map; 7 | import java.util.Map.Entry; 8 | import java.util.StringJoiner; 9 | 10 | import lombok.extern.slf4j.Slf4j; 11 | 12 | import com.getindata.connectors.http.HttpPostRequestCallback; 13 | import com.getindata.connectors.http.internal.utils.ConfigUtils; 14 | 15 | /** 16 | * A {@link HttpPostRequestCallback} that logs pairs of request and response as INFO level 17 | * logs using Slf4j. 18 | * 19 | *

Serving as a default implementation of {@link HttpPostRequestCallback} for 20 | * the {@link HttpLookupTableSource}. 21 | */ 22 | @Slf4j 23 | public class Slf4JHttpLookupPostRequestCallback 24 | implements HttpPostRequestCallback { 25 | 26 | @Override 27 | public void call( 28 | HttpResponse response, 29 | HttpLookupSourceRequestEntry requestEntry, 30 | String endpointUrl, 31 | Map headerMap) { 32 | 33 | HttpRequest httpRequest = requestEntry.getHttpRequest(); 34 | StringJoiner headers = new StringJoiner(";"); 35 | 36 | for (Entry> reqHeaders : httpRequest.headers().map().entrySet()) { 37 | StringJoiner values = new StringJoiner(";"); 38 | for (String value : reqHeaders.getValue()) { 39 | values.add(value); 40 | } 41 | String header = reqHeaders.getKey() + ": [" + values + "]"; 42 | headers.add(header); 43 | } 44 | 45 | if (response == null) { 46 | log.warn("Null Http response for request " + httpRequest.uri().toString()); 47 | 48 | log.info( 49 | "Got response for a request.\n Request:\n URL: {}\n " + 50 | "Method: {}\n Headers: {}\n Params/Body: {}\nResponse: null", 51 | httpRequest.uri().toString(), 52 | httpRequest.method(), 53 | headers, 54 | requestEntry.getLookupQueryInfo() 55 | ); 56 | } else { 57 | log.info( 58 | "Got response for a request.\n Request:\n URL: {}\n " + 59 | "Method: {}\n Headers: {}\n Params/Body: {}\nResponse: {}\n Body: {}", 60 | httpRequest.uri().toString(), 61 | httpRequest.method(), 62 | headers, 63 | requestEntry.getLookupQueryInfo(), 64 | response, 65 | response.body().replaceAll(ConfigUtils.UNIVERSAL_NEW_LINE_REGEXP, "") 66 | ); 67 | } 68 | 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/Slf4jHttpLookupPostRequestCallbackFactory.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import java.util.HashSet; 4 | import java.util.Set; 5 | 6 | import org.apache.flink.configuration.ConfigOption; 7 | 8 | import com.getindata.connectors.http.HttpPostRequestCallback; 9 | import com.getindata.connectors.http.HttpPostRequestCallbackFactory; 10 | 11 | /** 12 | * Factory for creating {@link Slf4JHttpLookupPostRequestCallback}. 13 | */ 14 | public class Slf4jHttpLookupPostRequestCallbackFactory 15 | implements HttpPostRequestCallbackFactory { 16 | 17 | public static final String IDENTIFIER = "slf4j-lookup-logger"; 18 | 19 | @Override 20 | public HttpPostRequestCallback createHttpPostRequestCallback() { 21 | return new Slf4JHttpLookupPostRequestCallback(); 22 | } 23 | 24 | @Override 25 | public String factoryIdentifier() { 26 | return IDENTIFIER; 27 | } 28 | 29 | @Override 30 | public Set> requiredOptions() { 31 | return new HashSet<>(); 32 | } 33 | 34 | @Override 35 | public Set> optionalOptions() { 36 | return new HashSet<>(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/TableSourceHelper.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import java.util.Collections; 4 | import java.util.List; 5 | 6 | import lombok.AccessLevel; 7 | import lombok.NoArgsConstructor; 8 | import org.apache.flink.table.data.GenericRowData; 9 | import org.apache.flink.table.data.RowData; 10 | import org.apache.flink.table.types.DataType; 11 | import org.apache.flink.table.types.logical.LogicalType; 12 | import org.apache.flink.table.types.logical.LogicalTypeRoot; 13 | import org.apache.flink.table.types.logical.utils.LogicalTypeChecks; 14 | import static org.apache.flink.table.types.logical.utils.LogicalTypeChecks.isCompositeType; 15 | 16 | @NoArgsConstructor(access = AccessLevel.PRIVATE) 17 | public final class TableSourceHelper { 18 | 19 | /** 20 | * Returns the first-level field names for the provided {@link DataType}. 21 | * 22 | *

Note: This method returns an empty list for every {@link DataType} that is not a 23 | * composite 24 | * type. 25 | * @param type logical type 26 | * @return List of field names 27 | */ 28 | public static List getFieldNames(LogicalType type) { 29 | 30 | if (type.getTypeRoot() == LogicalTypeRoot.DISTINCT_TYPE) { 31 | return getFieldNames(type.getChildren().get(0)); 32 | } else if (isCompositeType(type)) { 33 | return LogicalTypeChecks.getFieldNames(type); 34 | } 35 | return Collections.emptyList(); 36 | } 37 | 38 | /** 39 | * Builds {@link RowData} object based on provided list of values. 40 | * @param values values to use as {@link RowData} column values. 41 | * @return new {@link RowData} instance. 42 | */ 43 | public static RowData buildGenericRowData(List values) { 44 | GenericRowData row = new GenericRowData(values.size()); 45 | 46 | for (int i = 0; i < values.size(); ++i) { 47 | row.setField(i, values.get(i)); 48 | } 49 | 50 | return row; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/querycreators/ElasticSearchLiteQueryCreator.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup.querycreators; 2 | 3 | import java.util.Collection; 4 | import java.util.stream.Collectors; 5 | 6 | import org.apache.flink.table.data.RowData; 7 | 8 | import com.getindata.connectors.http.LookupArg; 9 | import com.getindata.connectors.http.LookupQueryCreator; 10 | import com.getindata.connectors.http.internal.table.lookup.LookupQueryInfo; 11 | import com.getindata.connectors.http.internal.table.lookup.LookupRow; 12 | 13 | /** 14 | * A {@link LookupQueryCreator} that prepares q 15 | * parameter GET query for ElasticSearch Search API using Lucene query string syntax (in 16 | * first versions of the ElasticSearch called Search 17 | * Lite). 18 | */ 19 | public class ElasticSearchLiteQueryCreator implements LookupQueryCreator { 20 | 21 | private static final String ENCODED_SPACE = "%20"; 22 | private static final String ENCODED_QUOTATION_MARK = "%22"; 23 | 24 | private final LookupRow lookupRow; 25 | 26 | public ElasticSearchLiteQueryCreator(LookupRow lookupRow) { 27 | this.lookupRow = lookupRow; 28 | } 29 | 30 | private static String processLookupArg(LookupArg arg) { 31 | return arg.getArgName() 32 | + ":" 33 | + ENCODED_QUOTATION_MARK 34 | + arg.getArgValue() 35 | + ENCODED_QUOTATION_MARK; 36 | } 37 | 38 | @Override 39 | public LookupQueryInfo createLookupQuery(RowData lookupDataRow) { 40 | Collection lookupArgs = lookupRow.convertToLookupArgs(lookupDataRow); 41 | 42 | var luceneQuery = lookupArgs.stream() 43 | .map(ElasticSearchLiteQueryCreator::processLookupArg) 44 | .collect(Collectors.joining(ENCODED_SPACE + "AND" + ENCODED_SPACE)); 45 | 46 | String lookupQuery = luceneQuery.isEmpty() ? "" : ("q=" + luceneQuery); 47 | 48 | return new LookupQueryInfo(lookupQuery); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/querycreators/ElasticSearchLiteQueryCreatorFactory.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup.querycreators; 2 | 3 | import java.util.Set; 4 | 5 | import org.apache.flink.configuration.ConfigOption; 6 | import org.apache.flink.configuration.ReadableConfig; 7 | import org.apache.flink.table.factories.DynamicTableFactory; 8 | 9 | import com.getindata.connectors.http.LookupQueryCreator; 10 | import com.getindata.connectors.http.LookupQueryCreatorFactory; 11 | import com.getindata.connectors.http.internal.table.lookup.LookupRow; 12 | 13 | 14 | /** 15 | * Factory for creating {@link ElasticSearchLiteQueryCreator}. 16 | */ 17 | public class ElasticSearchLiteQueryCreatorFactory implements LookupQueryCreatorFactory { 18 | 19 | public static final String IDENTIFIER = "elasticsearch-lite"; 20 | 21 | @Override 22 | public LookupQueryCreator createLookupQueryCreator( 23 | ReadableConfig readableConfig, 24 | LookupRow lookupRow, 25 | DynamicTableFactory.Context dynamicTableFactoryContext 26 | ) { 27 | return new ElasticSearchLiteQueryCreator(lookupRow); 28 | } 29 | 30 | @Override 31 | public String factoryIdentifier() { 32 | return IDENTIFIER; 33 | } 34 | 35 | @Override 36 | public Set> requiredOptions() { 37 | return Set.of(); 38 | } 39 | 40 | @Override 41 | public Set> optionalOptions() { 42 | return Set.of(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/querycreators/GenericGetQueryCreator.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup.querycreators; 2 | 3 | import java.nio.charset.StandardCharsets; 4 | import java.util.Collection; 5 | import java.util.stream.Collectors; 6 | 7 | import org.apache.flink.table.data.RowData; 8 | 9 | import com.getindata.connectors.http.LookupArg; 10 | import com.getindata.connectors.http.LookupQueryCreator; 11 | import com.getindata.connectors.http.internal.table.lookup.LookupQueryInfo; 12 | import com.getindata.connectors.http.internal.table.lookup.LookupRow; 13 | import com.getindata.connectors.http.internal.utils.uri.NameValuePair; 14 | import com.getindata.connectors.http.internal.utils.uri.URLEncodedUtils; 15 | 16 | /** 17 | * A {@link LookupQueryCreator} that builds an "ordinary" GET query, i.e. adds 18 | * joinColumn1=value1&joinColumn2=value2&... to the URI of the endpoint. 19 | */ 20 | public class GenericGetQueryCreator implements LookupQueryCreator { 21 | 22 | private final LookupRow lookupRow; 23 | 24 | public GenericGetQueryCreator(LookupRow lookupRow) { 25 | this.lookupRow = lookupRow; 26 | } 27 | 28 | @Override 29 | public LookupQueryInfo createLookupQuery(RowData lookupDataRow) { 30 | 31 | Collection lookupArgs = lookupRow.convertToLookupArgs(lookupDataRow); 32 | 33 | String lookupQuery = 34 | URLEncodedUtils.format( 35 | lookupArgs.stream() 36 | .map(arg -> new NameValuePair(arg.getArgName(), arg.getArgValue())) 37 | .collect(Collectors.toList()), 38 | StandardCharsets.UTF_8); 39 | 40 | return new LookupQueryInfo(lookupQuery); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/querycreators/GenericGetQueryCreatorFactory.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup.querycreators; 2 | 3 | import java.util.Set; 4 | 5 | import org.apache.flink.configuration.ConfigOption; 6 | import org.apache.flink.configuration.ReadableConfig; 7 | import org.apache.flink.table.factories.DynamicTableFactory; 8 | 9 | import com.getindata.connectors.http.LookupQueryCreator; 10 | import com.getindata.connectors.http.LookupQueryCreatorFactory; 11 | import com.getindata.connectors.http.internal.table.lookup.LookupRow; 12 | 13 | 14 | /** 15 | * Factory for creating {@link GenericGetQueryCreator}. 16 | */ 17 | public class GenericGetQueryCreatorFactory implements LookupQueryCreatorFactory { 18 | 19 | public static final String IDENTIFIER = "generic-get-query"; 20 | 21 | @Override 22 | public LookupQueryCreator createLookupQueryCreator( 23 | ReadableConfig readableConfig, 24 | LookupRow lookupRow, 25 | DynamicTableFactory.Context dynamicTableFactoryContext) { 26 | return new GenericGetQueryCreator(lookupRow); 27 | } 28 | 29 | @Override 30 | public String factoryIdentifier() { 31 | return IDENTIFIER; 32 | } 33 | 34 | @Override 35 | public Set> requiredOptions() { 36 | return Set.of(); 37 | } 38 | 39 | @Override 40 | public Set> optionalOptions() { 41 | return Set.of(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/querycreators/GenericJsonQueryCreator.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup.querycreators; 2 | 3 | import java.nio.charset.StandardCharsets; 4 | 5 | import org.apache.flink.api.common.serialization.SerializationSchema; 6 | import org.apache.flink.table.data.RowData; 7 | import org.apache.flink.util.FlinkRuntimeException; 8 | 9 | import com.getindata.connectors.http.LookupQueryCreator; 10 | import com.getindata.connectors.http.internal.table.lookup.LookupQueryInfo; 11 | import com.getindata.connectors.http.internal.utils.SerializationSchemaUtils; 12 | 13 | /** 14 | * A {@link LookupQueryCreator} that builds Json based body for REST requests, i.e. adds 15 | */ 16 | public class GenericJsonQueryCreator implements LookupQueryCreator { 17 | 18 | /** 19 | * The {@link SerializationSchema} to serialize {@link RowData} object. 20 | */ 21 | private final SerializationSchema jsonSerialization; 22 | 23 | private boolean schemaOpened = false; 24 | 25 | public GenericJsonQueryCreator(SerializationSchema jsonSerialization) { 26 | 27 | this.jsonSerialization = jsonSerialization; 28 | } 29 | 30 | /** 31 | * Creates a Jason string from given {@link RowData}. 32 | * 33 | * @param lookupDataRow {@link RowData} to serialize into Json string. 34 | * @return Json string created from lookupDataRow argument. 35 | */ 36 | @Override 37 | public LookupQueryInfo createLookupQuery(RowData lookupDataRow) { 38 | checkOpened(); 39 | String lookupQuery = 40 | new String(jsonSerialization.serialize(lookupDataRow), StandardCharsets.UTF_8); 41 | 42 | return new LookupQueryInfo(lookupQuery); 43 | } 44 | 45 | private void checkOpened() { 46 | if (!schemaOpened) { 47 | try { 48 | jsonSerialization.open( 49 | SerializationSchemaUtils 50 | .createSerializationInitContext(GenericJsonQueryCreator.class)); 51 | } catch (Exception e) { 52 | throw new FlinkRuntimeException( 53 | "Failed to initialize serialization schema for GenericJsonQueryCreatorFactory.", 54 | e); 55 | } 56 | schemaOpened = true; 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/querycreators/ObjectMapperAdapter.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup.querycreators; 2 | 3 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationFeature; 4 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.MapperFeature; 5 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; 6 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.SerializationFeature; 7 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.json.JsonMapper; 8 | 9 | /** 10 | * Centralizes the use of {@link ObjectMapper}. 11 | */ 12 | public class ObjectMapperAdapter { 13 | private static final ObjectMapper MAPPER = initialize(); 14 | 15 | private static ObjectMapper initialize() { 16 | final ObjectMapper mapper = JsonMapper.builder() 17 | .configure(MapperFeature.USE_STD_BEAN_NAMING, 18 | false).build(); 19 | mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); 20 | mapper.disable(SerializationFeature.WRITE_DATES_WITH_ZONE_ID); 21 | mapper.disable(DeserializationFeature.ADJUST_DATES_TO_CONTEXT_TIME_ZONE); 22 | return mapper; 23 | } 24 | 25 | public static ObjectMapper instance() { 26 | return MAPPER; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/lookup/querycreators/QueryFormatAwareConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup.querycreators; 2 | 3 | import java.util.Optional; 4 | 5 | import org.apache.flink.configuration.ConfigOption; 6 | import org.apache.flink.configuration.Configuration; 7 | import org.apache.flink.table.factories.SerializationFormatFactory; 8 | 9 | /** 10 | * An internal extension of Flink's {@link Configuration} class. This implementation uses {@link 11 | * PrefixedConfigOption} internally to decorate Flink's {@link ConfigOption} used for {@link 12 | * Configuration#getOptional(ConfigOption)} method. 13 | */ 14 | class QueryFormatAwareConfiguration extends Configuration { 15 | 16 | /** 17 | * Format name for {@link SerializationFormatFactory} identifier used as {@code 18 | * HttpLookupConnectorOptions#LOOKUP_REQUEST_FORMAT}. 19 | *

20 | * This will be used as prefix parameter for {@link PrefixedConfigOption}. 21 | */ 22 | private final String queryFormatName; 23 | 24 | QueryFormatAwareConfiguration(String queryFormatName, Configuration other) { 25 | super(other); 26 | this.queryFormatName = 27 | (queryFormatName.endsWith(".")) ? queryFormatName : queryFormatName + "."; 28 | } 29 | 30 | /** 31 | * Returns value for {@link ConfigOption} option which key is prefixed with "queryFormatName" 32 | * 33 | * @param option option which key will be prefixed with queryFormatName. 34 | * @return value for option after adding queryFormatName prefix 35 | */ 36 | @Override 37 | public Optional getOptional(ConfigOption option) { 38 | PrefixedConfigOption prefixedConfigOption = 39 | new PrefixedConfigOption<>(queryFormatName, option); 40 | return super.getOptional(prefixedConfigOption.getConfigOption()); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/sink/HttpDynamicSinkConnectorOptions.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.sink; 2 | 3 | import org.apache.flink.configuration.ConfigOption; 4 | import org.apache.flink.configuration.ConfigOptions; 5 | 6 | import static com.getindata.connectors.http.internal.config.HttpConnectorConfigConstants.SINK_REQUEST_CALLBACK_IDENTIFIER; 7 | 8 | /** 9 | * Table API options for {@link HttpDynamicSink}. 10 | */ 11 | public class HttpDynamicSinkConnectorOptions { 12 | 13 | public static final ConfigOption URL = 14 | ConfigOptions.key("url").stringType().noDefaultValue() 15 | .withDescription("The HTTP endpoint URL."); 16 | 17 | public static final ConfigOption INSERT_METHOD = 18 | ConfigOptions.key("insert-method") 19 | .stringType() 20 | .defaultValue("POST") 21 | .withDescription("Method used for requests built from SQL's INSERT."); 22 | 23 | public static final ConfigOption REQUEST_CALLBACK_IDENTIFIER = 24 | ConfigOptions.key(SINK_REQUEST_CALLBACK_IDENTIFIER) 25 | .stringType() 26 | .defaultValue(Slf4jHttpPostRequestCallbackFactory.IDENTIFIER); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/sink/Slf4jHttpPostRequestCallback.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.sink; 2 | 3 | import java.net.http.HttpResponse; 4 | import java.nio.charset.StandardCharsets; 5 | import java.util.Map; 6 | import java.util.stream.Collectors; 7 | 8 | import lombok.extern.slf4j.Slf4j; 9 | 10 | import com.getindata.connectors.http.HttpPostRequestCallback; 11 | import com.getindata.connectors.http.internal.sink.httpclient.HttpRequest; 12 | import com.getindata.connectors.http.internal.utils.ConfigUtils; 13 | 14 | /** 15 | * A {@link HttpPostRequestCallback} that logs pairs of request and response as INFO level 16 | * logs using Slf4j. 17 | * 18 | *

Serving as a default implementation of {@link HttpPostRequestCallback} for 19 | * the {@link HttpDynamicSink}. 20 | */ 21 | @Slf4j 22 | public class Slf4jHttpPostRequestCallback implements HttpPostRequestCallback { 23 | 24 | @Override 25 | public void call( 26 | HttpResponse response, 27 | HttpRequest requestEntry, 28 | String endpointUrl, 29 | Map headerMap) { 30 | 31 | String requestBody = requestEntry.getElements().stream() 32 | .map(element -> new String(element, StandardCharsets.UTF_8)) 33 | .collect(Collectors.joining()); 34 | 35 | if (response == null) { 36 | log.info( 37 | "Got response for a request.\n Request:\n " + 38 | "Method: {}\n Body: {}\n Response: null", 39 | requestEntry.getMethod(), 40 | requestBody 41 | ); 42 | } else { 43 | log.info( 44 | "Got response for a request.\n Request:\n " + 45 | "Method: {}\n Body: {}\n Response: {}\n Body: {}", 46 | requestEntry.method, 47 | requestBody, 48 | response, 49 | response.body().replaceAll(ConfigUtils.UNIVERSAL_NEW_LINE_REGEXP, "") 50 | ); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/table/sink/Slf4jHttpPostRequestCallbackFactory.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.sink; 2 | 3 | import java.util.HashSet; 4 | import java.util.Set; 5 | 6 | import org.apache.flink.configuration.ConfigOption; 7 | 8 | import com.getindata.connectors.http.HttpPostRequestCallback; 9 | import com.getindata.connectors.http.HttpPostRequestCallbackFactory; 10 | import com.getindata.connectors.http.internal.sink.httpclient.HttpRequest; 11 | 12 | /** 13 | * Factory for creating {@link Slf4jHttpPostRequestCallback}. 14 | */ 15 | public class Slf4jHttpPostRequestCallbackFactory 16 | implements HttpPostRequestCallbackFactory { 17 | 18 | public static final String IDENTIFIER = "slf4j-logger"; 19 | 20 | @Override 21 | public HttpPostRequestCallback createHttpPostRequestCallback() { 22 | return new Slf4jHttpPostRequestCallback(); 23 | } 24 | 25 | @Override 26 | public String factoryIdentifier() { 27 | return IDENTIFIER; 28 | } 29 | 30 | @Override 31 | public Set> requiredOptions() { 32 | return new HashSet<>(); 33 | } 34 | 35 | @Override 36 | public Set> optionalOptions() { 37 | return new HashSet<>(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/utils/ExceptionUtils.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.utils; 2 | 3 | import java.io.IOException; 4 | import java.io.PrintWriter; 5 | import java.io.StringWriter; 6 | import java.io.UncheckedIOException; 7 | 8 | import lombok.AccessLevel; 9 | import lombok.NoArgsConstructor; 10 | 11 | @NoArgsConstructor(access = AccessLevel.NONE) 12 | public final class ExceptionUtils { 13 | 14 | public static String stringifyException(Throwable e) { 15 | try (StringWriter stm = new StringWriter(); 16 | PrintWriter wrt = new PrintWriter(stm)) { 17 | 18 | e.printStackTrace(wrt); 19 | wrt.close(); 20 | return stm.toString(); 21 | 22 | } catch (IOException ioException) { 23 | throw new UncheckedIOException(ioException); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/utils/ProxyConfig.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.utils; 2 | 3 | import java.net.Authenticator; 4 | import java.net.PasswordAuthentication; 5 | import java.util.Optional; 6 | 7 | import lombok.Getter; 8 | 9 | @Getter 10 | public class ProxyConfig { 11 | 12 | private final String host; 13 | 14 | private final int port; 15 | 16 | private final Optional authenticator; 17 | 18 | public ProxyConfig(String host, int port, Optional proxyUsername, Optional proxyPassword) { 19 | this.host = host; 20 | this.port = port; 21 | 22 | if(proxyUsername.isPresent() && proxyPassword.isPresent()){ 23 | this.authenticator = Optional.of(new Authenticator() { 24 | @Override 25 | protected PasswordAuthentication getPasswordAuthentication() { 26 | if (getRequestorType().equals(RequestorType.PROXY) && getRequestingHost().equalsIgnoreCase(host)) { 27 | return new PasswordAuthentication(proxyUsername.get(), 28 | proxyPassword.get().toCharArray()); 29 | } else { 30 | return null; 31 | } 32 | } 33 | }); 34 | } else { 35 | this.authenticator = Optional.empty(); 36 | } 37 | 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/utils/SerializationSchemaUtils.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.utils; 2 | 3 | import org.apache.flink.metrics.MetricGroup; 4 | import org.apache.flink.metrics.groups.UnregisteredMetricsGroup; 5 | import org.apache.flink.util.SimpleUserCodeClassLoader; 6 | import org.apache.flink.util.UserCodeClassLoader; 7 | 8 | public final class SerializationSchemaUtils { 9 | 10 | private SerializationSchemaUtils() { 11 | 12 | } 13 | 14 | public static org.apache.flink.api.common.serialization.SerializationSchema 15 | .InitializationContext createSerializationInitContext(Class classForClassLoader) { 16 | 17 | return new org.apache.flink.api.common.serialization.SerializationSchema 18 | .InitializationContext() { 19 | 20 | @Override 21 | public MetricGroup getMetricGroup() { 22 | return new UnregisteredMetricsGroup(); 23 | } 24 | 25 | @Override 26 | public UserCodeClassLoader getUserCodeClassLoader() { 27 | return SimpleUserCodeClassLoader.create(classForClassLoader.getClassLoader()); 28 | } 29 | }; 30 | } 31 | 32 | public static org.apache.flink.api.common.serialization.DeserializationSchema 33 | .InitializationContext createDeserializationInitContext(Class classForClassLoader) { 34 | 35 | return new org.apache.flink.api.common.serialization.DeserializationSchema 36 | .InitializationContext() { 37 | 38 | @Override 39 | public MetricGroup getMetricGroup() { 40 | return new UnregisteredMetricsGroup(); 41 | } 42 | 43 | @Override 44 | public UserCodeClassLoader getUserCodeClassLoader() { 45 | return SimpleUserCodeClassLoader.create(classForClassLoader.getClassLoader()); 46 | } 47 | }; 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/utils/SynchronizedSerializationSchema.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.utils; 2 | 3 | import org.apache.flink.api.common.serialization.SerializationSchema; 4 | 5 | /** 6 | * Decorator which ensures that underlying SerializationSchema is called in thread-safe way. 7 | * 8 | * @param type 9 | */ 10 | public class SynchronizedSerializationSchema implements SerializationSchema { 11 | 12 | private final SerializationSchema delegate; 13 | 14 | public SynchronizedSerializationSchema(SerializationSchema delegate) { 15 | this.delegate = delegate; 16 | } 17 | 18 | @Override 19 | public void open(InitializationContext context) throws Exception { 20 | doOpen(context); 21 | } 22 | 23 | private synchronized void doOpen(InitializationContext context) throws Exception { 24 | this.delegate.open(context); 25 | } 26 | 27 | @Override 28 | public byte[] serialize(T element) { 29 | return syncSerialize(element); 30 | } 31 | 32 | private synchronized byte[] syncSerialize(T element) { 33 | return delegate.serialize(element); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/utils/ThreadUtils.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.utils; 2 | 3 | import java.lang.Thread.UncaughtExceptionHandler; 4 | 5 | import lombok.AccessLevel; 6 | import lombok.NoArgsConstructor; 7 | import lombok.extern.slf4j.Slf4j; 8 | 9 | import static com.getindata.connectors.http.internal.utils.ExceptionUtils.stringifyException; 10 | 11 | @Slf4j 12 | @NoArgsConstructor(access = AccessLevel.NONE) 13 | public final class ThreadUtils { 14 | 15 | public static final UncaughtExceptionHandler LOGGING_EXCEPTION_HANDLER = 16 | (t, e) -> log.warn("Thread:" + t + " exited with Exception:" + stringifyException(e)); 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/utils/uri/NameValuePair.java: -------------------------------------------------------------------------------- 1 | /* 2 | * ==================================================================== 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * ==================================================================== 20 | * 21 | * This software consists of voluntary contributions made by many 22 | * individuals on behalf of the Apache Software Foundation. For more 23 | * information on the Apache Software Foundation, please see 24 | * . 25 | * 26 | * ============================= NOTE ================================= 27 | * This code has been copied from 28 | * https://github.com/apache/httpcomponents-client/tree/rel/v4.5.13 29 | * and it was changed to use in this project. 30 | * ==================================================================== 31 | */ 32 | 33 | package com.getindata.connectors.http.internal.utils.uri; 34 | 35 | import lombok.Data; 36 | import org.apache.flink.util.Preconditions; 37 | 38 | @Data 39 | public class NameValuePair { 40 | 41 | private final String name; 42 | 43 | private final String value; 44 | 45 | /** 46 | * Default Constructor taking a name and a value. The value may be null. 47 | * 48 | * @param name The name. 49 | * @param value The value. 50 | */ 51 | public NameValuePair(final String name, final String value) { 52 | super(); 53 | this.name = Preconditions.checkNotNull(name, "Name may not be null"); 54 | this.value = value; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/getindata/connectors/http/internal/utils/uri/ParserCursor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * ==================================================================== 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * ==================================================================== 20 | * 21 | * This software consists of voluntary contributions made by many 22 | * individuals on behalf of the Apache Software Foundation. For more 23 | * information on the Apache Software Foundation, please see 24 | * . 25 | * 26 | * ============================= NOTE ================================= 27 | * This code has been copied from 28 | * https://github.com/apache/httpcomponents-client/tree/rel/v4.5.13 29 | * and it was changed to use in this project. 30 | * ==================================================================== 31 | */ 32 | 33 | package com.getindata.connectors.http.internal.utils.uri; 34 | 35 | import lombok.Getter; 36 | 37 | /** 38 | * This class represents a context of a parsing operation: 39 | *

    40 | *
  • the current position the parsing operation is expected to start at
    • 41 | *
  • the bounds limiting the scope of the parsing operation
    • 42 | *
43 | */ 44 | @Getter 45 | class ParserCursor { 46 | 47 | private final int lowerBound; 48 | 49 | private final int upperBound; 50 | 51 | private int pos; 52 | 53 | ParserCursor(final int lowerBound, final int upperBound) { 54 | super(); 55 | if (lowerBound < 0) { 56 | throw new IndexOutOfBoundsException("Lower bound cannot be negative"); 57 | } 58 | if (lowerBound > upperBound) { 59 | throw new IndexOutOfBoundsException("Lower bound cannot be greater then upper bound"); 60 | } 61 | this.lowerBound = lowerBound; 62 | this.upperBound = upperBound; 63 | this.pos = lowerBound; 64 | } 65 | 66 | void updatePos(final int pos) { 67 | if (pos < this.lowerBound) { 68 | throw new IndexOutOfBoundsException( 69 | "pos: " + pos + " < lowerBound: " + this.lowerBound); 70 | } 71 | if (pos > this.upperBound) { 72 | throw new IndexOutOfBoundsException( 73 | "pos: " + pos + " > upperBound: " + this.upperBound); 74 | } 75 | this.pos = pos; 76 | } 77 | 78 | boolean atEnd() { 79 | return this.pos >= this.upperBound; 80 | } 81 | 82 | @Override 83 | public String toString() { 84 | return "[" 85 | + this.lowerBound 86 | + '>' 87 | + this.pos 88 | + '>' 89 | + this.upperBound 90 | + ']'; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory: -------------------------------------------------------------------------------- 1 | com.getindata.connectors.http.internal.table.lookup.HttpLookupTableSourceFactory 2 | com.getindata.connectors.http.internal.table.lookup.querycreators.ElasticSearchLiteQueryCreatorFactory 3 | com.getindata.connectors.http.internal.table.lookup.querycreators.GenericGetQueryCreatorFactory 4 | com.getindata.connectors.http.internal.table.lookup.querycreators.GenericJsonQueryCreatorFactory 5 | com.getindata.connectors.http.internal.table.lookup.Slf4jHttpLookupPostRequestCallbackFactory 6 | com.getindata.connectors.http.internal.table.sink.HttpDynamicTableSinkFactory 7 | com.getindata.connectors.http.internal.table.sink.Slf4jHttpPostRequestCallbackFactory 8 | com.getindata.connectors.http.internal.table.lookup.querycreators.GenericJsonAndUrlQueryCreatorFactory -------------------------------------------------------------------------------- /src/test/java/com/getindata/StreamTableJob.java: -------------------------------------------------------------------------------- 1 | package com.getindata; 2 | 3 | import org.apache.flink.api.common.restartstrategy.RestartStrategies; 4 | import org.apache.flink.api.java.utils.ParameterTool; 5 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 6 | import org.apache.flink.table.api.Table; 7 | import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; 8 | 9 | public class StreamTableJob { 10 | 11 | public static void main(String[] args) { 12 | 13 | ParameterTool parameters = ParameterTool.fromSystemProperties(); 14 | parameters = parameters.mergeWith(ParameterTool.fromArgs(args)); 15 | 16 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 17 | // env.enableCheckpointing(5000); 18 | env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1000, 1000)); 19 | env.setParallelism(1); 20 | env.disableOperatorChaining(); 21 | env.getConfig().setGlobalJobParameters(parameters); 22 | 23 | StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env); 24 | 25 | tableEnv.executeSql( 26 | "CREATE TABLE Orders (id STRING, id2 STRING, proc_time AS PROCTIME())" 27 | + " WITH (" 28 | + "'connector' = 'datagen', 'rows-per-second' = '1', 'fields.id.kind' = 'sequence'," 29 | + " 'fields.id.start' = '1', 'fields.id.end' = '120'," 30 | + " 'fields.id2.kind' = 'sequence', 'fields.id2.start' = '2'," 31 | + " 'fields.id2.end' = '120')" 32 | ); 33 | tableEnv.executeSql( 34 | "CREATE TABLE Customers (id STRING, id2 STRING, msg STRING, uuid STRING, isActive STRING, balance STRING) WITH ('connector' = 'rest-lookup', 'url' = 'http://localhost:8080/client', " 35 | + "'asyncPolling' = 'true', " 36 | + "'field.isActive.path' = '$.details.isActive', " 37 | + "'field.balance.path' = '$.details.nestedDetails.balance')"); 38 | 39 | Table resultTable = 40 | tableEnv.sqlQuery( 41 | "SELECT o.id, o.id2, c.msg, c.uuid, c.isActive, c.balance FROM Orders AS o " 42 | + "JOIN Customers FOR SYSTEM_TIME AS OF o.proc_time AS c " 43 | + "ON o.id = c.id AND o.id2 = c.id2"); 44 | 45 | /* DataStream rowDataStream = tableEnv.toDataStream(resultTable); 46 | rowDataStream.print();*/ 47 | 48 | // Table result = tableEnv.sqlQuery("SELECT * FROM Orders"); 49 | // Table result = tableEnv.sqlQuery("SELECT * FROM Customers"); 50 | // Table result = tableEnv.sqlQuery("SELECT * FROM T WHERE T.id > 10"); 51 | 52 | resultTable.execute().print(); 53 | 54 | // env.execute(); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/ExceptionUtilsTest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.junit.jupiter.api.Test; 5 | import static org.assertj.core.api.Assertions.assertThat; 6 | 7 | import com.getindata.connectors.http.internal.utils.ExceptionUtils; 8 | 9 | @Slf4j 10 | class ExceptionUtilsTest { 11 | 12 | @Test 13 | void shouldConvertStackTrace() { 14 | String stringifyException = 15 | ExceptionUtils.stringifyException(new RuntimeException("Test Exception")); 16 | assertThat(stringifyException).contains("java.lang.RuntimeException: Test Exception"); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/TestHelper.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http; 2 | 3 | import java.net.URI; 4 | import java.nio.file.Files; 5 | import java.nio.file.Path; 6 | import java.util.Arrays; 7 | import java.util.Objects; 8 | 9 | import lombok.AccessLevel; 10 | import lombok.NoArgsConstructor; 11 | import org.junit.jupiter.api.Assertions; 12 | import static org.assertj.core.api.Assertions.assertThat; 13 | 14 | @NoArgsConstructor(access = AccessLevel.PRIVATE) 15 | public final class TestHelper { 16 | 17 | private static final TestHelper INSTANCE = new TestHelper(); 18 | 19 | public static String readTestFile(String pathToFile) { 20 | try { 21 | URI uri = Objects.requireNonNull(INSTANCE.getClass().getResource(pathToFile)).toURI(); 22 | return Files.readString(Path.of(uri)); 23 | } catch (Exception e) { 24 | throw new RuntimeException(e); 25 | } 26 | } 27 | 28 | public static void assertPropertyArray( 29 | String[] headerArray, 30 | String propertyName, 31 | String expectedValue) { 32 | // important thing is that we have property followed by its value. 33 | for (int i = 0; i < headerArray.length; i++) { 34 | if (headerArray[i].equals(propertyName)) { 35 | assertThat(headerArray[i + 1]) 36 | .withFailMessage("Property Array does not contain property name, value pairs.") 37 | .isEqualTo(expectedValue); 38 | return; 39 | } 40 | } 41 | Assertions.fail( 42 | String.format( 43 | "Missing property name [%s] in header array %s.", 44 | propertyName, 45 | Arrays.toString(headerArray)) 46 | ); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/TestLookupPostRequestCallbackFactory.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http; 2 | 3 | import java.util.HashSet; 4 | import java.util.Set; 5 | 6 | import org.apache.flink.configuration.ConfigOption; 7 | 8 | import com.getindata.connectors.http.internal.table.lookup.HttpLookupSourceRequestEntry; 9 | 10 | public class TestLookupPostRequestCallbackFactory 11 | implements HttpPostRequestCallbackFactory { 12 | 13 | public static final String TEST_LOOKUP_POST_REQUEST_CALLBACK_IDENT = 14 | "test-lookup-request-callback"; 15 | 16 | @Override 17 | public HttpPostRequestCallback createHttpPostRequestCallback() { 18 | return new HttpPostRequestCallbackFactoryTest.TestLookupPostRequestCallback(); 19 | } 20 | 21 | @Override 22 | public String factoryIdentifier() { return TEST_LOOKUP_POST_REQUEST_CALLBACK_IDENT; } 23 | 24 | @Override 25 | public Set> requiredOptions() { return new HashSet<>(); } 26 | 27 | @Override 28 | public Set> optionalOptions() { return new HashSet<>(); } 29 | } 30 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/TestPostRequestCallbackFactory.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http; 2 | 3 | import java.util.HashSet; 4 | import java.util.Set; 5 | 6 | import org.apache.flink.configuration.ConfigOption; 7 | 8 | import com.getindata.connectors.http.internal.sink.httpclient.HttpRequest; 9 | 10 | public class TestPostRequestCallbackFactory implements HttpPostRequestCallbackFactory { 11 | 12 | public static final String TEST_POST_REQUEST_CALLBACK_IDENT = "test-request-callback"; 13 | 14 | @Override 15 | public HttpPostRequestCallback createHttpPostRequestCallback() { 16 | return new HttpPostRequestCallbackFactoryTest.TestPostRequestCallback(); 17 | } 18 | 19 | @Override 20 | public String factoryIdentifier() { return TEST_POST_REQUEST_CALLBACK_IDENT; } 21 | 22 | @Override 23 | public Set> requiredOptions() { return new HashSet<>(); } 24 | 25 | @Override 26 | public Set> optionalOptions() { return new HashSet<>(); } 27 | } 28 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/app/HttpStubApp.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.app; 2 | 3 | import com.github.tomakehurst.wiremock.WireMockServer; 4 | import com.github.tomakehurst.wiremock.core.WireMockConfiguration; 5 | import com.github.tomakehurst.wiremock.stubbing.StubMapping; 6 | import lombok.extern.slf4j.Slf4j; 7 | import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; 8 | import static com.github.tomakehurst.wiremock.client.WireMock.get; 9 | import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; 10 | 11 | @Slf4j 12 | public class HttpStubApp { 13 | 14 | private static final String URL = "/client"; 15 | 16 | private static WireMockServer wireMockServer; 17 | 18 | @SuppressWarnings("unchecked") 19 | public static void main(String[] args) { 20 | wireMockServer = 21 | new WireMockServer( 22 | WireMockConfiguration.wireMockConfig().port(8080).extensions(JsonTransform.class)); 23 | wireMockServer.start(); 24 | 25 | wireMockServer.addStubMapping(setupServerStub()); 26 | } 27 | 28 | private static StubMapping setupServerStub() { 29 | return wireMockServer.stubFor( 30 | get(urlPathEqualTo(URL)) 31 | .willReturn( 32 | aResponse() 33 | .withTransformers(JsonTransform.NAME))); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/BasicAuthHeaderValuePreprocessorTest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal; 2 | 3 | import org.junit.jupiter.params.ParameterizedTest; 4 | import org.junit.jupiter.params.provider.CsvSource; 5 | import static org.assertj.core.api.Assertions.assertThat; 6 | 7 | class BasicAuthHeaderValuePreprocessorTest { 8 | 9 | @ParameterizedTest 10 | @CsvSource({ 11 | "user:password, Basic dXNlcjpwYXNzd29yZA==, false", 12 | "Basic dXNlcjpwYXNzd29yZA==, Basic dXNlcjpwYXNzd29yZA==, false", 13 | "abc123, abc123, true", 14 | "Basic dXNlcjpwYXNzd29yZA==, Basic dXNlcjpwYXNzd29yZA==, true", 15 | "Bearer dXNlcjpwYXNzd29yZA==, Bearer dXNlcjpwYXNzd29yZA==, true" 16 | }) 17 | public void testAuthorizationHeaderPreprocess( 18 | String headerRawValue, 19 | String expectedHeaderValue, 20 | boolean useRawAuthHeader) { 21 | BasicAuthHeaderValuePreprocessor preprocessor = 22 | new BasicAuthHeaderValuePreprocessor(useRawAuthHeader); 23 | String headerValue = preprocessor.preprocessHeaderValue(headerRawValue); 24 | assertThat(headerValue).isEqualTo(expectedHeaderValue); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/ComposeHeaderPreprocessorTest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal; 2 | 3 | import org.junit.jupiter.params.ParameterizedTest; 4 | import org.junit.jupiter.params.provider.CsvSource; 5 | import static org.assertj.core.api.Assertions.assertThat; 6 | 7 | public class ComposeHeaderPreprocessorTest { 8 | @ParameterizedTest 9 | @CsvSource({ 10 | "a, a", 11 | "a123, a123", 12 | "user:password, user:password", 13 | "Basic dXNlcjpwYXNzd29yZA==, Basic dXNlcjpwYXNzd29yZA==" 14 | }) 15 | public void testNoPreprocessors(String rawValue, String expectedValue) { 16 | var noPreprocessorHeaderPreprocessor = new ComposeHeaderPreprocessor(null); 17 | var obtainedValue = noPreprocessorHeaderPreprocessor 18 | .preprocessValueForHeader("someHeader", rawValue); 19 | assertThat(obtainedValue).isEqualTo(expectedValue); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/HttpsConnectionTestBase.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal; 2 | 3 | import java.util.Properties; 4 | 5 | import com.github.tomakehurst.wiremock.WireMockServer; 6 | 7 | import com.getindata.connectors.http.HttpPostRequestCallback; 8 | import com.getindata.connectors.http.internal.sink.httpclient.HttpRequest; 9 | import com.getindata.connectors.http.internal.table.sink.Slf4jHttpPostRequestCallback; 10 | import com.getindata.connectors.http.internal.utils.HttpHeaderUtils; 11 | 12 | public abstract class HttpsConnectionTestBase { 13 | 14 | public static final int SERVER_PORT = 9090; 15 | 16 | public static final int HTTPS_SERVER_PORT = 8443; 17 | 18 | protected static final String ENDPOINT = "/myendpoint"; 19 | 20 | protected static final String CERTS_PATH = "src/test/resources/security/certs/"; 21 | 22 | protected static final String SERVER_KEYSTORE_PATH = 23 | "src/test/resources/security/certs/serverKeyStore.jks"; 24 | 25 | protected static final String SERVER_TRUSTSTORE_PATH = 26 | "src/test/resources/security/certs/serverTrustStore.jks"; 27 | 28 | protected WireMockServer wireMockServer; 29 | 30 | protected Properties properties; 31 | 32 | protected HeaderPreprocessor headerPreprocessor; 33 | 34 | protected HttpPostRequestCallback postRequestCallback = 35 | new Slf4jHttpPostRequestCallback(); 36 | 37 | public void setUp() { 38 | this.properties = new Properties(); 39 | this.headerPreprocessor = HttpHeaderUtils.createBasicAuthorizationHeaderPreprocessor(); 40 | } 41 | 42 | public void tearDown() { 43 | if (wireMockServer != null) { 44 | wireMockServer.stop(); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/config/ConfigExceptionTest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.config; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import static org.assertj.core.api.Assertions.assertThat; 5 | 6 | class ConfigExceptionTest { 7 | 8 | @Test 9 | public void testTemplateMessageWithNull() { 10 | ConfigException exception = new ConfigException("myProp", -1, null); 11 | assertThat(exception.getMessage()).isEqualTo("Invalid value -1 for configuration myProp"); 12 | } 13 | 14 | @Test 15 | public void testTemplateMessage() { 16 | ConfigException exception = new ConfigException("myProp", -1, "Invalid test value."); 17 | assertThat(exception.getMessage()) 18 | .isEqualTo("Invalid value -1 for configuration myProp: Invalid test value."); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/retry/RetryConfigProviderTest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.retry; 2 | 3 | import java.util.stream.IntStream; 4 | 5 | import org.apache.flink.configuration.Configuration; 6 | import org.junit.jupiter.api.Test; 7 | import static org.junit.jupiter.api.Assertions.assertEquals; 8 | import static org.junit.jupiter.api.Assertions.assertThrows; 9 | import static org.mockito.Mockito.mock; 10 | import static org.mockito.Mockito.mockStatic; 11 | 12 | class RetryConfigProviderTest { 13 | 14 | @Test 15 | void verifyFixedDelayRetryConfig() { 16 | var config = new Configuration(); 17 | config.setString("gid.connector.http.source.lookup.retry-strategy.type", "fixed-delay"); 18 | config.setString("gid.connector.http.source.lookup.retry-strategy.fixed-delay.delay", "10s"); 19 | config.setInteger("lookup.max-retries", 12); 20 | 21 | var retryConfig = RetryConfigProvider.create(config); 22 | 23 | assertEquals(13, retryConfig.getMaxAttempts()); 24 | IntStream.range(1, 12).forEach(attempt -> 25 | assertEquals(10000, retryConfig.getIntervalFunction().apply(attempt)) 26 | ); 27 | } 28 | 29 | @Test 30 | void verifyExponentialDelayConfig() { 31 | var config = new Configuration(); 32 | config.setString("gid.connector.http.source.lookup.retry-strategy.type", "exponential-delay"); 33 | config.setString("gid.connector.http.source.lookup.retry-strategy.exponential-delay.initial-backoff", "15ms"); 34 | config.setString("gid.connector.http.source.lookup.retry-strategy.exponential-delay.max-backoff", "120ms"); 35 | config.setInteger("gid.connector.http.source.lookup.retry-strategy.exponential-delay.backoff-multiplier", 2); 36 | config.setInteger("lookup.max-retries", 6); 37 | 38 | var retryConfig = RetryConfigProvider.create(config); 39 | var intervalFunction = retryConfig.getIntervalFunction(); 40 | 41 | assertEquals(7, retryConfig.getMaxAttempts()); 42 | assertEquals(15, intervalFunction.apply(1)); 43 | assertEquals(30, intervalFunction.apply(2)); 44 | assertEquals(60, intervalFunction.apply(3)); 45 | assertEquals(120, intervalFunction.apply(4)); 46 | assertEquals(120, intervalFunction.apply(5)); 47 | assertEquals(120, intervalFunction.apply(6)); 48 | } 49 | 50 | @Test 51 | void failWhenStrategyIsUnsupported() { 52 | var config = new Configuration(); 53 | config.setString("gid.connector.http.source.lookup.retry-strategy.type", "dummy"); 54 | 55 | try (var mockedStatic = mockStatic(RetryStrategyType.class)) { 56 | var dummyStrategy = mock(RetryStrategyType.class); 57 | mockedStatic.when(() -> RetryStrategyType.fromCode("dummy")).thenReturn(dummyStrategy); 58 | 59 | assertThrows(IllegalArgumentException.class, 60 | () -> RetryConfigProvider.create(config)); 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/retry/RetryStrategyTypeTest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.retry; 2 | 3 | import java.util.stream.Stream; 4 | 5 | import org.junit.jupiter.api.Test; 6 | import org.junit.jupiter.params.ParameterizedTest; 7 | import org.junit.jupiter.params.provider.Arguments; 8 | import org.junit.jupiter.params.provider.MethodSource; 9 | import org.junit.jupiter.params.provider.ValueSource; 10 | import static org.junit.jupiter.api.Assertions.assertEquals; 11 | import static org.junit.jupiter.api.Assertions.assertThrows; 12 | 13 | class RetryStrategyTypeTest { 14 | 15 | static Stream inputArguments() { 16 | return Stream.of( 17 | Arguments.of("FIXED-DELAY", RetryStrategyType.FIXED_DELAY), 18 | Arguments.of("fixed-delay", RetryStrategyType.FIXED_DELAY), 19 | Arguments.of("exponential-delay", RetryStrategyType.EXPONENTIAL_DELAY), 20 | Arguments.of("EXPONENTIAL-DELAY", RetryStrategyType.EXPONENTIAL_DELAY) 21 | ); 22 | } 23 | 24 | @ParameterizedTest 25 | @MethodSource("inputArguments") 26 | void parseFromCodes(String code, RetryStrategyType expectedType) { 27 | var result = RetryStrategyType.fromCode(code); 28 | 29 | assertEquals(expectedType, result); 30 | } 31 | 32 | @ParameterizedTest 33 | @ValueSource(strings = { 34 | "fixed_delay", 35 | "FIXED_DELAY", 36 | "ABC", 37 | "FIXED-DELA", 38 | "exponential_delay" 39 | }) 40 | void failWhenCodeIsIllegal(String code) { 41 | assertThrows(IllegalArgumentException.class, () -> RetryStrategyType.fromCode(code)); 42 | } 43 | 44 | @Test 45 | void failWhenCodeIsNull() { 46 | assertThrows(NullPointerException.class, () -> RetryStrategyType.fromCode(null)); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/sink/HttpSinkBuilderTest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.sink; 2 | 3 | import java.nio.charset.StandardCharsets; 4 | import java.util.List; 5 | import java.util.concurrent.CompletableFuture; 6 | 7 | import org.apache.flink.connector.base.sink.writer.ElementConverter; 8 | import org.junit.jupiter.api.Test; 9 | import static org.junit.jupiter.api.Assertions.assertThrows; 10 | 11 | import com.getindata.connectors.http.HttpSink; 12 | import com.getindata.connectors.http.internal.SinkHttpClient; 13 | import com.getindata.connectors.http.internal.SinkHttpClientResponse; 14 | 15 | public class HttpSinkBuilderTest { 16 | 17 | private static final ElementConverter ELEMENT_CONVERTER = 18 | (s, context) -> new HttpSinkRequestEntry("POST", s.getBytes(StandardCharsets.UTF_8)); 19 | 20 | @Test 21 | public void testEmptyUrl() { 22 | assertThrows( 23 | IllegalArgumentException.class, 24 | () -> HttpSink.builder().setElementConverter(ELEMENT_CONVERTER) 25 | .setSinkHttpClientBuilder( 26 | ( 27 | properties, 28 | httpPostRequestCallback, 29 | headerPreprocessor, 30 | requestSubmitterFactory) -> new MockHttpClient()) 31 | .setEndpointUrl("") 32 | .build() 33 | ); 34 | } 35 | 36 | @Test 37 | public void testNullUrl() { 38 | assertThrows( 39 | IllegalArgumentException.class, 40 | () -> HttpSink.builder() 41 | .setElementConverter(ELEMENT_CONVERTER) 42 | .setSinkHttpClientBuilder( 43 | ( 44 | properties, 45 | httpPostRequestCallback, 46 | headerPreprocessor, 47 | requestSubmitterFactory) -> new MockHttpClient()) 48 | .build() 49 | ); 50 | } 51 | 52 | @Test 53 | public void testNullHttpClient() { 54 | assertThrows( 55 | NullPointerException.class, 56 | () -> HttpSink.builder() 57 | .setElementConverter(ELEMENT_CONVERTER) 58 | .setSinkHttpClientBuilder(null) 59 | .setEndpointUrl("localhost:8000") 60 | .build() 61 | ); 62 | } 63 | 64 | private static class MockHttpClient implements SinkHttpClient { 65 | 66 | MockHttpClient() {} 67 | 68 | @Override 69 | public CompletableFuture putRequests( 70 | List requestEntries, String endpointUrl) { 71 | throw new RuntimeException("Mock implementation of HttpClient"); 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/sink/HttpSinkWriterStateSerializerTest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.sink; 2 | 3 | import java.io.IOException; 4 | import java.nio.charset.StandardCharsets; 5 | 6 | import org.apache.flink.connector.base.sink.writer.BufferedRequestState; 7 | import org.apache.flink.connector.base.sink.writer.ElementConverter; 8 | import org.junit.jupiter.api.Test; 9 | import static org.apache.flink.connector.base.sink.writer.AsyncSinkWriterTestUtils.assertThatBufferStatesAreEqual; 10 | import static org.apache.flink.connector.base.sink.writer.AsyncSinkWriterTestUtils.getTestState; 11 | 12 | public class HttpSinkWriterStateSerializerTest { 13 | 14 | private static final ElementConverter ELEMENT_CONVERTER = 15 | (s, _context) -> 16 | new HttpSinkRequestEntry("POST", s.getBytes(StandardCharsets.UTF_8)); 17 | 18 | @Test 19 | public void testSerializeAndDeserialize() throws IOException { 20 | BufferedRequestState expectedState = 21 | getTestState(ELEMENT_CONVERTER, 22 | httpSinkRequestEntry -> Math.toIntExact(httpSinkRequestEntry.getSizeInBytes())); 23 | 24 | HttpSinkWriterStateSerializer serializer = new HttpSinkWriterStateSerializer(); 25 | BufferedRequestState actualState = 26 | serializer.deserialize(1, serializer.serialize(expectedState)); 27 | 28 | assertThatBufferStatesAreEqual(actualState, expectedState); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/sink/httpclient/BatchRequestSubmitterFactoryTest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.sink.httpclient; 2 | 3 | import java.util.Properties; 4 | 5 | import org.junit.jupiter.api.Test; 6 | import org.junit.jupiter.params.ParameterizedTest; 7 | import org.junit.jupiter.params.provider.ValueSource; 8 | import static org.assertj.core.api.Assertions.assertThat; 9 | import static org.junit.jupiter.api.Assertions.assertThrows; 10 | 11 | import com.getindata.connectors.http.internal.config.ConfigException; 12 | import com.getindata.connectors.http.internal.config.HttpConnectorConfigConstants; 13 | 14 | class BatchRequestSubmitterFactoryTest { 15 | 16 | @ParameterizedTest 17 | @ValueSource(ints = {0, -1}) 18 | public void shouldThrowIfInvalidDefaultSize(int invalidArgument) { 19 | assertThrows( 20 | IllegalArgumentException.class, 21 | () -> new BatchRequestSubmitterFactory(invalidArgument) 22 | ); 23 | } 24 | 25 | @Test 26 | public void shouldCreateSubmitterWithDefaultBatchSize() { 27 | 28 | int defaultBatchSize = 10; 29 | BatchRequestSubmitter submitter = new BatchRequestSubmitterFactory(defaultBatchSize) 30 | .createSubmitter(new Properties(), new String[0]); 31 | 32 | assertThat(submitter.getBatchSize()).isEqualTo(defaultBatchSize); 33 | } 34 | 35 | @ParameterizedTest 36 | @ValueSource(strings = {"1", "2"}) 37 | public void shouldCreateSubmitterWithCustomBatchSize(String batchSize) { 38 | 39 | Properties properties = new Properties(); 40 | properties.setProperty( 41 | HttpConnectorConfigConstants.SINK_HTTP_BATCH_REQUEST_SIZE, 42 | batchSize 43 | ); 44 | 45 | BatchRequestSubmitter submitter = new BatchRequestSubmitterFactory(10) 46 | .createSubmitter(properties, new String[0]); 47 | 48 | assertThat(submitter.getBatchSize()).isEqualTo(Integer.valueOf(batchSize)); 49 | } 50 | 51 | @ParameterizedTest 52 | @ValueSource(strings = {"0", "-1"}) 53 | public void shouldThrowIfBatchSizeToSmall(String invalidBatchSize) { 54 | 55 | Properties properties = new Properties(); 56 | properties.setProperty( 57 | HttpConnectorConfigConstants.SINK_HTTP_BATCH_REQUEST_SIZE, 58 | invalidBatchSize 59 | ); 60 | 61 | BatchRequestSubmitterFactory factory = new BatchRequestSubmitterFactory(10); 62 | 63 | assertThrows( 64 | ConfigException.class, 65 | () -> factory.createSubmitter(properties, new String[0]) 66 | ); 67 | } 68 | 69 | @ParameterizedTest 70 | @ValueSource(strings = {"1.1", "2,2", "hello"}) 71 | public void shouldThrowIfInvalidBatchSize(String invalidBatchSize) { 72 | 73 | Properties properties = new Properties(); 74 | properties.setProperty( 75 | HttpConnectorConfigConstants.SINK_HTTP_BATCH_REQUEST_SIZE, 76 | invalidBatchSize 77 | ); 78 | 79 | BatchRequestSubmitterFactory factory = new BatchRequestSubmitterFactory(10); 80 | 81 | assertThrows( 82 | ConfigException.class, 83 | () -> factory.createSubmitter(properties, new String[0]) 84 | ); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClientFactoryTest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import org.apache.flink.api.common.serialization.DeserializationSchema; 4 | import org.apache.flink.table.data.RowData; 5 | import org.apache.flink.util.ConfigurationException; 6 | import org.junit.jupiter.api.BeforeEach; 7 | import org.junit.jupiter.api.Test; 8 | import static org.assertj.core.api.Assertions.assertThat; 9 | import static org.mockito.Mockito.mock; 10 | 11 | class JavaNetHttpPollingClientFactoryTest { 12 | 13 | private JavaNetHttpPollingClientFactory factory; 14 | 15 | @BeforeEach 16 | public void setUp() { 17 | factory = new JavaNetHttpPollingClientFactory(mock(GetRequestFactory.class)); 18 | } 19 | 20 | @Test 21 | @SuppressWarnings("unchecked") 22 | void shouldCreateClient() throws ConfigurationException { 23 | 24 | assertThat( 25 | factory.createPollClient( 26 | HttpLookupConfig.builder().build(), 27 | (DeserializationSchema) mock(DeserializationSchema.class)) 28 | ).isInstanceOf(JavaNetHttpPollingClient.class); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/table/lookup/RowDataSingleValueLookupSchemaEntryTest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import java.util.List; 4 | 5 | import org.apache.flink.table.api.DataTypes; 6 | import org.apache.flink.table.data.GenericRowData; 7 | import org.apache.flink.table.data.RowData; 8 | import org.junit.jupiter.api.Test; 9 | import static org.assertj.core.api.Assertions.assertThat; 10 | 11 | import com.getindata.connectors.http.LookupArg; 12 | 13 | class RowDataSingleValueLookupSchemaEntryTest { 14 | 15 | // TODO Convert this to parametrized test and check all Flink types (Int, String etc). 16 | @Test 17 | public void shouldConvertFromSingleValue() { 18 | 19 | RowDataSingleValueLookupSchemaEntry entry = new RowDataSingleValueLookupSchemaEntry( 20 | "col1", 21 | RowData.createFieldGetter(DataTypes.BOOLEAN().getLogicalType(), 0) 22 | ); 23 | 24 | List lookupArgs = entry.convertToLookupArg(GenericRowData.of(true)); 25 | 26 | assertThat(lookupArgs).containsExactly(new LookupArg("col1", "true")); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/table/lookup/TableSourceHelperTest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup; 2 | 3 | import org.apache.flink.table.types.DataType; 4 | import org.apache.flink.table.types.logical.LogicalType; 5 | import org.apache.flink.table.types.logical.LogicalTypeRoot; 6 | import org.junit.jupiter.api.BeforeEach; 7 | import org.junit.jupiter.api.Test; 8 | import org.junit.jupiter.api.extension.ExtendWith; 9 | import org.mockito.Mock; 10 | import org.mockito.junit.jupiter.MockitoExtension; 11 | import static org.assertj.core.api.Assertions.assertThat; 12 | import static org.mockito.Mockito.when; 13 | 14 | @ExtendWith(MockitoExtension.class) 15 | class TableSourceHelperTest { 16 | 17 | @Mock 18 | private DataType dataType; 19 | 20 | @Mock 21 | private LogicalType logicalType; 22 | 23 | @BeforeEach 24 | public void setUp() { 25 | when(dataType.getLogicalType()).thenReturn(logicalType); 26 | } 27 | 28 | @Test 29 | void testNotComposite() { 30 | when(logicalType.getTypeRoot()).thenReturn(LogicalTypeRoot.BIGINT); 31 | 32 | assertThat(TableSourceHelper.getFieldNames(dataType.getLogicalType())).isEmpty(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/table/lookup/querycreators/CustomFormatFactory.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup.querycreators; 2 | 3 | import java.util.Collections; 4 | import java.util.Set; 5 | 6 | import org.apache.flink.api.common.serialization.SerializationSchema; 7 | import org.apache.flink.configuration.ConfigOption; 8 | import org.apache.flink.configuration.ConfigOptions; 9 | import org.apache.flink.configuration.ReadableConfig; 10 | import org.apache.flink.table.connector.format.EncodingFormat; 11 | import org.apache.flink.table.data.RowData; 12 | import org.apache.flink.table.factories.DynamicTableFactory.Context; 13 | import org.apache.flink.table.factories.FactoryUtil; 14 | import org.apache.flink.table.factories.SerializationFormatFactory; 15 | import org.apache.flink.table.factories.TestFormatFactory.EncodingFormatMock; 16 | 17 | public class CustomFormatFactory implements SerializationFormatFactory { 18 | 19 | public static final String IDENTIFIER = "query-creator-test-format"; 20 | public static final String REQUIRED_OPTION = "required-option-one"; 21 | 22 | /** 23 | * TODO remove static - used for testing only 24 | */ 25 | static boolean requiredOptionsWereUsed = false; 26 | 27 | @Override 28 | public EncodingFormat> createEncodingFormat( 29 | Context context, 30 | ReadableConfig readableConfig) { 31 | FactoryUtil.validateFactoryOptions(this, readableConfig); 32 | return new EncodingFormatMock(","); 33 | } 34 | 35 | @Override 36 | public String factoryIdentifier() { 37 | return IDENTIFIER; 38 | } 39 | 40 | @Override 41 | public Set> requiredOptions() { 42 | requiredOptionsWereUsed = true; 43 | return Set.of(ConfigOptions.key(REQUIRED_OPTION).stringType().noDefaultValue()); 44 | } 45 | 46 | @Override 47 | public Set> optionalOptions() { 48 | return Collections.emptySet(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/table/lookup/querycreators/CustomJsonFormatFactory.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup.querycreators; 2 | 3 | import java.util.Collections; 4 | import java.util.Set; 5 | 6 | import org.apache.flink.api.common.serialization.SerializationSchema; 7 | import org.apache.flink.configuration.ConfigOption; 8 | import org.apache.flink.configuration.ConfigOptions; 9 | import org.apache.flink.configuration.ReadableConfig; 10 | import org.apache.flink.formats.json.JsonFormatFactory; 11 | import org.apache.flink.table.connector.format.EncodingFormat; 12 | import org.apache.flink.table.data.RowData; 13 | import org.apache.flink.table.factories.DynamicTableFactory.Context; 14 | import org.apache.flink.table.factories.FactoryUtil; 15 | import org.apache.flink.table.factories.SerializationFormatFactory; 16 | 17 | public class CustomJsonFormatFactory extends JsonFormatFactory 18 | implements SerializationFormatFactory { 19 | 20 | public static final String IDENTIFIER = "json-query-creator-test-format"; 21 | public static final String REQUIRED_OPTION = "required-option-one"; 22 | 23 | /** 24 | * Consider removing this static only used for testing only 25 | */ 26 | static boolean requiredOptionsWereUsed = false; 27 | 28 | @Override 29 | public EncodingFormat> createEncodingFormat( 30 | Context context, 31 | ReadableConfig readableConfig) { 32 | FactoryUtil.validateFactoryOptions(this, readableConfig); 33 | return super.createEncodingFormat(context, readableConfig); 34 | } 35 | 36 | @Override 37 | public String factoryIdentifier() { 38 | return IDENTIFIER; 39 | } 40 | 41 | @Override 42 | public Set> requiredOptions() { 43 | requiredOptionsWereUsed = true; 44 | return Set.of(ConfigOptions.key(REQUIRED_OPTION).stringType().noDefaultValue()); 45 | } 46 | 47 | @Override 48 | public Set> optionalOptions() { 49 | return Collections.emptySet(); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/table/lookup/querycreators/GenericJsonQueryCreatorTest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup.querycreators; 2 | 3 | import java.util.List; 4 | 5 | import org.apache.flink.api.common.serialization.SerializationSchema; 6 | import org.apache.flink.configuration.Configuration; 7 | import org.apache.flink.formats.json.JsonFormatFactory; 8 | import org.apache.flink.table.api.DataTypes; 9 | import org.apache.flink.table.data.GenericRowData; 10 | import org.apache.flink.table.data.RowData; 11 | import org.apache.flink.table.data.StringData; 12 | import org.apache.flink.table.factories.DynamicTableFactory.Context; 13 | import org.apache.flink.table.types.DataType; 14 | import org.junit.jupiter.api.BeforeEach; 15 | import org.junit.jupiter.api.Test; 16 | import org.junit.jupiter.api.extension.ExtendWith; 17 | import org.mockito.Mock; 18 | import org.mockito.junit.jupiter.MockitoExtension; 19 | import static org.assertj.core.api.Assertions.assertThat; 20 | 21 | import com.getindata.connectors.http.internal.table.lookup.LookupQueryInfo; 22 | import static com.getindata.connectors.http.internal.table.lookup.HttpLookupTableSourceFactory.row; 23 | 24 | @ExtendWith(MockitoExtension.class) 25 | class GenericJsonQueryCreatorTest { 26 | 27 | @Mock 28 | private Context dynamicTableFactoryContext; 29 | 30 | private GenericJsonQueryCreator jsonQueryCreator; 31 | 32 | @BeforeEach 33 | public void setUp() { 34 | 35 | DataType lookupPhysicalDataType = row(List.of( 36 | DataTypes.FIELD("id", DataTypes.INT()), 37 | DataTypes.FIELD("uuid", DataTypes.STRING()) 38 | ) 39 | ); 40 | 41 | SerializationSchema jsonSerializer = 42 | new JsonFormatFactory() 43 | .createEncodingFormat(dynamicTableFactoryContext, new Configuration()) 44 | .createRuntimeEncoder(null, lookupPhysicalDataType); 45 | 46 | this.jsonQueryCreator = new GenericJsonQueryCreator(jsonSerializer); 47 | } 48 | 49 | @Test 50 | public void shouldSerializeToJson() { 51 | GenericRowData row = new GenericRowData(2); 52 | row.setField(0, 11); 53 | row.setField(1, StringData.fromString("myUuid")); 54 | 55 | LookupQueryInfo lookupQuery = this.jsonQueryCreator.createLookupQuery(row); 56 | assertThat(lookupQuery.getBodyBasedUrlQueryParameters().isEmpty()); 57 | assertThat(lookupQuery.getLookupQuery()).isEqualTo("{\"id\":11,\"uuid\":\"myUuid\"}"); 58 | } 59 | 60 | @Test 61 | public void shouldSerializeToJsonTwice() { 62 | GenericRowData row = new GenericRowData(2); 63 | row.setField(0, 11); 64 | row.setField(1, StringData.fromString("myUuid")); 65 | 66 | // Call createLookupQuery two times 67 | // to check that serialization schema is not opened Two times. 68 | this.jsonQueryCreator.createLookupQuery(row); 69 | LookupQueryInfo lookupQuery = this.jsonQueryCreator.createLookupQuery(row); 70 | assertThat(lookupQuery.getBodyBasedUrlQueryParameters().isEmpty()); 71 | assertThat(lookupQuery.getLookupQuery()).isEqualTo("{\"id\":11,\"uuid\":\"myUuid\"}"); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/table/lookup/querycreators/PathBean.java: -------------------------------------------------------------------------------- 1 | /* 2 | * © Copyright IBM Corp. 2025 3 | */ 4 | package com.getindata.connectors.http.internal.table.lookup.querycreators; 5 | 6 | import lombok.Data; 7 | 8 | @Data 9 | public class PathBean { 10 | private String key1; 11 | } 12 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/table/lookup/querycreators/PersonBean.java: -------------------------------------------------------------------------------- 1 | /* 2 | * © Copyright IBM Corp. 2025 3 | */ 4 | package com.getindata.connectors.http.internal.table.lookup.querycreators; 5 | 6 | import lombok.Data; 7 | 8 | @Data 9 | public class PersonBean { 10 | private final String firstName; 11 | private final String lastName; 12 | } 13 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/table/lookup/querycreators/QueryCreatorUtils.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup.querycreators; 2 | 3 | import java.util.Collections; 4 | 5 | import org.apache.flink.api.common.serialization.SerializationSchema; 6 | import org.apache.flink.configuration.Configuration; 7 | import org.apache.flink.table.api.Schema; 8 | import org.apache.flink.table.catalog.CatalogTable; 9 | import org.apache.flink.table.catalog.ObjectIdentifier; 10 | import org.apache.flink.table.catalog.ResolvedCatalogTable; 11 | import org.apache.flink.table.catalog.ResolvedSchema; 12 | import org.apache.flink.table.connector.format.EncodingFormat; 13 | import org.apache.flink.table.data.RowData; 14 | import org.apache.flink.table.factories.DynamicTableFactory; 15 | import org.apache.flink.table.factories.FactoryUtil; 16 | import org.apache.flink.table.factories.SerializationFormatFactory; 17 | 18 | import com.getindata.connectors.http.internal.table.lookup.LookupRow; 19 | 20 | public class QueryCreatorUtils { 21 | public static DynamicTableFactory.Context getTableContext(Configuration config, 22 | ResolvedSchema resolvedSchema) { 23 | 24 | return new FactoryUtil.DefaultDynamicTableContext( 25 | ObjectIdentifier.of("default", "default", "test"), 26 | new ResolvedCatalogTable( 27 | CatalogTable.of( 28 | Schema.newBuilder().fromResolvedSchema(resolvedSchema).build(), 29 | null, 30 | Collections.emptyList(), 31 | Collections.emptyMap()), 32 | resolvedSchema), 33 | Collections.emptyMap(), 34 | config, 35 | Thread.currentThread().getContextClassLoader(), 36 | false 37 | ); 38 | } 39 | public static SerializationSchema getRowDataSerializationSchema(LookupRow lookupRow, 40 | DynamicTableFactory.Context dynamicTableFactoryContext, 41 | String formatIdentifier, 42 | QueryFormatAwareConfiguration queryFormatAwareConfiguration) { 43 | SerializationFormatFactory jsonFormatFactory = 44 | FactoryUtil.discoverFactory( 45 | dynamicTableFactoryContext.getClassLoader(), 46 | SerializationFormatFactory.class, 47 | formatIdentifier 48 | ); 49 | 50 | EncodingFormat> 51 | encoder = jsonFormatFactory.createEncodingFormat( 52 | dynamicTableFactoryContext, 53 | queryFormatAwareConfiguration 54 | ); 55 | 56 | final SerializationSchema serializationSchema = 57 | encoder.createRuntimeEncoder(null, lookupRow.getLookupPhysicalRowDataType()); 58 | return serializationSchema; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/table/lookup/querycreators/QueryFormatAwareConfigurationTest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.table.lookup.querycreators; 2 | 3 | import java.util.Collections; 4 | import java.util.Optional; 5 | 6 | import org.apache.flink.configuration.ConfigOption; 7 | import org.apache.flink.configuration.ConfigOptions; 8 | import org.apache.flink.configuration.Configuration; 9 | import org.junit.jupiter.api.Test; 10 | import static org.assertj.core.api.Assertions.assertThat; 11 | 12 | class QueryFormatAwareConfigurationTest { 13 | 14 | private static final ConfigOption configOption = ConfigOptions.key("key") 15 | .stringType() 16 | .noDefaultValue(); 17 | 18 | @Test 19 | public void testWithDot() { 20 | QueryFormatAwareConfiguration queryConfig = new QueryFormatAwareConfiguration( 21 | "prefix.", Configuration.fromMap(Collections.singletonMap("prefix.key", "val")) 22 | ); 23 | 24 | Optional optional = queryConfig.getOptional(configOption); 25 | assertThat(optional.get()).isEqualTo("val"); 26 | } 27 | 28 | @Test 29 | public void testWithoutDot() { 30 | QueryFormatAwareConfiguration queryConfig = new QueryFormatAwareConfiguration( 31 | "prefix", Configuration.fromMap(Collections.singletonMap("prefix.key", "val")) 32 | ); 33 | 34 | Optional optional = queryConfig.getOptional(configOption); 35 | assertThat(optional.get()).isEqualTo("val"); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/utils/HttpHeaderUtilsTest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.utils; 2 | import java.time.Duration; 3 | 4 | import org.apache.flink.configuration.Configuration; 5 | import org.junit.jupiter.api.Test; 6 | import static org.assertj.core.api.AssertionsForClassTypes.assertThat; 7 | 8 | import com.getindata.connectors.http.internal.HeaderPreprocessor; 9 | import static com.getindata.connectors.http.internal.table.lookup.HttpLookupConnectorOptions.*; 10 | 11 | 12 | 13 | public class HttpHeaderUtilsTest { 14 | @Test 15 | void shouldCreateOIDCHeaderPreprocessorTest() { 16 | Configuration configuration = new Configuration(); 17 | HeaderPreprocessor headerPreprocessor 18 | = HttpHeaderUtils.createOIDCHeaderPreprocessor(configuration); 19 | assertThat(headerPreprocessor).isNull(); 20 | configuration.setString(SOURCE_LOOKUP_OIDC_AUTH_TOKEN_ENDPOINT_URL.key(), "http://aaa"); 21 | configuration.setString(SOURCE_LOOKUP_OIDC_AUTH_TOKEN_REQUEST.key(), "ccc"); 22 | configuration.set(SOURCE_LOOKUP_OIDC_AUTH_TOKEN_EXPIRY_REDUCTION, Duration.ofSeconds(1)); 23 | headerPreprocessor 24 | = HttpHeaderUtils.createOIDCHeaderPreprocessor(configuration); 25 | assertThat(headerPreprocessor).isNotNull(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/utils/uri/CharArrayBufferTest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.utils.uri; 2 | 3 | import java.util.stream.Stream; 4 | 5 | import org.junit.jupiter.api.Assertions; 6 | import org.junit.jupiter.api.Test; 7 | import org.junit.jupiter.params.ParameterizedTest; 8 | import org.junit.jupiter.params.provider.Arguments; 9 | import org.junit.jupiter.params.provider.MethodSource; 10 | import static org.assertj.core.api.Assertions.assertThat; 11 | import static org.junit.jupiter.api.Assertions.assertAll; 12 | import static org.junit.jupiter.api.Assertions.assertThrows; 13 | 14 | class CharArrayBufferTest { 15 | 16 | @Test 17 | public void testInvalidCapacity() { 18 | assertThrows(IllegalArgumentException.class, () -> new CharArrayBuffer(0)); 19 | } 20 | 21 | @Test 22 | public void testExpandCapacity() { 23 | String testText = "Hello My Friend"; 24 | 25 | CharArrayBuffer charArrayBuffer = new CharArrayBuffer(1); 26 | charArrayBuffer.append(testText); 27 | 28 | assertThat(charArrayBuffer.length()).isEqualTo(testText.length()); 29 | } 30 | 31 | @Test 32 | public void testSubSequence() { 33 | String testText = "Hello My Friend"; 34 | 35 | CharArrayBuffer charArrayBuffer = new CharArrayBuffer(1); 36 | charArrayBuffer.append(testText); 37 | 38 | assertAll(() -> { 39 | Assertions.assertThrows(IndexOutOfBoundsException.class, 40 | () -> charArrayBuffer.subSequence(-1, 1)); 41 | Assertions.assertThrows(IndexOutOfBoundsException.class, 42 | () -> charArrayBuffer.subSequence(1, -1)); 43 | Assertions.assertThrows(IndexOutOfBoundsException.class, 44 | () -> charArrayBuffer.subSequence(2, 1)); 45 | Assertions.assertThrows(IndexOutOfBoundsException.class, 46 | () -> charArrayBuffer.subSequence(2, testText.length() + 5)); 47 | assertThat(charArrayBuffer.subSequence(2, 10).toString()).isEqualTo("llo My Fri"); 48 | } 49 | ); 50 | } 51 | 52 | private static Stream appendArgs() { 53 | return Stream.of( 54 | Arguments.of("", "baseString"), 55 | Arguments.of(" ", "baseString "), 56 | Arguments.of(null, "baseStringnull") 57 | ); 58 | } 59 | 60 | @ParameterizedTest 61 | @MethodSource("appendArgs") 62 | public void testAppend(String stringToAppend, String expected) { 63 | CharArrayBuffer charArrayBuffer = new CharArrayBuffer(1); 64 | charArrayBuffer.append("baseString"); 65 | 66 | assertAll(() -> { 67 | assertThat(charArrayBuffer.toString()).isEqualTo("baseString"); 68 | charArrayBuffer.append(stringToAppend); 69 | assertThat(charArrayBuffer.toString()).isEqualTo(expected); 70 | } 71 | ); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/utils/uri/ParserCursorTest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.utils.uri; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import static org.junit.jupiter.api.Assertions.assertAll; 5 | import static org.junit.jupiter.api.Assertions.assertThrows; 6 | 7 | class ParserCursorTest { 8 | 9 | @Test 10 | public void testBoundsValidation() { 11 | 12 | assertAll(() -> { 13 | assertThrows(IndexOutOfBoundsException.class, () -> new ParserCursor(-1, 1)); 14 | assertThrows(IndexOutOfBoundsException.class, () -> new ParserCursor(1, -1)); 15 | } 16 | ); 17 | } 18 | 19 | @Test 20 | public void testUpdatePostValidation() { 21 | ParserCursor cursor = new ParserCursor(1, 2); 22 | 23 | assertAll(() -> { 24 | assertThrows(IndexOutOfBoundsException.class, () -> cursor.updatePos(0)); 25 | assertThrows(IndexOutOfBoundsException.class, () -> cursor.updatePos(3)); 26 | } 27 | ); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/test/java/com/getindata/connectors/http/internal/utils/uri/TokenParserTest.java: -------------------------------------------------------------------------------- 1 | package com.getindata.connectors.http.internal.utils.uri; 2 | 3 | import org.junit.jupiter.params.ParameterizedTest; 4 | import org.junit.jupiter.params.provider.CsvSource; 5 | import static org.assertj.core.api.Assertions.assertThat; 6 | 7 | class TokenParserTest { 8 | 9 | @ParameterizedTest 10 | @CsvSource({"a,a", "aa,aa", "a a,a a", "a ,a", " a,a"}) 11 | public void testParse(String toParse, String expected) { 12 | 13 | CharArrayBuffer charBuff = new CharArrayBuffer(toParse.length()); 14 | charBuff.append(toParse); 15 | 16 | TokenParser tokenParser = new TokenParser(); 17 | String actual = tokenParser.parseToken( 18 | charBuff, 19 | new ParserCursor(0, toParse.length()), 20 | null 21 | ); 22 | 23 | assertThat(actual).isEqualTo(expected); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory: -------------------------------------------------------------------------------- 1 | com.getindata.connectors.http.TestPostRequestCallbackFactory 2 | com.getindata.connectors.http.TestLookupPostRequestCallbackFactory 3 | com.getindata.connectors.http.internal.table.lookup.querycreators.CustomFormatFactory 4 | com.getindata.connectors.http.internal.table.lookup.querycreators.CustomJsonFormatFactory -------------------------------------------------------------------------------- /src/test/resources/auth/AuthResult.json: -------------------------------------------------------------------------------- 1 | { 2 | "access_token": "test", 3 | "expires_in": 1 4 | } -------------------------------------------------------------------------------- /src/test/resources/http-array-result-with-nulls/HttpResult.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": "COUNTER1", 4 | "uuid": "UUID", 5 | "msg": "Returned HTTP message for parameter PARAM, COUNTER", 6 | "age": 30, 7 | "eyeColor": "green", 8 | "name": "Marva Fischer", 9 | "gender": "female", 10 | "details": { 11 | "isActive": true, 12 | "nestedDetails": { 13 | "index": 0, 14 | "guid": "d81fc542-6b49-4d59-8fb9-d57430d4871d", 15 | "balance": "$1,729.34" 16 | } 17 | } 18 | }, 19 | null 20 | ] -------------------------------------------------------------------------------- /src/test/resources/http-array-result/HttpResult.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": "COUNTER1", 4 | "uuid": "UUID", 5 | "msg": "Returned HTTP message for parameter PARAM, COUNTER", 6 | "age": 30, 7 | "eyeColor": "green", 8 | "name": "Marva Fischer", 9 | "gender": "female", 10 | "details": { 11 | "isActive": true, 12 | "nestedDetails": { 13 | "index": 0, 14 | "guid": "d81fc542-6b49-4d59-8fb9-d57430d4871d", 15 | "balance": "$1,729.34" 16 | } 17 | } 18 | }, 19 | { 20 | "id": "COUNTER2", 21 | "uuid": "UUID", 22 | "msg": "Returned HTTP message for parameter PARAM, COUNTER", 23 | "age": 40, 24 | "eyeColor": "brown", 25 | "name": "John Doe", 26 | "gender": "male", 27 | "details": { 28 | "isActive": false, 29 | "nestedDetails": { 30 | "index": 0, 31 | "guid": "d81fc542-6b49-4d59-8fb9-d57430d4871d", 32 | "balance": "$22,001.99" 33 | } 34 | } 35 | } 36 | ] -------------------------------------------------------------------------------- /src/test/resources/http/HttpResult.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "COUNTER", 3 | "uuid": "UUID", 4 | "picture": "http://placehold.it/32x32", 5 | "msg": "Returned HTTP message for parameter PARAM, COUNTER", 6 | "age": 30, 7 | "eyeColor": "green", 8 | "name": "Marva Fischer", 9 | "gender": "female", 10 | "company": "SILODYNE", 11 | "email": "marvafischer@silodyne.com", 12 | "phone": "+1 (990) 562-2120", 13 | "address": "601 Auburn Place, Bynum, New York, 7057", 14 | "about": "Proident Lorem et duis nisi tempor elit occaecat laboris dolore magna Lorem consequat. Deserunt velit minim nisi consectetur duis amet labore cupidatat. Pariatur sunt occaecat qui reprehenderit ipsum ex culpa ullamco ex duis adipisicing commodo sunt. Ad cupidatat magna ad in officia irure aute duis culpa et. Magna esse adipisicing consequat occaecat. Excepteur amet dolore occaecat sit officia dolore elit in cupidatat non anim.", 15 | "registered": "2020-07-11T11:13:32 -02:00", 16 | "latitude": -35.237843, 17 | "longitude": 60.386104, 18 | "tags": [ 19 | "officia", 20 | "eiusmod", 21 | "labore", 22 | "ex", 23 | "aliqua", 24 | "consectetur", 25 | "excepteur" 26 | ], 27 | "friends": [ 28 | { 29 | "id": 0, 30 | "name": "Kemp Newman" 31 | }, 32 | { 33 | "id": 1, 34 | "name": "Sears Blackburn" 35 | }, 36 | { 37 | "id": 2, 38 | "name": "Lula Rogers" 39 | } 40 | ], 41 | "details": { 42 | "isActive": true, 43 | "nestedDetails": { 44 | "index": 0, 45 | "guid": "d81fc542-6b49-4d59-8fb9-d57430d4871d", 46 | "balance": "$1,729.34" 47 | } 48 | }, 49 | "greeting": "Hello, Marva Fischer! You have 7 unread messages.", 50 | "favoriteFruit": "banana" 51 | } -------------------------------------------------------------------------------- /src/test/resources/json/sink/allInOneBatch.txt: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": 1, 4 | "first_name": "Ninette", 5 | "last_name": "Clee", 6 | "gender": "Female", 7 | "stock": "CDZI", 8 | "currency": "RUB", 9 | "tx_date": "2021-08-24 15:22:59" 10 | }, 11 | { 12 | "id": 2, 13 | "first_name": "Rob", 14 | "last_name": "Zombie", 15 | "gender": "Male", 16 | "stock": "DGICA", 17 | "currency": "GBP", 18 | "tx_date": "2021-10-25 20:53:54" 19 | }, 20 | { 21 | "id": 3, 22 | "first_name": "Adam", 23 | "last_name": "Jones", 24 | "gender": "Male", 25 | "stock": "DGICA", 26 | "currency": "PLN", 27 | "tx_date": "2021-10-26 20:53:54" 28 | }, 29 | { 30 | "id": 4, 31 | "first_name": "Danny", 32 | "last_name": "Carey", 33 | "gender": "Male", 34 | "stock": "DGICA", 35 | "currency": "USD", 36 | "tx_date": "2021-10-27 20:53:54" 37 | }, 38 | { 39 | "id": 5, 40 | "first_name": "Bob", 41 | "last_name": "Dylan", 42 | "gender": "Male", 43 | "stock": "DGICA", 44 | "currency": "USD", 45 | "tx_date": "2021-10-28 20:53:54" 46 | } 47 | ] -------------------------------------------------------------------------------- /src/test/resources/json/sink/fourSingleEventBatches.txt: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": 2, 4 | "first_name": "Rob", 5 | "last_name": "Zombie", 6 | "gender": "Male", 7 | "stock": "DGICA", 8 | "currency": "GBP", 9 | "tx_date": "2021-10-25 20:53:54" 10 | } 11 | ] 12 | #-----# 13 | [ 14 | { 15 | "id": 3, 16 | "first_name": "Adam", 17 | "last_name": "Jones", 18 | "gender": "Male", 19 | "stock": "DGICA", 20 | "currency": "PLN", 21 | "tx_date": "2021-10-26 20:53:54" 22 | } 23 | ] 24 | #-----# 25 | [ 26 | { 27 | "id": 4, 28 | "first_name": "Danny", 29 | "last_name": "Carey", 30 | "gender": "Male", 31 | "stock": "DGICA", 32 | "currency": "USD", 33 | "tx_date": "2021-10-27 20:53:54" 34 | } 35 | ] 36 | #-----# 37 | [ 38 | { 39 | "id": 1, 40 | "first_name": "Ninette", 41 | "last_name": "Clee", 42 | "gender": "Female", 43 | "stock": "CDZI", 44 | "currency": "RUB", 45 | "tx_date": "2021-08-24 15:22:59" 46 | } 47 | ] 48 | #-----# 49 | [ 50 | { 51 | "id": 5, 52 | "first_name": "Bob", 53 | "last_name": "Dylan", 54 | "gender": "Male", 55 | "stock": "DGICA", 56 | "currency": "USD", 57 | "tx_date": "2021-10-28 20:53:54" 58 | } 59 | ] 60 | 61 | -------------------------------------------------------------------------------- /src/test/resources/json/sink/threeBatches.txt: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": 1, 4 | "first_name": "Ninette", 5 | "last_name": "Clee", 6 | "gender": "Female", 7 | "stock": "CDZI", 8 | "currency": "RUB", 9 | "tx_date": "2021-08-24 15:22:59" 10 | }, 11 | { 12 | "id": 2, 13 | "first_name": "Rob", 14 | "last_name": "Zombie", 15 | "gender": "Male", 16 | "stock": "DGICA", 17 | "currency": "GBP", 18 | "tx_date": "2021-10-25 20:53:54" 19 | } 20 | ] 21 | #-----# 22 | [ 23 | { 24 | "id": 3, 25 | "first_name": "Adam", 26 | "last_name": "Jones", 27 | "gender": "Male", 28 | "stock": "DGICA", 29 | "currency": "PLN", 30 | "tx_date": "2021-10-26 20:53:54" 31 | }, 32 | { 33 | "id": 4, 34 | "first_name": "Danny", 35 | "last_name": "Carey", 36 | "gender": "Male", 37 | "stock": "DGICA", 38 | "currency": "USD", 39 | "tx_date": "2021-10-27 20:53:54" 40 | } 41 | ] 42 | #-----# 43 | [ 44 | { 45 | "id": 5, 46 | "first_name": "Bob", 47 | "last_name": "Dylan", 48 | "gender": "Male", 49 | "stock": "DGICA", 50 | "currency": "USD", 51 | "tx_date": "2021-10-28 20:53:54" 52 | } 53 | ] 54 | -------------------------------------------------------------------------------- /src/test/resources/json/sink/twoBatches.txt: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": 4, 4 | "first_name": "Danny", 5 | "last_name": "Carey", 6 | "gender": "Male", 7 | "stock": "DGICA", 8 | "currency": "USD", 9 | "tx_date": "2021-10-27 20:53:54" 10 | }, 11 | { 12 | "id": 5, 13 | "first_name": "Bob", 14 | "last_name": "Dylan", 15 | "gender": "Male", 16 | "stock": "DGICA", 17 | "currency": "USD", 18 | "tx_date": "2021-10-28 20:53:54" 19 | } 20 | ] 21 | #-----# 22 | [ 23 | { 24 | "id": 1, 25 | "first_name": "Ninette", 26 | "last_name": "Clee", 27 | "gender": "Female", 28 | "stock": "CDZI", 29 | "currency": "RUB", 30 | "tx_date": "2021-08-24 15:22:59" 31 | }, 32 | { 33 | "id": 2, 34 | "first_name": "Rob", 35 | "last_name": "Zombie", 36 | "gender": "Male", 37 | "stock": "DGICA", 38 | "currency": "GBP", 39 | "tx_date": "2021-10-25 20:53:54" 40 | }, 41 | { 42 | "id": 3, 43 | "first_name": "Adam", 44 | "last_name": "Jones", 45 | "gender": "Male", 46 | "stock": "DGICA", 47 | "currency": "PLN", 48 | "tx_date": "2021-10-26 20:53:54" 49 | } 50 | ] 51 | -------------------------------------------------------------------------------- /src/test/resources/security/certs/ca.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIF6zCCA9OgAwIBAgIUSvHdDiIwAqJOu62LVdcpm4RRBjEwDQYJKoZIhvcNAQEL 3 | BQAwgYMxCzAJBgNVBAYTAlBMMRIwEAYDVQQIDAlQb21vcnNraWUxDzANBgNVBAcM 4 | BkdkeW5pYTEMMAoGA1UECgwDR0lEMQwwCgYDVQQLDANEZXYxEjAQBgNVBAMMCWxv 5 | Y2FsaG9zdDEfMB0GCSqGSIb3DQEJARYQdGVzdEBleGFtcGxlLmNvbTAgFw0yMjA5 6 | MDIxMDA4MTNaGA8yMDUwMDExODEwMDgxM1owgYMxCzAJBgNVBAYTAlBMMRIwEAYD 7 | VQQIDAlQb21vcnNraWUxDzANBgNVBAcMBkdkeW5pYTEMMAoGA1UECgwDR0lEMQww 8 | CgYDVQQLDANEZXYxEjAQBgNVBAMMCWxvY2FsaG9zdDEfMB0GCSqGSIb3DQEJARYQ 9 | dGVzdEBleGFtcGxlLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB 10 | ALfeo3R23wPWcQ9rh312ChsJli1UJ8N9XJ/65PrkwBFpuOwN4y3Zh1mNZniJfCsc 11 | QtioOQAzFtHF3erziRWy+waZ5Nc1iNAP5T3HFzT6MmP7FEnAr2H1rbhN32ow3WCd 12 | gYEpXWhZKBF2mls6bJxoCWKVT9361vdeiPJZ/j80B+9G8xZL81oZAOPKp2FTUsLA 13 | +6BTZQsqFw0P+CfodrbDHU1C78CXWZP8mG1iAV/i4Yo88lzcUoGfZ2afwpRyUQXl 14 | 6TsQx8BxMnbD4oE9KLJ27DKKK9qPYKsbilwrkCB01+CpnvL/b9knMp/EDOYt2aUd 15 | U5ks4dNut1aDms36UNyZQJ020caHHE7mup/Ho3BFQtCns2kFdRteOjQ5KAC3gNaD 16 | +jWlaIKWAQQcBrqqrklh3ps9/3mW+cdog0k0CEngq7p1Mo4ijWWC8fjl14SI+nMZ 17 | 6aVtHyaijr3UXswgWTpyQNIhhtnSMAb1Fde1Be41XSu4crOl5SClG0PMU5BZ9ATH 18 | K9hNFJk4kQGm5seZmiRpwFA150vef2kpwjWOFnO6f9jUAbgSLQXO1fh71lTAkBv1 19 | Gju1riKIg31dZ6bnntCk9pfPRW3AKYOT5aNqT0+TX3SgbLlhig7olH4uF/NS5kd+ 20 | 4SjqYdlNC0ZrMKh4+fPZiZVnB0ptsG47RPrNpaiCdvoLAgMBAAGjUzBRMB0GA1Ud 21 | DgQWBBSUpn/0wzYXXTy7RFVfvaeTwfCBhTAfBgNVHSMEGDAWgBSUpn/0wzYXXTy7 22 | RFVfvaeTwfCBhTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQCg 23 | 7xilaVWt8VOVYKaZB0lBKC7qMmV1SV/7mC2bIGjyEw4FKFw+E5Z4YrkxvTRmlAqi 24 | A/yDVWKhK2urG07CUAyD9vtj4rrUSYO4JCWBkVTvHxt0aSBTxj3PKf4wLL8EzBUF 25 | YUneeEnEhgQALr8TpIs5TrwFIHipIrYsg8IMECgquJBMgpPCBLC5nTdJD1dCoDB3 26 | 5jJ6NxeraXhFY9r/fp/nuH2FMD5V5flpgTvhercaFHP1P9crlQHgliyYppM9r7DB 27 | wffGb7NJuntXd5v7xzhB4dkICGVOovC47hKLmRZ6+w5o5iwpdHHTVecY88N9RYxY 28 | 3bqO975+mbcfLBDAPAFii0F1y1/Xkm1YqVcj0omrnT/bHmbkEnpLXX9N9kGtY1ch 29 | PVka47F8pkOah1bA+IBgjeMBsFRNgTLao/eKBo1P8r9tq/RGvQzlJ2sj+tTRvu9k 30 | c6fDnqeIW+ViVHwpoSINL1+8JcyxHp/SwSMZcI7sNU4XeCWYHd/08jTZ/h9ycdba 31 | 8r6g8LAeoOe6yUPzC9I0pLGMnjvOYJfgsWJMvAF1vFnXreqUu5rVPOtX5BR3E4T2 32 | UAXqPPYzo4IgbjsK0z1sqf/mf1lr/twNFzeIfmIkqabn5mjMkIT+ys8t+cN/9u/5 33 | 8NKl+zkVF/xQe4Knj8n9FIen5hVg3O5dIiLs15itNw== 34 | -----END CERTIFICATE----- 35 | -------------------------------------------------------------------------------- /src/test/resources/security/certs/ca.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIJJwIBAAKCAgEAt96jdHbfA9ZxD2uHfXYKGwmWLVQnw31cn/rk+uTAEWm47A3j 3 | LdmHWY1meIl8KxxC2Kg5ADMW0cXd6vOJFbL7Bpnk1zWI0A/lPccXNPoyY/sUScCv 4 | YfWtuE3fajDdYJ2BgSldaFkoEXaaWzpsnGgJYpVP3frW916I8ln+PzQH70bzFkvz 5 | WhkA48qnYVNSwsD7oFNlCyoXDQ/4J+h2tsMdTULvwJdZk/yYbWIBX+LhijzyXNxS 6 | gZ9nZp/ClHJRBeXpOxDHwHEydsPigT0osnbsMoor2o9gqxuKXCuQIHTX4Kme8v9v 7 | 2Scyn8QM5i3ZpR1TmSzh0263VoOazfpQ3JlAnTbRxoccTua6n8ejcEVC0KezaQV1 8 | G146NDkoALeA1oP6NaVogpYBBBwGuqquSWHemz3/eZb5x2iDSTQISeCrunUyjiKN 9 | ZYLx+OXXhIj6cxnppW0fJqKOvdRezCBZOnJA0iGG2dIwBvUV17UF7jVdK7hys6Xl 10 | IKUbQ8xTkFn0BMcr2E0UmTiRAabmx5maJGnAUDXnS95/aSnCNY4Wc7p/2NQBuBIt 11 | Bc7V+HvWVMCQG/UaO7WuIoiDfV1npuee0KT2l89FbcApg5Plo2pPT5NfdKBsuWGK 12 | DuiUfi4X81LmR37hKOph2U0LRmswqHj589mJlWcHSm2wbjtE+s2lqIJ2+gsCAwEA 13 | AQKCAgAtt46+u3ux8ZTE98CFgl764ARWGvGUGsx3/qMGevACpF3VW5Kb6NVNf+n1 14 | mIGmGm3IfvNFSG6pE+CctZblLU5bEtKia+4rhXhDRwhPJNJ4p4og2GcDHVnnVH1z 15 | Ytj4V9FGcymFc32R8gJpInq03O1QdL2Z/O+MLBYy5AtiXaaHaQWyCopkYYkmGlJJ 16 | 0bTnNaKa94FNKFb46XqTOaqiwJq9hiIkqb0dHZYsxEyBMeFQRN0iF6nwDtlf2+M+ 17 | k7nvlH3MMNzpzXt1qjjcV3+R+B28ZzyGKFYp1TwutKouV78BXktmMfr2bFea/90R 18 | Ml9GPad2k4FG1+V//lL0WgPXBcfZo0iUhn3xb/Fi/dlE2YcDdMTtroy6T0FH+fcd 19 | 43QG3Dqll7sW5pCFidZpMcdGzW4gC9ZRKX6jIslHg/Z51zrBun2E5+vG5RjM3Emk 20 | bRWGUGd4u/ahB3VpgFIvHTDJ7w2Jo5o2NdWtOdu86qQFhs3uBcrettaIXWIGipLv 21 | YulBK5JKlHjVn8oWtc61mi7W/82SZiG3kt+My/83OdDs/c5rNjZ+kQHzt43JvZCl 22 | jmqCqQI/rdW1dEw9zTlgBxQr374JqhJ6Gcz8y+YRTYQJtdt4BRTlS9ZjqcPFY/i9 23 | TlSeT4xXcTX5zr9+OAXbL9LHopdbU5S6LUv21m8JkO8reQSXgQKCAQEA3bFJQWgj 24 | 3bJDzIExUchP3cLILUnhcq/vWNLfcQuns/C+UGtO32725ACjcRfUAJkQ1SpkdTfR 25 | Pxe8zTNqBEH8gbwv+wSFmb9rZFFX8MkNSSK2m4dSapWMwc2gph8/aQOusXQNwuG7 26 | k4Yus/Sgun0ssB7KDXbQFOEJmmsBlulR09lIMcNyyUE704lMzxcbPxPE62LUP26v 27 | 6as2AZ+JcSOTKvpxaIaFO22csU7FtddbgmVvn/bbHUsd1ueGyaJcHUo6JWmB0VU9 28 | bVGpvhvtn8zgTxhtcebL0uqzjVwwS77Ss8I+drY3TNHWn+HNemqN15GF7gWLOX9c 29 | pj6mDMytCXghQQKCAQEA1FLwMTnRLIZri9ZuurwkIfV9zYbJz7Uzp2WUvL4C1duI 30 | KuPuCxHejbFtXJ1IjBljGlJzebURtgluwKHJmqPi+fZqoyIFniNkmEfPAyZU1XJI 31 | jKJ7D6NV1sNDZf47c3LzgNq61yywZ+ihoUxe895tix445+PTKi7CjSGCHhK5yYaX 32 | 2wobQUTksrZkzFvV7NhHTSerwTOzNYA4/DW0X3Vcejtf/t7wKI2+modCI4b67fSN 33 | /woxkM3bAFjcgibz0Qa5sWXXCKfV9wZj+bHI0wyiiui0zop+o93OVuyZ1ZbVHGVU 34 | aM7a6HcxzwvA13ktRZYfm7HE9BddTLpLA2qcW/o8SwKCAQA7QGL/5EGRw09znSP9 35 | Y7hU4PZ3hEOJT5R8/P9TS3YrFbEH9WZDOXkrPzBvzZqP/lkNYCMxYRW/8yw2/ALd 36 | 2HUQh6ebQ34htZOSYFtMs6+lWG3N68EPU6EnOzowrDFac+2A6C+1Yner2jUpS7yZ 37 | m1MST2hZaOP8cJkMeIUYPIvuPyiYhFBtkc3Xn96zis1xcmszLlrkCUVjAquQi39E 38 | iemPlxgwg6OgWo+CpDcbfs+qbkoDSwsNLUJwabkQsyBDg88+qJBVzp54VVc0xGoi 39 | R7O78B9wTes0GB6dLYKK/Bl8ifH/8HufQIZIfSVaDpiSgQZ2s2yelXfBxwTCbwHg 40 | ggcBAoIBAA1eA102BKNG+zJcl7AN9EBnEjqER+RZxhy92nVfiRIpthLtMqljwJ0v 41 | FHGGPXEEkCayFi8NZUYOpfhetzCTLNEL4H7Qv4XLOKQVsUTDfwfJICQqfpDneZoB 42 | xRaAwQFttULhgm3VlG2WSca75iZ4VHEzv+VphHvdYcY829JE34l+fcT+h+S+D5/o 43 | t8Pd7EPzAdorqzaIqLmvjubiAq8X1nCjmFGVe40yD1PvPAy8jMO6Z84suB4jt6v3 44 | 7DoNdcAtKjeu77DN0DyzHEw2EyymWP1h1TBDq+FpL3ptfunhix00i/HlRAbqqioc 45 | 1VE7gcwlvWux73Jmniscd2iJp7pqsK0CggEAWtr1wmCKyrGhCpBEkkOiNTnbkrdT 46 | OIZF+TzalIkxpRRqLsAh8z0CbLrNpHrR1fi2ZU4e7tOi2oEscs6gYesF+ZyNS8Vf 47 | AXmUFlG4derjn3zIDwqyAHxqP/yLNwdl3op7V0/po2UtVkFWgfCCvsyYyi9GC8Ya 48 | o2DNeRv0X4GLSwIYcvjxJRjVl3YidIsKoQxWoHFm8FCjh7+qdratSh9ZfL/p3ZNU 49 | i/BLOD4gOeGlVxZKmPAGu87F644GcoS1pzwtPjnKgVBnGAdW7jA69MpgrXUnGVua 50 | CTli4GJAlY73u9qypwY55EpdW174KDcOzpBqWnij3vK/Z554IgKcsUZgSg== 51 | -----END RSA PRIVATE KEY----- 52 | -------------------------------------------------------------------------------- /src/test/resources/security/certs/ca.srl: -------------------------------------------------------------------------------- 1 | 37A0763BD55E9F2C39139AE6B0335CDED242CF29 2 | -------------------------------------------------------------------------------- /src/test/resources/security/certs/client.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFkTCCA3kCFDegdjvVXp8sOROa5rAzXN7SQs8pMA0GCSqGSIb3DQEBCwUAMIGD 3 | MQswCQYDVQQGEwJQTDESMBAGA1UECAwJUG9tb3Jza2llMQ8wDQYDVQQHDAZHZHlu 4 | aWExDDAKBgNVBAoMA0dJRDEMMAoGA1UECwwDRGV2MRIwEAYDVQQDDAlsb2NhbGhv 5 | c3QxHzAdBgkqhkiG9w0BCQEWEHRlc3RAZXhhbXBsZS5jb20wIBcNMjIwOTAyMTAy 6 | MDIwWhgPMjA1MDAxMTgxMDIwMjBaMIGDMQswCQYDVQQGEwJQTDESMBAGA1UECAwJ 7 | UG9tb3Jza2llMQ8wDQYDVQQHDAZHZHluaWExDDAKBgNVBAoMA0dJRDEMMAoGA1UE 8 | CwwDRGV2MRIwEAYDVQQDDAlsb2NhbGhvc3QxHzAdBgkqhkiG9w0BCQEWEHRlc3RA 9 | ZXhhbXBsZS5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDExIkq 10 | qDtrysEcOrjmPzTtfQO+3QzgeA6rHcJ9WkeI5GkWUXUIJsqlnQfG3rbNv/yApMyG 11 | 5dAmxmm+XkaulCioXiCIe6tOkND6ZI3r0pg7b8HLC6DfaGMVidN8deBRaDB/s0sG 12 | jd0L6XWVvvftDuGGwTQTO1M+Bd2yXX/g5nbIDRK9SpBsnS/XlKOYzE8vteZkHiZp 13 | ZCwRsA2Ym+0NeYPQHQs2Kl4Vjzc5XligXDfFr43uFBCWFh5J89baWpFFINjhbq+F 14 | ocqKKpAa4XlO7yzL/hbvpcTFNwQqEfcv1/vKmEfwS5BlVrp0OWXtv2nwSqc6wnkG 15 | WQCTkNXovJJb/uu9WncRZWuwyPE9zyeFbtgXuZICXsVgZjJQhPOqBoK2JEWlqOVd 16 | fE1/nzANCVFueKjfOqQv3NqGewveLjB7xlGbTsWtjOhOhZHvXVHm5hnHJv43s3aZ 17 | fo5HP19Es6VSqfb8Wqa9JWSGmHO4u9eCIL0FKntz1gIUPq4NwlTG31bJdops5eYS 18 | YJW8+1DMGhPoXZQbEGmpPTJ9jPxkBiEbVzguu+ZHawDKgfeh9Lfup1rl67Dc813r 19 | byF1PxREU+Ykh7yjWYorEdsGAxQSJIJVyhDDcXoxloyhNx/28ldpf3Bqm4denaRT 20 | o3+lcZAX3XVUN6CEymY8llnLw9o3a6f7KWOJTwIDAQABMA0GCSqGSIb3DQEBCwUA 21 | A4ICAQBO41EDeW2OG14vnXzxUcfIJRfwAxW/PqB8SFrl4Upxc9SkdMNjtO4rMqnA 22 | Y8JM7iuVJ/+fDLetnhTkKNDLCb8JHXiB0aWjrOCP2wj61xzyZyJA0gcHJ8uNx6o3 23 | R25iJMxDIep7jfxcFc1FaaAYB/83j8+5+byGL9jWzzHJnZZ0RV8AJBcmV4KAHLi0 24 | 4+lxIN9jAbfPRn5ugyNxS1FReAbsePJ/x1dOFyCli/0oSRnW1edJhvR3QeMr9+yE 25 | ar95Rpq63YPaphd7aK/h2+x+9pkVUaZQbO1+K7nZzBqFQ2F4TMbLJW4tRcRtvLxw 26 | VCgi/VhcG6yAkgbuxPZdwrOQBZ5DR9BtkjllJuXB+z0az7IQDYWtq9UIgB65/cHF 27 | cand0dFvlLscZRSHIGNyfhiR8D20fZW/zL9MPNg1qO2WNjxSSmG5Wm1Lp/qwTcA1 28 | bvhWrIOtAwNxoN0PZjoQTOsozog5MtT6J9MiOTNTewQ62675aCjogqzEQDsEJ1aK 29 | 2mddVsoyYAkKAOXOG/wHbbaI92Sl685u9k5akpFrtsdOGGqk8FRLvxbB/nwoeAsH 30 | Eyrfz6lZewWbDQ1yTorP7sLkfqXL0w0glhfPTiTSMABrXjm8lZp44SJ+MS/fycXt 31 | QeZnH5W9UKbUpCE2yswIxcyfU8f2wJWPxp4xptSAq/42EuC6EQ== 32 | -----END CERTIFICATE----- 33 | -------------------------------------------------------------------------------- /src/test/resources/security/certs/client.csr: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIIE4jCCAsoCAQAwgYMxCzAJBgNVBAYTAlBMMRIwEAYDVQQIDAlQb21vcnNraWUx 3 | DzANBgNVBAcMBkdkeW5pYTEMMAoGA1UECgwDR0lEMQwwCgYDVQQLDANEZXYxEjAQ 4 | BgNVBAMMCWxvY2FsaG9zdDEfMB0GCSqGSIb3DQEJARYQdGVzdEBleGFtcGxlLmNv 5 | bTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMTEiSqoO2vKwRw6uOY/ 6 | NO19A77dDOB4Dqsdwn1aR4jkaRZRdQgmyqWdB8bets2//ICkzIbl0CbGab5eRq6U 7 | KKheIIh7q06Q0PpkjevSmDtvwcsLoN9oYxWJ03x14FFoMH+zSwaN3QvpdZW+9+0O 8 | 4YbBNBM7Uz4F3bJdf+DmdsgNEr1KkGydL9eUo5jMTy+15mQeJmlkLBGwDZib7Q15 9 | g9AdCzYqXhWPNzleWKBcN8Wvje4UEJYWHknz1tpakUUg2OFur4WhyooqkBrheU7v 10 | LMv+Fu+lxMU3BCoR9y/X+8qYR/BLkGVWunQ5Ze2/afBKpzrCeQZZAJOQ1ei8klv+ 11 | 671adxFla7DI8T3PJ4Vu2Be5kgJexWBmMlCE86oGgrYkRaWo5V18TX+fMA0JUW54 12 | qN86pC/c2oZ7C94uMHvGUZtOxa2M6E6Fke9dUebmGccm/jezdpl+jkc/X0SzpVKp 13 | 9vxapr0lZIaYc7i714IgvQUqe3PWAhQ+rg3CVMbfVsl2imzl5hJglbz7UMwaE+hd 14 | lBsQaak9Mn2M/GQGIRtXOC675kdrAMqB96H0t+6nWuXrsNzzXetvIXU/FERT5iSH 15 | vKNZiisR2wYDFBIkglXKEMNxejGWjKE3H/byV2l/cGqbh16dpFOjf6VxkBfddVQ3 16 | oITKZjyWWcvD2jdrp/spY4lPAgMBAAGgGTAXBgkqhkiG9w0BCQcxCgwIcGFzc3dv 17 | cmQwDQYJKoZIhvcNAQELBQADggIBAIaNmHL2TwnhjfKZNipnKNE0Akfj9GtYRsCw 18 | xeXZ66DS9gnAPj9ZoBkHci1I1sSxgGMYG9t2a4VAyFHKPtvEbVanYHkVJJBeIkwb 19 | 4JzC8+JucASp/8X1VnTvvZcoHnhknVJ6rC6AJXshm8W/vz1JTkqw4FGQ2Dt2vVWB 20 | ZUXGj3gG2iC+D2xGLPjtcTkcmJSSTJRhGMwyF2OJiDNw4s+W5guT70i/sVbcfIC0 21 | w1D5pStGHaIvKdVaFxNbuJRX525KvNmfkb41LXXRbgWvFLoKsW1ECfKrBb021uSL 22 | n6ICF1r4xKhw+WJOmlAXavVs9/RqeozEJPIvMS78pdMHdAn1wwQ4QkT0hGhXXN4M 23 | wfbUz9TPx/8Hp+09afd1a3f9l71HvfCFP+t7zIr/Rq6PgnhyxaeZlv+r8E3x8HgP 24 | h4/PUBGbJygtPkpsqpfqyhiV9SCzGT9a5YmhgZLFeeZOmPIt7KrCscrqEZcjWa96 25 | tm41VTpkwW4ZdYio3cadWdDXzzGT+3WPoBXTvuvYNooT5X8BESlTEgWcFkyjPGET 26 | 4xyaLqDlNKKGMPb5MQl5COYj6ZNQEqdyZfpMe744ytZ2UOHwfp44BDWbmE/xxbgA 27 | AnhnHyaO+Gm6Ahnj6MtW7nw3uTm6xgpHlEmjZ3EojihRoMRL4+CI/PgvOFyYeHH0 28 | XihqMyVs 29 | -----END CERTIFICATE REQUEST----- 30 | -------------------------------------------------------------------------------- /src/test/resources/security/certs/client.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIJKQIBAAKCAgEAxMSJKqg7a8rBHDq45j807X0Dvt0M4HgOqx3CfVpHiORpFlF1 3 | CCbKpZ0Hxt62zb/8gKTMhuXQJsZpvl5GrpQoqF4giHurTpDQ+mSN69KYO2/Bywug 4 | 32hjFYnTfHXgUWgwf7NLBo3dC+l1lb737Q7hhsE0EztTPgXdsl1/4OZ2yA0SvUqQ 5 | bJ0v15SjmMxPL7XmZB4maWQsEbANmJvtDXmD0B0LNipeFY83OV5YoFw3xa+N7hQQ 6 | lhYeSfPW2lqRRSDY4W6vhaHKiiqQGuF5Tu8sy/4W76XExTcEKhH3L9f7yphH8EuQ 7 | ZVa6dDll7b9p8EqnOsJ5BlkAk5DV6LySW/7rvVp3EWVrsMjxPc8nhW7YF7mSAl7F 8 | YGYyUITzqgaCtiRFpajlXXxNf58wDQlRbnio3zqkL9zahnsL3i4we8ZRm07FrYzo 9 | ToWR711R5uYZxyb+N7N2mX6ORz9fRLOlUqn2/FqmvSVkhphzuLvXgiC9BSp7c9YC 10 | FD6uDcJUxt9WyXaKbOXmEmCVvPtQzBoT6F2UGxBpqT0yfYz8ZAYhG1c4LrvmR2sA 11 | yoH3ofS37qda5euw3PNd628hdT8URFPmJIe8o1mKKxHbBgMUEiSCVcoQw3F6MZaM 12 | oTcf9vJXaX9wapuHXp2kU6N/pXGQF911VDeghMpmPJZZy8PaN2un+yljiU8CAwEA 13 | AQKCAgAXwH2qR4p8tYngBXqiWIjkDg4wsPBH/FYQ89OxW/98l83TotuUs1IM8cqp 14 | h+LbLUymnr4NnpokxBipoSm5EythP5N/xiXkoviG8TJsgAxeSFxFTU8LCMfFJtD/ 15 | mh8M1J3eHF93GBLRyEmoXNgCpqQ0pwfJNoUS3JRpXn38QPWfqmxTqzeJ1OOci1XA 16 | MRFx+ewos7cBS9VCAVVy0Yeo870klwzuaejXg2UwKKJcISWeWFWyontqEptvY97Q 17 | ltQoWxGzDLBlWiVfzkl85TDGhXqL2U0deo4pHD4Fcgu7m7/KMd2lNQnmIbT6KJ+U 18 | NmHsXqaXIjiHuqCmL3qS0Ih+UGRWTdS4s0XbFJxT279F4p9oY1ddCxQtkwh/Axva 19 | pY5t2tL7NFA5totFgdkZyXu5VDAa2cu0gBoa3YIoP6x7TKJFMqZKum846Qpivcyx 20 | aUGMkO5i3egyJzGTQHLfK0aKrWHK70qQSQDCXkqpakfPk4MwgrnlDDL0bwESTWU0 21 | /BZm7mzfFjSVqe0Tpw3dQT6+RRBRNK6bKcn9cXvln/1OYdVMAIwwDhFXpOePp0oO 22 | 1bLeVtjWrcf9WgxsVyySxPHgY4zJ0JgX8kXaC9tuFydcwQ2CGzgIK5zcOdBOW0ZJ 23 | 7j12Xb5NWTTE8R/ydmzqv6kjnRxGL6ihq7mAosLLGjxg1eKoIQKCAQEA7zgGNqYR 24 | abpLsnqaKhVieBBKZBplnKDmHafj1nBgE4oRtNF3139GvuX91UTrFWseqk7PEnZ2 25 | jp9BqyOWP5zhW9CDeB8QS3G6c9wg3BxmoCZRstB8cjJo+hSewXfLkEsZ/NjcF56F 26 | N5IDcpQHaVDZtCVTGVLozzRVx+Jk5a1s8Og2PmA5SpRV/5/qX2Z6Du6TUCreignB 27 | MGiFJ7CL4TMbXBWQgHccr52bifih8YoucJqHJlz3z5j8TWP73bMFvlHves2Ldo/q 28 | rzkIxiZphbPTxf4gAd+eHTjrtsE85gupPK8bP1MlOcxPQueYVQGh0eS+eiOtljYw 29 | 1R2kS0SN9vHA3wKCAQEA0pIoytOGokBY6tY451PoMtsNXOfcKwBw/EDgc2kMNi21 30 | 9EpTYk5mA7Qt/bt1HqgFbdokQFLQvKkAxMhDCvmom0+ZLpQhD+s1Qw0Edo8RSzze 31 | Qk4oMyrUE5UnQHdPYPpIQ+60s8dihYMhHJEuLLPpQaQnuUlP8FlZRx+gOfLJILWM 32 | V1hrNQwP9n8iac9T1806Lx3i/Up60Tl0BRRrYAaBPdTFupFCs7uB+OoCxoXNVUCt 33 | aWDD7FzvhjEDUvBRg3LgvbNBAaeU0QI57AyDssya/hhjI7TAWmD4itC9OzaXIk9Y 34 | Uj42qwn6TNZ54jjjzZSTvKwCWjlGVcHKAhV4D70VkQKCAQEAj0z1U+lZhwdlby5I 35 | iaszyI6hsaOqeRBdqSg4sO2ycCIGh1Xvv3TsEVE/JHHbjYIU5Zq+KuYyES/Pq9Bu 36 | of69dT3Mc7nQDhG7ysn0GUceJxOtXzvLQLF3909A5s53cCG9GJ1kAvYgAXEDhbCt 37 | W+Zl86dkR0hGVFGajVnObVPdbdb9H5KO32FKvw2x3aJKvgUm1CBKHVDInNIP6f21 38 | XZs66Ye1f29bDMySLy10/Xd0aFn5Wf7ZKFAR295oM5+ZcjAtEFhvEgAWaw70dZbv 39 | IXvfAb1AuT6WvhSqB1lsvLYBlGulG+MX6q68UGzh8RJAOLcZb5tN0iz2RcQy9i/E 40 | B0k86wKCAQEAsP0S9nBW0Je/ZRvMVnvLRruh/r4P68ECNrSdM8RO9Fb1gjDa5104 41 | F9LheUyQA8V8UroxEDPLqVdwAvbEizwPQQPloCX3qDat3P7bCn/0LEYsFpU06tUL 42 | zq3mgSQHf16/UQIZtqZXJBt0wFsy8XYDEB5XGUEKxN2E0eg+N5bfAPQ30s7JxZuu 43 | 6yQvG7qdIV0pk6LOQNHSa3ucDgz8t4igJwVpcQhZhVKMy1h+HrkmPosPduxToZ73 44 | 4NXfkl3WVeNADZLWh/qAdjEl2Ecohye8+ugqpD9wEkdBgQQ0mcmnvLnuZapPLvJD 45 | BUScy82/g/NnPgViMM3FVZ1+GsRK1we6UQKCAQBSj6uUQd+gsndR/heA4MqNgacW 46 | YMPB9IabxfNW1iwYMnUMyC2+3q3rL63HzMfWorGJENkGy5akueJ169X/0ALkJC1H 47 | 0HSuYeE5P/4mKEMXbC1QDNItxlQoPfelAYWdwPToi26T1BVk0XGCS3y4DRsM5/NB 48 | AyZVaib9F4rWFdOJMr2sqG45kgOl1ypbdwRE6Qe9fHZdHmnRLstnw9fpxFMOcp2+ 49 | dIHjp/bICS3hiIS8+n43dtD+a9t+gZWy0df3by83ANYlTiC2sBErHqNxgFXCVpvG 50 | FmFic0uXcmPvwpaY7FNghOx7qBC3k27fyML084FL+EuyWYostypsWSfMfkVo 51 | -----END RSA PRIVATE KEY----- 52 | -------------------------------------------------------------------------------- /src/test/resources/security/certs/clientPrivateKey.der: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/flink-http-connector/bc340abe08c54a79f7153e554008b061724505dd/src/test/resources/security/certs/clientPrivateKey.der -------------------------------------------------------------------------------- /src/test/resources/security/certs/clientPrivateKey.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDExIkqqDtrysEc 3 | OrjmPzTtfQO+3QzgeA6rHcJ9WkeI5GkWUXUIJsqlnQfG3rbNv/yApMyG5dAmxmm+ 4 | XkaulCioXiCIe6tOkND6ZI3r0pg7b8HLC6DfaGMVidN8deBRaDB/s0sGjd0L6XWV 5 | vvftDuGGwTQTO1M+Bd2yXX/g5nbIDRK9SpBsnS/XlKOYzE8vteZkHiZpZCwRsA2Y 6 | m+0NeYPQHQs2Kl4Vjzc5XligXDfFr43uFBCWFh5J89baWpFFINjhbq+FocqKKpAa 7 | 4XlO7yzL/hbvpcTFNwQqEfcv1/vKmEfwS5BlVrp0OWXtv2nwSqc6wnkGWQCTkNXo 8 | vJJb/uu9WncRZWuwyPE9zyeFbtgXuZICXsVgZjJQhPOqBoK2JEWlqOVdfE1/nzAN 9 | CVFueKjfOqQv3NqGewveLjB7xlGbTsWtjOhOhZHvXVHm5hnHJv43s3aZfo5HP19E 10 | s6VSqfb8Wqa9JWSGmHO4u9eCIL0FKntz1gIUPq4NwlTG31bJdops5eYSYJW8+1DM 11 | GhPoXZQbEGmpPTJ9jPxkBiEbVzguu+ZHawDKgfeh9Lfup1rl67Dc813rbyF1PxRE 12 | U+Ykh7yjWYorEdsGAxQSJIJVyhDDcXoxloyhNx/28ldpf3Bqm4denaRTo3+lcZAX 13 | 3XVUN6CEymY8llnLw9o3a6f7KWOJTwIDAQABAoICABfAfapHiny1ieAFeqJYiOQO 14 | DjCw8Ef8VhDz07Fb/3yXzdOi25SzUgzxyqmH4tstTKaevg2emiTEGKmhKbkTK2E/ 15 | k3/GJeSi+IbxMmyADF5IXEVNTwsIx8Um0P+aHwzUnd4cX3cYEtHISahc2AKmpDSn 16 | B8k2hRLclGleffxA9Z+qbFOrN4nU45yLVcAxEXH57CiztwFL1UIBVXLRh6jzvSSX 17 | DO5p6NeDZTAoolwhJZ5YVbKie2oSm29j3tCW1ChbEbMMsGVaJV/OSXzlMMaFeovZ 18 | TR16jikcPgVyC7ubv8ox3aU1CeYhtPoon5Q2YexeppciOIe6oKYvepLQiH5QZFZN 19 | 1LizRdsUnFPbv0Xin2hjV10LFC2TCH8DG9qljm3a0vs0UDm2i0WB2RnJe7lUMBrZ 20 | y7SAGhrdgig/rHtMokUypkq6bzjpCmK9zLFpQYyQ7mLd6DInMZNAct8rRoqtYcrv 21 | SpBJAMJeSqlqR8+TgzCCueUMMvRvARJNZTT8FmbubN8WNJWp7ROnDd1BPr5FEFE0 22 | rpspyf1xe+Wf/U5h1UwAjDAOEVek54+nSg7Vst5W2Natx/1aDGxXLJLE8eBjjMnQ 23 | mBfyRdoL224XJ1zBDYIbOAgrnNw50E5bRknuPXZdvk1ZNMTxH/J2bOq/qSOdHEYv 24 | qKGruYCiwssaPGDV4qghAoIBAQDvOAY2phFpukuyepoqFWJ4EEpkGmWcoOYdp+PW 25 | cGATihG00XfXf0a+5f3VROsVax6qTs8SdnaOn0GrI5Y/nOFb0IN4HxBLcbpz3CDc 26 | HGagJlGy0HxyMmj6FJ7Bd8uQSxn82NwXnoU3kgNylAdpUNm0JVMZUujPNFXH4mTl 27 | rWzw6DY+YDlKlFX/n+pfZnoO7pNQKt6KCcEwaIUnsIvhMxtcFZCAdxyvnZuJ+KHx 28 | ii5wmocmXPfPmPxNY/vdswW+Ue96zYt2j+qvOQjGJmmFs9PF/iAB354dOOu2wTzm 29 | C6k8rxs/UyU5zE9C55hVAaHR5L56I62WNjDVHaRLRI328cDfAoIBAQDSkijK04ai 30 | QFjq1jjnU+gy2w1c59wrAHD8QOBzaQw2LbX0SlNiTmYDtC39u3UeqAVt2iRAUtC8 31 | qQDEyEMK+aibT5kulCEP6zVDDQR2jxFLPN5CTigzKtQTlSdAd09g+khD7rSzx2KF 32 | gyEckS4ss+lBpCe5SU/wWVlHH6A58skgtYxXWGs1DA/2fyJpz1PXzTovHeL9SnrR 33 | OXQFFGtgBoE91MW6kUKzu4H46gLGhc1VQK1pYMPsXO+GMQNS8FGDcuC9s0EBp5TR 34 | AjnsDIOyzJr+GGMjtMBaYPiK0L07NpciT1hSPjarCfpM1nniOOPNlJO8rAJaOUZV 35 | wcoCFXgPvRWRAoIBAQCPTPVT6VmHB2VvLkiJqzPIjqGxo6p5EF2pKDiw7bJwIgaH 36 | Ve+/dOwRUT8kcduNghTlmr4q5jIRL8+r0G6h/r11PcxzudAOEbvKyfQZRx4nE61f 37 | O8tAsXf3T0DmzndwIb0YnWQC9iABcQOFsK1b5mXzp2RHSEZUUZqNWc5tU91t1v0f 38 | ko7fYUq/DbHdokq+BSbUIEodUMic0g/p/bVdmzrph7V/b1sMzJIvLXT9d3RoWflZ 39 | /tkoUBHb3mgzn5lyMC0QWG8SABZrDvR1lu8he98BvUC5Ppa+FKoHWWy8tgGUa6Ub 40 | 4xfqrrxQbOHxEkA4txlvm03SLPZFxDL2L8QHSTzrAoIBAQCw/RL2cFbQl79lG8xW 41 | e8tGu6H+vg/rwQI2tJ0zxE70VvWCMNrnXTgX0uF5TJADxXxSujEQM8upV3AC9sSL 42 | PA9BA+WgJfeoNq3c/tsKf/QsRiwWlTTq1QvOreaBJAd/Xr9RAhm2plckG3TAWzLx 43 | dgMQHlcZQQrE3YTR6D43lt8A9DfSzsnFm67rJC8bup0hXSmTos5A0dJre5wODPy3 44 | iKAnBWlxCFmFUozLWH4euSY+iw927FOhnvfg1d+SXdZV40ANktaH+oB2MSXYRyiH 45 | J7z66CqkP3ASR0GBBDSZyae8ue5lqk8u8kMFRJzLzb+D82c+BWIwzcVVnX4axErX 46 | B7pRAoIBAFKPq5RB36Cyd1H+F4Dgyo2BpxZgw8H0hpvF81bWLBgydQzILb7eresv 47 | rcfMx9aisYkQ2QbLlqS54nXr1f/QAuQkLUfQdK5h4Tk//iYoQxdsLVAM0i3GVCg9 48 | 96UBhZ3A9OiLbpPUFWTRcYJLfLgNGwzn80EDJlVqJv0XitYV04kyvayobjmSA6XX 49 | Klt3BETpB718dl0eadEuy2fD1+nEUw5ynb50geOn9sgJLeGIhLz6fjd20P5r236B 50 | lbLR1/dvLzcA1iVOILawESseo3GAVcJWm8YWYWJzS5dyY+/ClpjsU2CE7HuoELeT 51 | bt/IwvTzgUv4S7JZiiy3KmxZJ8x+RWg= 52 | -----END PRIVATE KEY----- 53 | -------------------------------------------------------------------------------- /src/test/resources/security/certs/client_keyStore.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/flink-http-connector/bc340abe08c54a79f7153e554008b061724505dd/src/test/resources/security/certs/client_keyStore.p12 -------------------------------------------------------------------------------- /src/test/resources/security/certs/server.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFkTCCA3kCFDegdjvVXp8sOROa5rAzXN7SQs8oMA0GCSqGSIb3DQEBCwUAMIGD 3 | MQswCQYDVQQGEwJQTDESMBAGA1UECAwJUG9tb3Jza2llMQ8wDQYDVQQHDAZHZHlu 4 | aWExDDAKBgNVBAoMA0dJRDEMMAoGA1UECwwDRGV2MRIwEAYDVQQDDAlsb2NhbGhv 5 | c3QxHzAdBgkqhkiG9w0BCQEWEHRlc3RAZXhhbXBsZS5jb20wIBcNMjIwOTAyMTAx 6 | MTM0WhgPMjA1MDAxMTgxMDExMzRaMIGDMQswCQYDVQQGEwJQTDESMBAGA1UECAwJ 7 | UG9tb3Jza2llMQ8wDQYDVQQHDAZHZHluaWExDDAKBgNVBAoMA0dJRDEMMAoGA1UE 8 | CwwDRGV2MRIwEAYDVQQDDAlsb2NhbGhvc3QxHzAdBgkqhkiG9w0BCQEWEHRlc3RA 9 | ZXhhbXBsZS5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCoiGal 10 | V6QZ3OVoje3uWFeIvAHT+wlhDjnhg4RseaiC2XkSiOPgpgKIFIfmPviPFJPjrzU9 11 | 95DS3Sjb8va2CdWPyC+zZWqxZtom5eREXvQDW3jfKlsZOrcgyeTXaASs6Y8dXzzA 12 | 10KpeJrR+/qmSs1a781YcPSVGvazFTFrYyJJ5mLmqIUOokNhjUHuRveDnkxj4vJe 13 | rnVkoQasjAkJcVbCBWTtGu60ygPhsvU627jk2IENGhRDznDxkZjByBDKng1N99lr 14 | dASDSFJN8xir5ocV70560IAoyg/qMnZuF0U7np+mkYyYfUcnyijSsTZ4shx2NLZq 15 | 6KtuzZ7lcfdVV0o9pt3K+trt6eO9cQA88KJZ/k7icP0lzxUCafQyQaUxzwycrjZN 16 | Req/97FSxJs9ldE1XE278ruSs+5K/AIfoxTedGteFg10ZomyfK0TISD27tuptIqq 17 | GtN6BHEMKol/pK/wken+nTHAHu+XYQRUh+JeHgKPDukUIG0mVKLMQ9b4a3XzOlbn 18 | yNbLL21Zyhgdzo7SnNRjG1vyVA+xiFUTKBChj/r5FqRa/g1ZYl0mOHza3YowC1YC 19 | adwUAtl60MzVRmXudP7d3IdMe1bfGHEr3l9vzE12MctTmtB8yiMN2JrJxGeQwGMt 20 | k91DSPbscmHR0bYlpChc5GRZHVbLuEwApCaCgwIDAQABMA0GCSqGSIb3DQEBCwUA 21 | A4ICAQAfiVuy2nMJknfmIZRWVfRv87Afq2bIM4t4jz1tsTM69TcOKMWRaa/zVysH 22 | iqCd5LxCEu08+OxmzvN1aMx59mXp4V3Hs/FGVmh5NEReawWjj5S63AXfr9by0ED9 23 | LnC9G61o6IiWNOZ9kVhXlRNz2tTHP1v593Z34SzQxECjTasrF9jSEDg6kYrE3ZJy 24 | ffcPzDst6MP1eZJxWnap/EvKGirJZNlRWFQJEMGtZEPASN93hAWkwDwyBJqE4rK3 25 | kk9PqmbxOKKPHIcZWdiMlBO7u4Q7p2JvCjP3Vf31cX3bt4BsLbkDtE5i8+woLcsL 26 | BA3+WzupjZPxW014AVwcADFMNsyUye+T8FpK332/Sl5IvSU3hGyoZ51IyJeDKAE/ 27 | soefoy8ZuEh3nM9hlBrrSnOIDPsOuS4GiQXr5HWThCFdfT9MPI3D49OqNNnS4aN9 28 | W3zlgwCMDlOjZ5D8tKJcD3sIr3f0dlk0aFS6W/A0T5fN0HNStnPQUFD4ldvTAOHE 29 | j76eWDbRXBp/2TZ8MP2TlfFMV95VTg9bT217XZcdXAbWTcUwzQ9rStWn4WkHNQt/ 30 | h9H6EOkxYBnegpbQdQ/+2Y7wRX4crJkn5ODgerdcmf1CaQLNJbEFQmSTvqdW/gaT 31 | 0clo0uAzNjnEMDYuZzVkCtf1I4vz0NEN/7ySUcO0aYE/EAN48w== 32 | -----END CERTIFICATE----- 33 | -------------------------------------------------------------------------------- /src/test/resources/security/certs/server.csr: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIIE4jCCAsoCAQAwgYMxCzAJBgNVBAYTAlBMMRIwEAYDVQQIDAlQb21vcnNraWUx 3 | DzANBgNVBAcMBkdkeW5pYTEMMAoGA1UECgwDR0lEMQwwCgYDVQQLDANEZXYxEjAQ 4 | BgNVBAMMCWxvY2FsaG9zdDEfMB0GCSqGSIb3DQEJARYQdGVzdEBleGFtcGxlLmNv 5 | bTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKiIZqVXpBnc5WiN7e5Y 6 | V4i8AdP7CWEOOeGDhGx5qILZeRKI4+CmAogUh+Y++I8Uk+OvNT33kNLdKNvy9rYJ 7 | 1Y/IL7NlarFm2ibl5ERe9ANbeN8qWxk6tyDJ5NdoBKzpjx1fPMDXQql4mtH7+qZK 8 | zVrvzVhw9JUa9rMVMWtjIknmYuaohQ6iQ2GNQe5G94OeTGPi8l6udWShBqyMCQlx 9 | VsIFZO0a7rTKA+Gy9TrbuOTYgQ0aFEPOcPGRmMHIEMqeDU332Wt0BINIUk3zGKvm 10 | hxXvTnrQgCjKD+oydm4XRTuen6aRjJh9RyfKKNKxNniyHHY0tmroq27NnuVx91VX 11 | Sj2m3cr62u3p471xADzwoln+TuJw/SXPFQJp9DJBpTHPDJyuNk1F6r/3sVLEmz2V 12 | 0TVcTbvyu5Kz7kr8Ah+jFN50a14WDXRmibJ8rRMhIPbu26m0iqoa03oEcQwqiX+k 13 | r/CR6f6dMcAe75dhBFSH4l4eAo8O6RQgbSZUosxD1vhrdfM6VufI1ssvbVnKGB3O 14 | jtKc1GMbW/JUD7GIVRMoEKGP+vkWpFr+DVliXSY4fNrdijALVgJp3BQC2XrQzNVG 15 | Ze50/t3ch0x7Vt8YcSveX2/MTXYxy1Oa0HzKIw3YmsnEZ5DAYy2T3UNI9uxyYdHR 16 | tiWkKFzkZFkdVsu4TACkJoKDAgMBAAGgGTAXBgkqhkiG9w0BCQcxCgwIcGFzc3dv 17 | cmQwDQYJKoZIhvcNAQELBQADggIBAJeukQjBorFW+FMnNuLVBOws9xrkScSBAWq5 18 | FucSFE75FjGVEeOSrWQZlw4zt/trlFgK5f18HHQ2DwftdDcyLKitCVoH+dOB1lXs 19 | xI1afWiFS/2XhSLriXcCX8sk2AoWgHOyrPSVCVa8EonDp1sjs3FBN+VvgtNqscUf 20 | Vho1xdSYST3xG7DGOtN+4IiFfArGzWLrJ/++YGsbfnufNlXgYRqw8GvCKMauUkUN 21 | 4J8PBljbCZ9/3Wtxb0UrLpN4Lmm0WyPnzRulsFW2nzibR410NI6AJ6zzXxZQw0Jb 22 | 3w7sXVI1yQc33fQMTU5j2HwBhh8AldxPZ4BHsF9foaJMr/0ffOcVjLL/wHI5knIt 23 | 1YI+kQ/liyg3aimUrd0HbZbI9dV8fDp83gQuBojqTPprPyK3bVab9mymOXzOxDvW 24 | Gl7b44UH+vKfExVRDwR4pYVAB0CaEFFqYaLF9HIQNGHy6LC1KWoKKLXwKeaSakVC 25 | 5Pf7fVeBuUA8JBlAO3gWONrDvvuaC7tX4lBD8lfsSbs9XFoNwKRuL3huvanYCN2S 26 | wfM+cNo2DPAV6Er49TlO5UaNt9TyyF5x/yoO/kXpEffheS0sMYwVAKRC5sq7tFBu 27 | yzw5AgwR3MCim6v8LT/bfvkLWmaxzcJjYtHX5AWYq5UOKa4MUUIoapvXzvdSgPzs 28 | 4RyxqMwu 29 | -----END CERTIFICATE REQUEST----- 30 | -------------------------------------------------------------------------------- /src/test/resources/security/certs/server.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIJKQIBAAKCAgEAqIhmpVekGdzlaI3t7lhXiLwB0/sJYQ454YOEbHmogtl5Eojj 3 | 4KYCiBSH5j74jxST4681PfeQ0t0o2/L2tgnVj8gvs2VqsWbaJuXkRF70A1t43ypb 4 | GTq3IMnk12gErOmPHV88wNdCqXia0fv6pkrNWu/NWHD0lRr2sxUxa2MiSeZi5qiF 5 | DqJDYY1B7kb3g55MY+LyXq51ZKEGrIwJCXFWwgVk7RrutMoD4bL1Otu45NiBDRoU 6 | Q85w8ZGYwcgQyp4NTffZa3QEg0hSTfMYq+aHFe9OetCAKMoP6jJ2bhdFO56fppGM 7 | mH1HJ8oo0rE2eLIcdjS2auirbs2e5XH3VVdKPabdyvra7enjvXEAPPCiWf5O4nD9 8 | Jc8VAmn0MkGlMc8MnK42TUXqv/exUsSbPZXRNVxNu/K7krPuSvwCH6MU3nRrXhYN 9 | dGaJsnytEyEg9u7bqbSKqhrTegRxDCqJf6Sv8JHp/p0xwB7vl2EEVIfiXh4Cjw7p 10 | FCBtJlSizEPW+Gt18zpW58jWyy9tWcoYHc6O0pzUYxtb8lQPsYhVEygQoY/6+Rak 11 | Wv4NWWJdJjh82t2KMAtWAmncFALZetDM1UZl7nT+3dyHTHtW3xhxK95fb8xNdjHL 12 | U5rQfMojDdiaycRnkMBjLZPdQ0j27HJh0dG2JaQoXORkWR1Wy7hMAKQmgoMCAwEA 13 | AQKCAgEAjGBw/y+s5E32V8xAbtLutlnDPApRmaH73Ddryd7b2YHDMwkKpkggu7A+ 14 | Mi4pYyREutVph5/55WmJawtJK/+s/wJDmS3UXMfK1bOPGtPsdA9w8pGZJ76cYpdI 15 | IZPmsRApwNFfDVE/mWF3s4grEvAjdKdhZ1VOpBO9gRUaANSl7uATaQTdL39AmLTM 16 | QeiiqeLkxh52ux6dcaAeqwJJVLAvD5TmqAhS5AuQiCJD2S7DJvqfI4ahcok10kDv 17 | +Bjow5w6gML9Ol98ks1eoMbnuedehYq619lrpo4IC47UzoZbiHlriZaagTl4V1Aw 18 | CK7LLGVPB7aIayw5/yUOQRdYaKE4mB9XQzt4xec7Q+oYubC0jo4NxbaOzXfUT+az 19 | qO1PjBy4XP5GG4nFU8NbG1GZ3jZUFPOr+/t558ckhTmWfXgec6xga5+Y3lEfd+lX 20 | yuxqhjamWMf9CfInNq4zWp/P6XsrVdftawawBfP8/Clk9DWbACK4twLmhiKzbrTP 21 | uE+vsf0fqMjjfx1EPSpuBVMRXAZoKjFnAVEMhUSCSqSBgqWkJQTO82kpL6DWmSTo 22 | axYCTtjafB7Z8xJE7u2gBn5DG3iG4WyKosHyVxfoOJtsFSa3Nx9OdMFubtGWAq9K 23 | aXmvtb0HZMiuJ2+LcH8eMUH0uo3dOQZ85SrjhxVvMbYSbiLx5OECggEBANjtL41U 24 | NOPZz7JiyB+3mDuzlAEzWk5pHGn5H96LftlCdI4+o3O/A8ZCpskxMnplmlnwQ/X/ 25 | 4vTGSul+bzRFhhd/v4sgeDkypxKDEhOH+T63yYD9THHxdY5pNCaamNiNoXiDTlcP 26 | X+yk8YH4w406A2sobB+OpF1A/pWkkvFRy7gqPJVSS686xFrsrZswRqheY6x8E+L8 27 | NFF059oAA6chyMpBSC9Vgu1x9zP2NuqMRKZSeSxANMGahxfxiqznt0b7gq2m64g7 28 | zSzqlPHGElKbaLwGpeeP+S94pefvVdCzD8XSGZ/mjOT+niqVD4HlkMj3USACX2CX 29 | jL3RaL9S7BxuOh8CggEBAMbjtTVOcLCkySii4Opv5A6/J8YANrvYyg3YceuuJhS9 30 | Qu/8pQm/slKj+LOs5nhvGRZuunOsWGuHT7/YnT/NDr2VYrPMctEJ8QR0rma7cDkh 31 | 3wvQEA8Jjo5VFpkqYRvCyPDBJd8sFvsu+mfUEhCD5eFQO630Byh7GqHxdnWxV+9y 32 | eo7TPKIoN8JPfQmZyitUBAte4YmvMXcUWSJpD7H5UgYuLnlJEAgGF6yMCh42C9Yw 33 | /oEwK0fpTc0kDyM7HiF4cYC3fOp0At6MoJL1iBi8U/GUeXANnknHqKMmHX8JQEQK 34 | wUeajBjJX1fT26AmuxDs3oQVRyTNe6/8NU7UtMKycx0CggEAIjtbs6vEJOisU/Qu 35 | rIKmbVVrV70o1RLVPwvnMZFF1TcdeYz88jkEeXbvJvSyiIocD3FQ6DMXN5VLIQmo 36 | ttD51h98tJHNJwmNP+ibI6XjeTg7fjV6qMh3yp9jcWwYDqRbBM+Dv01Xz9o/mkkN 37 | Yu3yuChK17gEzAqDok20Ooyz3z4bDV1NQDu/3LFYvBKkM9GhXQi/gqRWBTyzkb1K 38 | 1kcIjpI6t+2AmcPQRMzn4Z98/dop8Qe4y4x9IRQm1B5E19i8yHZ82KTHPf1Ov9CH 39 | 9606m+FYsPARXKJnW/By1FpNdOQw0wDo3wInJ4UaS19Ut8H/dVOqhW2k8hdntjl5 40 | tQrX9QKCAQEAgC3u8kz8aYu6LRtT8yNriml9+6jVnZjpF6VAPlE9L66OPlbIEpFe 41 | Hs6YkUvEOSf4tjFkrvoK/jn1lsHHDoBcmKh3NUN1V+2dTG7am1D/0YJX2qgcQWS6 42 | YFbMOqhKb3bIxAjnHH6wzO+I6HOPtN8cv2pTHBgiqN812drhqpZCAwr3m6bd8/AR 43 | h4RtJEveNLBv7tgWaQLk0Ubm7ztgpyX0zBDlWLtSvqkWhti2CKRyWne2/pGJw03E 44 | G4q6yszl7VJPbFJVClD8LGvbbPsa3D47p302CpZsRkaME6GT7vSKW4/G4xviYHMf 45 | pF3gtMVhWFYgTmXcNQbNSS01OfnyCWyyrQKCAQBX1M/venfxexle8+HfiJ5Dd7DB 46 | JgdoSr5zx0eCiLnzEUskdLhVhhVKED6CXzPLGH/9go+6Z+hoSDnYEuDn7Uq4uqyn 47 | NwfHKJfVl0uA8cvC91hvW3Cd1hXdc7ib2WXij87Bi4mSYP8qV1Xt/+biymOd7NvA 48 | i6mbN/j+YQN/IOdQDOwJwSAdygPoFqO3PHlqNz8gw7/es9Ed2Ru6+9ID4izx5COr 49 | qdW0YATmMtYFpMOo255kmgFiF80Vzg8p4Vmh4Hj3p8b3OhqR7fTdffKeDub6c/bi 50 | Q36Bfutsk7V04B7u5AxpOKBRD6zKVzE94u1bn+rbf9cYwaEst59YVvS+NTgG 51 | -----END RSA PRIVATE KEY----- 52 | -------------------------------------------------------------------------------- /src/test/resources/security/certs/serverKeyStore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/flink-http-connector/bc340abe08c54a79f7153e554008b061724505dd/src/test/resources/security/certs/serverKeyStore.jks -------------------------------------------------------------------------------- /src/test/resources/security/certs/serverTrustStore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/flink-http-connector/bc340abe08c54a79f7153e554008b061724505dd/src/test/resources/security/certs/serverTrustStore.jks --------------------------------------------------------------------------------