├── .gitattributes ├── .github ├── dependabot.yml └── workflows │ ├── release.yml │ └── test.yml ├── .gitignore ├── LICENSE ├── README.md ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── lib ├── build.gradle └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── dmathieu │ │ │ └── kafka │ │ │ └── opensearch │ │ │ ├── ConfigCallbackHandler.java │ │ │ ├── DataConverter.java │ │ │ ├── EnumRecommender.java │ │ │ ├── Mapping.java │ │ │ ├── OffsetTracker.java │ │ │ ├── OpenSearchClient.java │ │ │ ├── OpenSearchSinkConnector.java │ │ │ ├── OpenSearchSinkConnectorConfig.java │ │ │ ├── OpenSearchSinkTask.java │ │ │ ├── RetryUtil.java │ │ │ ├── UnsafeX509ExtendedTrustManager.java │ │ │ ├── Validator.java │ │ │ └── Version.java │ └── resources │ │ └── kafka-connect-opensearch-version.properties │ └── test │ ├── java │ └── com │ │ └── dmathieu │ │ └── kafka │ │ └── opensearch │ │ ├── DataConverterTest.java │ │ ├── MappingTest.java │ │ ├── OffsetTrackerTest.java │ │ ├── OpenSearchClientTest.java │ │ ├── OpenSearchSinkConnectorConfigTest.java │ │ ├── OpenSearchSinkConnectorTest.java │ │ ├── OpenSearchSinkTaskTest.java │ │ ├── PartitionPauserTest.java │ │ ├── RetryUtilTest.java │ │ ├── ValidatorTest.java │ │ ├── helper │ │ ├── NetworkErrorContainer.java │ │ ├── OpenSearchContainer.java │ │ └── OpenSearchHelperClient.java │ │ └── integration │ │ ├── BaseConnectorIT.java │ │ ├── BlockingTransformer.java │ │ ├── OpenSearchConnectorBaseIT.java │ │ ├── OpenSearchConnectorIT.java │ │ ├── OpenSearchConnectorKerberosIT.java │ │ ├── OpenSearchConnectorNetworkIT.java │ │ └── OpenSearchSinkTaskIT.java │ └── resources │ ├── default │ ├── instances.yml │ ├── opensearch.yml │ └── start-opensearch.sh │ ├── kerberos │ ├── instances.yml │ └── opensearch.yml │ ├── log4j.properties │ └── mockito-extensions │ └── org.mockito.plugins.MockMaker └── settings.gradle /.gitattributes: -------------------------------------------------------------------------------- 1 | # 2 | # https://help.github.com/articles/dealing-with-line-endings/ 3 | # 4 | # These are explicitly windows files and should use crlf 5 | *.bat text eol=crlf 6 | 7 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gradle 4 | directory: "/" 5 | schedule: 6 | interval: weekly 7 | time: "04:00" 8 | open-pull-requests-limit: 10 9 | reviewers: 10 | - dmathieu 11 | allow: 12 | - dependency-type: direct 13 | - dependency-type: indirect 14 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | release: 4 | types: [published] 5 | 6 | jobs: 7 | release: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Check out repository 11 | uses: actions/checkout@v2 12 | with: 13 | fetch-depth: 0 14 | - name: Set up JDK 11 15 | uses: actions/setup-java@v2 16 | with: 17 | java-version: '11' 18 | distribution: 'adopt' 19 | cache: gradle 20 | - name: Set Version 21 | env: 22 | TAG_NAME: ${{ github.event.release.tag_name }} 23 | run: sed -i "s/DEV/${TAG_NAME}/g" lib/src/main/resources/kafka-connect-opensearch-version.properties 24 | - name: Build project 25 | run: |- 26 | ./gradlew shadowJar 27 | mv lib/build/libs/lib-all.jar kafka-connect-opensearch.jar 28 | - name: Release 29 | uses: softprops/action-gh-release@v1 30 | if: startsWith(github.ref, 'refs/tags/') 31 | with: 32 | files: kafka-connect-opensearch.jar 33 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Java CI with Gradle 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | build: 6 | runs-on: ubuntu-latest 7 | steps: 8 | - uses: actions/checkout@v2 9 | - name: Set up JDK 11 10 | uses: actions/setup-java@v2 11 | with: 12 | java-version: '11' 13 | distribution: 'adopt' 14 | cache: gradle 15 | - name: Build with Gradle 16 | run: ./gradlew build 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore Gradle project-specific cache directory 2 | .gradle 3 | 4 | # Ignore Gradle build output directory 5 | build 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dmathieu/kafka-connect-opensearch/26d7c2d4d6d3c497ef4ad9e275be3339d2ec435d/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kafka Connect OpenSearch Connector 2 | 3 | **DEPRECATED**: This repoitory is deprecated. Please use [the aiven one](https://github.com/aiven/opensearch-connector-for-apache-kafka) instead. 4 | 5 | 6 | kafka-connect-opensearch is a fork of Confluent's [kafka-connect-elasticsearch](https://github.com/confluentinc/kafka-connect-elasticsearch). 7 | It allows runnig a [Kafka Connector](http://kafka.apache.org/documentation.html#connect) for copying data between Kafka and OpenSearch. 8 | 9 | # Usage 10 | 11 | ## Installation 12 | 13 | Each release publishes a JAR of the package which can be downloaded and used directly. 14 | 15 | ``` 16 | KAFKA_CONNECT_OPENSEARCH_VERSION=0.0.2 17 | curl -L -o /tmp/kafka-connect-opensearch.jar https://github.com/dmathieu/kafka-connect-opensearch/releases/download/${KAFKA_CONNECT_OPENSEARCH_VERSION}/kafka-connect-opensearch.jar 18 | ``` 19 | 20 | Once you have the JAR file, move it to a path recognised by Java, such as `/usr/share/java/`. 21 | 22 | ## Configuration 23 | 24 | Once the JAR is installed, you can configure a connector using it. Create your connector using the `com.dmathieu.kafka.opensearch.OpenSearchSinkConnector` class. 25 | 26 | ``` 27 | { 28 | [...] 29 | "connector.class": "com.dmathieu.kafka.opensearch.OpenSearchSinkConnector", 30 | [...] 31 | } 32 | ``` 33 | 34 | # About the license 35 | 36 | Everything coming from the Confluent fork is licensed under the [Confluent Community License](LICENSE). 37 | Everything that has changed from the Elasticsearch version is licensed under the MIT license. 38 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dmathieu/kafka-connect-opensearch/26d7c2d4d6d3c497ef4ad9e275be3339d2ec435d/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | # 21 | # Gradle start up script for POSIX generated by Gradle. 22 | # 23 | # Important for running: 24 | # 25 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 26 | # noncompliant, but you have some other compliant shell such as ksh or 27 | # bash, then to run this script, type that shell name before the whole 28 | # command line, like: 29 | # 30 | # ksh Gradle 31 | # 32 | # Busybox and similar reduced shells will NOT work, because this script 33 | # requires all of these POSIX shell features: 34 | # * functions; 35 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 36 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 37 | # * compound commands having a testable exit status, especially «case»; 38 | # * various built-in commands including «command», «set», and «ulimit». 39 | # 40 | # Important for patching: 41 | # 42 | # (2) This script targets any POSIX shell, so it avoids extensions provided 43 | # by Bash, Ksh, etc; in particular arrays are avoided. 44 | # 45 | # The "traditional" practice of packing multiple parameters into a 46 | # space-separated string is a well documented source of bugs and security 47 | # problems, so this is (mostly) avoided, by progressively accumulating 48 | # options in "$@", and eventually passing that to Java. 49 | # 50 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 51 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 52 | # see the in-line comments for details. 53 | # 54 | # There are tweaks for specific operating systems such as AIX, CygWin, 55 | # Darwin, MinGW, and NonStop. 56 | # 57 | # (3) This script is generated from the Groovy template 58 | # https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 59 | # within the Gradle project. 60 | # 61 | # You can find Gradle at https://github.com/gradle/gradle/. 62 | # 63 | ############################################################################## 64 | 65 | # Attempt to set APP_HOME 66 | 67 | # Resolve links: $0 may be a link 68 | app_path=$0 69 | 70 | # Need this for daisy-chained symlinks. 71 | while 72 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 73 | [ -h "$app_path" ] 74 | do 75 | ls=$( ls -ld "$app_path" ) 76 | link=${ls#*' -> '} 77 | case $link in #( 78 | /*) app_path=$link ;; #( 79 | *) app_path=$APP_HOME$link ;; 80 | esac 81 | done 82 | 83 | APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit 84 | 85 | APP_NAME="Gradle" 86 | APP_BASE_NAME=${0##*/} 87 | 88 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 89 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 90 | 91 | # Use the maximum available, or set MAX_FD != -1 to use that value. 92 | MAX_FD=maximum 93 | 94 | warn () { 95 | echo "$*" 96 | } >&2 97 | 98 | die () { 99 | echo 100 | echo "$*" 101 | echo 102 | exit 1 103 | } >&2 104 | 105 | # OS specific support (must be 'true' or 'false'). 106 | cygwin=false 107 | msys=false 108 | darwin=false 109 | nonstop=false 110 | case "$( uname )" in #( 111 | CYGWIN* ) cygwin=true ;; #( 112 | Darwin* ) darwin=true ;; #( 113 | MSYS* | MINGW* ) msys=true ;; #( 114 | NONSTOP* ) nonstop=true ;; 115 | esac 116 | 117 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 118 | 119 | 120 | # Determine the Java command to use to start the JVM. 121 | if [ -n "$JAVA_HOME" ] ; then 122 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 123 | # IBM's JDK on AIX uses strange locations for the executables 124 | JAVACMD=$JAVA_HOME/jre/sh/java 125 | else 126 | JAVACMD=$JAVA_HOME/bin/java 127 | fi 128 | if [ ! -x "$JAVACMD" ] ; then 129 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 130 | 131 | Please set the JAVA_HOME variable in your environment to match the 132 | location of your Java installation." 133 | fi 134 | else 135 | JAVACMD=java 136 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 137 | 138 | Please set the JAVA_HOME variable in your environment to match the 139 | location of your Java installation." 140 | fi 141 | 142 | # Increase the maximum file descriptors if we can. 143 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 144 | case $MAX_FD in #( 145 | max*) 146 | MAX_FD=$( ulimit -H -n ) || 147 | warn "Could not query maximum file descriptor limit" 148 | esac 149 | case $MAX_FD in #( 150 | '' | soft) :;; #( 151 | *) 152 | ulimit -n "$MAX_FD" || 153 | warn "Could not set maximum file descriptor limit to $MAX_FD" 154 | esac 155 | fi 156 | 157 | # Collect all arguments for the java command, stacking in reverse order: 158 | # * args from the command line 159 | # * the main class name 160 | # * -classpath 161 | # * -D...appname settings 162 | # * --module-path (only if needed) 163 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 164 | 165 | # For Cygwin or MSYS, switch paths to Windows format before running java 166 | if "$cygwin" || "$msys" ; then 167 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 168 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 169 | 170 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 171 | 172 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 173 | for arg do 174 | if 175 | case $arg in #( 176 | -*) false ;; # don't mess with options #( 177 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 178 | [ -e "$t" ] ;; #( 179 | *) false ;; 180 | esac 181 | then 182 | arg=$( cygpath --path --ignore --mixed "$arg" ) 183 | fi 184 | # Roll the args list around exactly as many times as the number of 185 | # args, so each arg winds up back in the position where it started, but 186 | # possibly modified. 187 | # 188 | # NB: a `for` loop captures its iteration list before it begins, so 189 | # changing the positional parameters here affects neither the number of 190 | # iterations, nor the values presented in `arg`. 191 | shift # remove old arg 192 | set -- "$@" "$arg" # push replacement arg 193 | done 194 | fi 195 | 196 | # Collect all arguments for the java command; 197 | # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of 198 | # shell script including quotes and variable substitutions, so put them in 199 | # double quotes to make sure that they get re-expanded; and 200 | # * put everything else in single quotes, so that it's not re-expanded. 201 | 202 | set -- \ 203 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 204 | -classpath "$CLASSPATH" \ 205 | org.gradle.wrapper.GradleWrapperMain \ 206 | "$@" 207 | 208 | # Use "xargs" to parse quoted args. 209 | # 210 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 211 | # 212 | # In Bash we could simply go: 213 | # 214 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 215 | # set -- "${ARGS[@]}" "$@" 216 | # 217 | # but POSIX shell has neither arrays nor command substitution, so instead we 218 | # post-process each arg (as a line of input to sed) to backslash-escape any 219 | # character that might be a shell metacharacter, then use eval to reverse 220 | # that process (while maintaining the separation between arguments), and wrap 221 | # the whole thing up as a single "set" statement. 222 | # 223 | # This will of course break if any of these variables contains a newline or 224 | # an unmatched quote. 225 | # 226 | 227 | eval "set -- $( 228 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 229 | xargs -n1 | 230 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 231 | tr '\n' ' ' 232 | )" '"$@"' 233 | 234 | exec "$JAVACMD" "$@" 235 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /lib/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'com.github.johnrengelman.shadow' version '7.1.2' 3 | id 'java' 4 | } 5 | 6 | repositories { 7 | mavenCentral() 8 | 9 | maven { 10 | url 'https://packages.confluent.io/maven/' 11 | } 12 | } 13 | 14 | ext { 15 | kafkaVersion = "6.2.1-ce" 16 | esVersion = "7.9.3" 17 | } 18 | 19 | dependencies { 20 | implementation 'io.confluent:common-utils:7.1.1' 21 | 22 | implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:$esVersion" 23 | implementation "org.elasticsearch:elasticsearch:$esVersion" 24 | 25 | implementation "org.apache.kafka:connect-api:$kafkaVersion" 26 | implementation "org.apache.kafka:connect-json:$kafkaVersion" 27 | implementation "org.apache.kafka:kafka-streams:$kafkaVersion" 28 | 29 | testImplementation "org.apache.kafka:kafka_2.13:$kafkaVersion" 30 | testImplementation "org.apache.kafka:kafka_2.13:$kafkaVersion:test" 31 | 32 | testImplementation "org.apache.kafka:connect-runtime:$kafkaVersion" 33 | testImplementation "org.apache.kafka:connect-runtime:$kafkaVersion:test" 34 | testImplementation "org.apache.kafka:kafka-clients:$kafkaVersion:test" 35 | 36 | implementation "org.apache.hadoop:hadoop-minikdc:3.3.2" 37 | implementation 'org.apache.httpcomponents:httpclient:4.5.13' 38 | implementation 'org.slf4j:slf4j-api:1.7.36' 39 | implementation 'com.google.code.gson:gson:2.9.0' 40 | implementation 'org.awaitility:awaitility:4.2.0' 41 | 42 | testImplementation 'org.junit.jupiter:junit-jupiter:5.8.2' 43 | testImplementation 'org.assertj:assertj-core:3.22.0' 44 | testImplementation 'com.github.tomakehurst:wiremock-jre8:2.33.2' 45 | testImplementation 'org.testcontainers:elasticsearch:1.16.2' 46 | 47 | testImplementation 'org.mockito:mockito-core:4.6.0' 48 | testImplementation 'org.mockito:mockito-junit-jupiter:4.6.0' 49 | } 50 | 51 | tasks.named('test') { 52 | useJUnitPlatform() 53 | } 54 | -------------------------------------------------------------------------------- /lib/src/main/java/com/dmathieu/kafka/opensearch/ConfigCallbackHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch; 17 | 18 | import com.sun.security.auth.module.Krb5LoginModule; 19 | import java.security.AccessController; 20 | import java.security.PrivilegedActionException; 21 | import java.security.PrivilegedExceptionAction; 22 | import java.security.KeyManagementException; 23 | import java.util.Collections; 24 | import java.util.HashMap; 25 | import java.util.HashSet; 26 | import java.util.Map; 27 | import javax.net.ssl.HostnameVerifier; 28 | import javax.net.ssl.SSLContext; 29 | import javax.net.ssl.TrustManager; 30 | import javax.security.auth.Subject; 31 | import javax.security.auth.kerberos.KerberosPrincipal; 32 | import javax.security.auth.login.AppConfigurationEntry; 33 | import javax.security.auth.login.Configuration; 34 | import javax.security.auth.login.LoginContext; 35 | import org.apache.http.HttpHost; 36 | import org.apache.http.auth.AuthSchemeProvider; 37 | import org.apache.http.auth.AuthScope; 38 | import org.apache.http.auth.KerberosCredentials; 39 | import org.apache.http.auth.UsernamePasswordCredentials; 40 | import org.apache.http.client.CredentialsProvider; 41 | import org.apache.http.client.config.AuthSchemes; 42 | import org.apache.http.client.config.RequestConfig; 43 | import org.apache.http.config.Lookup; 44 | import org.apache.http.config.Registry; 45 | import org.apache.http.config.RegistryBuilder; 46 | import org.apache.http.conn.ssl.NoopHostnameVerifier; 47 | import org.apache.http.conn.ssl.SSLConnectionSocketFactory; 48 | import org.apache.http.impl.auth.SPNegoSchemeFactory; 49 | import org.apache.http.impl.client.BasicCredentialsProvider; 50 | import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; 51 | import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager; 52 | import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor; 53 | import org.apache.http.impl.nio.reactor.IOReactorConfig; 54 | import org.apache.http.nio.conn.NoopIOSessionStrategy; 55 | import org.apache.http.nio.conn.SchemeIOSessionStrategy; 56 | import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; 57 | import org.apache.http.nio.reactor.ConnectingIOReactor; 58 | import org.apache.http.nio.reactor.IOReactorException; 59 | import org.apache.kafka.common.network.Mode; 60 | import org.apache.kafka.common.security.ssl.SslFactory; 61 | import org.apache.kafka.connect.errors.ConnectException; 62 | import org.elasticsearch.client.RestClientBuilder.HttpClientConfigCallback; 63 | import org.ietf.jgss.GSSCredential; 64 | import org.ietf.jgss.GSSException; 65 | import org.ietf.jgss.GSSManager; 66 | import org.ietf.jgss.Oid; 67 | import org.slf4j.Logger; 68 | import org.slf4j.LoggerFactory; 69 | 70 | import com.dmathieu.kafka.opensearch.UnsafeX509ExtendedTrustManager; 71 | 72 | public class ConfigCallbackHandler implements HttpClientConfigCallback { 73 | 74 | private static final Logger log = LoggerFactory.getLogger(ConfigCallbackHandler.class); 75 | 76 | private static final Oid SPNEGO_OID = spnegoOid(); 77 | 78 | private final OpenSearchSinkConnectorConfig config; 79 | 80 | public ConfigCallbackHandler(OpenSearchSinkConnectorConfig config) { 81 | this.config = config; 82 | } 83 | 84 | /** 85 | * Customizes the client according to the configurations and starts the connection reaping thread. 86 | * 87 | * @param builder the HttpAsyncClientBuilder 88 | * @return the builder 89 | */ 90 | @Override 91 | public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder builder) { 92 | RequestConfig requestConfig = RequestConfig.custom() 93 | .setContentCompressionEnabled(config.compression()) 94 | .setConnectTimeout(config.connectionTimeoutMs()) 95 | .setConnectionRequestTimeout(config.readTimeoutMs()) 96 | .setSocketTimeout(config.readTimeoutMs()) 97 | .build(); 98 | 99 | builder.setConnectionManager(createConnectionManager()) 100 | .setDefaultRequestConfig(requestConfig); 101 | 102 | configureAuthentication(builder); 103 | 104 | if (config.isKerberosEnabled()) { 105 | configureKerberos(builder); 106 | } 107 | configureSslContext(builder); 108 | 109 | if (config.isKerberosEnabled()) { 110 | log.info("Using Kerberos connection to {}.", config.connectionUrls()); 111 | } else { 112 | log.info("Using SSL connection to {}.", config.connectionUrls()); 113 | } 114 | 115 | return builder; 116 | } 117 | 118 | /** 119 | * Configures HTTP authentication and proxy authentication according to the client configuration. 120 | * 121 | * @param builder the HttpAsyncClientBuilder 122 | */ 123 | private void configureAuthentication(HttpAsyncClientBuilder builder) { 124 | CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); 125 | if (config.isAuthenticatedConnection()) { 126 | config.connectionUrls().forEach(url -> credentialsProvider.setCredentials( 127 | new AuthScope(HttpHost.create(url)), 128 | new UsernamePasswordCredentials(config.username(), config.password().value()) 129 | ) 130 | ); 131 | builder.setDefaultCredentialsProvider(credentialsProvider); 132 | } 133 | 134 | if (config.isBasicProxyConfigured()) { 135 | HttpHost proxy = new HttpHost(config.proxyHost(), config.proxyPort()); 136 | builder.setProxy(proxy); 137 | 138 | if (config.isProxyWithAuthenticationConfigured()) { 139 | credentialsProvider.setCredentials( 140 | new AuthScope(proxy), 141 | new UsernamePasswordCredentials(config.proxyUsername(), config.proxyPassword().value()) 142 | ); 143 | } 144 | 145 | builder.setDefaultCredentialsProvider(credentialsProvider); 146 | } 147 | } 148 | 149 | /** 150 | * Creates a connection manager for the client. 151 | * 152 | * @return the connection manager 153 | */ 154 | private PoolingNHttpClientConnectionManager createConnectionManager() { 155 | try { 156 | PoolingNHttpClientConnectionManager cm; 157 | IOReactorConfig ioReactorConfig = IOReactorConfig.custom() 158 | .setConnectTimeout(config.connectionTimeoutMs()) 159 | .setSoTimeout(config.readTimeoutMs()) 160 | .build(); 161 | ConnectingIOReactor ioReactor = new DefaultConnectingIOReactor(ioReactorConfig); 162 | 163 | SSLContext sslContext = sslContext(); 164 | HostnameVerifier hostnameVerifier = SSLConnectionSocketFactory.getDefaultHostnameVerifier(); 165 | if (config.shouldDisableHostnameVerification()) { 166 | hostnameVerifier = new NoopHostnameVerifier(); 167 | 168 | try { 169 | sslContext.init(null, new TrustManager[]{ UnsafeX509ExtendedTrustManager.getInstance() }, null); 170 | } catch (KeyManagementException e) { 171 | log.info("Failed setting unsafe trust manager"); 172 | } 173 | } 174 | 175 | Registry reg = RegistryBuilder.create() 176 | .register("http", NoopIOSessionStrategy.INSTANCE) 177 | .register("https", new SSLIOSessionStrategy(sslContext, hostnameVerifier)) 178 | .build(); 179 | 180 | cm = new PoolingNHttpClientConnectionManager(ioReactor, reg); 181 | 182 | // Allowing up to two http connections per processing thread to a given host 183 | int maxPerRoute = Math.max(10, config.maxInFlightRequests() * 2); 184 | cm.setDefaultMaxPerRoute(maxPerRoute); 185 | // And for the global limit, with multiply the per-host limit 186 | // by the number of potential different ES hosts 187 | cm.setMaxTotal(maxPerRoute * config.connectionUrls().size()); 188 | 189 | log.debug("Connection pool config: maxPerRoute: {}, maxTotal {}", 190 | cm.getDefaultMaxPerRoute(), 191 | cm.getMaxTotal()); 192 | 193 | return cm; 194 | } catch (IOReactorException e) { 195 | throw new ConnectException("Unable to open OpenSearchClient.", e); 196 | } 197 | } 198 | 199 | /** 200 | * Configures the client to use Kerberos authentication. Overrides any proxy or basic auth 201 | * credentials. 202 | * 203 | * @param builder the HttpAsyncClientBuilder to configure 204 | * @return the configured builder 205 | */ 206 | private HttpAsyncClientBuilder configureKerberos(HttpAsyncClientBuilder builder) { 207 | GSSManager gssManager = GSSManager.getInstance(); 208 | Lookup authSchemeRegistry = 209 | RegistryBuilder.create() 210 | .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory()) 211 | .build(); 212 | builder.setDefaultAuthSchemeRegistry(authSchemeRegistry); 213 | 214 | try { 215 | LoginContext loginContext = loginContext(); 216 | GSSCredential credential = Subject.doAs( 217 | loginContext.getSubject(), 218 | (PrivilegedExceptionAction) () -> gssManager.createCredential( 219 | null, 220 | GSSCredential.DEFAULT_LIFETIME, 221 | SPNEGO_OID, 222 | GSSCredential.INITIATE_ONLY 223 | ) 224 | ); 225 | CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); 226 | credentialsProvider.setCredentials( 227 | new AuthScope( 228 | AuthScope.ANY_HOST, 229 | AuthScope.ANY_PORT, 230 | AuthScope.ANY_REALM, 231 | AuthSchemes.SPNEGO 232 | ), 233 | new KerberosCredentials(credential) 234 | ); 235 | builder.setDefaultCredentialsProvider(credentialsProvider); 236 | } catch (PrivilegedActionException e) { 237 | throw new ConnectException(e); 238 | } 239 | 240 | return builder; 241 | } 242 | 243 | /** 244 | * Configures the client to use SSL if configured. 245 | * 246 | * @param builder the HttpAsyncClientBuilder 247 | */ 248 | private void configureSslContext(HttpAsyncClientBuilder builder) { 249 | SSLContext sslContext = sslContext(); 250 | 251 | HostnameVerifier hostnameVerifier = SSLConnectionSocketFactory.getDefaultHostnameVerifier(); 252 | if (config.shouldDisableHostnameVerification()) { 253 | hostnameVerifier = new NoopHostnameVerifier(); 254 | 255 | try { 256 | sslContext.init(null, new TrustManager[]{ UnsafeX509ExtendedTrustManager.getInstance() }, null); 257 | } catch (KeyManagementException e) { 258 | log.info("Failed setting unsafe trust manager"); 259 | } 260 | } 261 | 262 | builder.setSSLContext(sslContext); 263 | builder.setSSLHostnameVerifier(hostnameVerifier); 264 | builder.setSSLStrategy(new SSLIOSessionStrategy(sslContext, hostnameVerifier)); 265 | } 266 | 267 | /** 268 | * Gets the SslContext for the client. 269 | */ 270 | private SSLContext sslContext() { 271 | SslFactory sslFactory = new SslFactory(Mode.CLIENT, null, false); 272 | sslFactory.configure(config.sslConfigs()); 273 | 274 | try { 275 | // try AK <= 2.2 first 276 | log.debug("Trying AK 2.2 SslFactory methods."); 277 | return (SSLContext) SslFactory.class.getDeclaredMethod("sslContext").invoke(sslFactory); 278 | } catch (Exception e) { 279 | // must be running AK 2.3+ 280 | log.debug("Could not find AK 2.2 SslFactory methods. Trying AK 2.3+ methods for SslFactory."); 281 | 282 | Object sslEngine; 283 | try { 284 | // try AK <= 2.6 second 285 | sslEngine = SslFactory.class.getDeclaredMethod("sslEngineBuilder").invoke(sslFactory); 286 | log.debug("Using AK 2.2-2.5 SslFactory methods."); 287 | } catch (Exception ex) { 288 | // must be running AK 2.6+ 289 | log.debug( 290 | "Could not find AK 2.3-2.5 SslFactory methods. Trying AK 2.6+ methods for SslFactory." 291 | ); 292 | try { 293 | sslEngine = SslFactory.class.getDeclaredMethod("sslEngineFactory").invoke(sslFactory); 294 | log.debug("Using AK 2.6+ SslFactory methods."); 295 | } catch (Exception exc) { 296 | throw new ConnectException("Failed to find methods for SslFactory.", exc); 297 | } 298 | } 299 | 300 | try { 301 | return (SSLContext) sslEngine 302 | .getClass() 303 | .getDeclaredMethod("sslContext") 304 | .invoke(sslEngine); 305 | } catch (Exception ex) { 306 | throw new ConnectException("Could not create SSLContext.", ex); 307 | } 308 | } 309 | } 310 | 311 | /** 312 | * Logs in and returns a login context for the given kerberos user principle. 313 | * 314 | * @return the login context 315 | * @throws PrivilegedActionException if the login failed 316 | */ 317 | private LoginContext loginContext() throws PrivilegedActionException { 318 | Configuration conf = new Configuration() { 319 | @Override 320 | public AppConfigurationEntry[] getAppConfigurationEntry(String name) { 321 | return new AppConfigurationEntry[] { 322 | new AppConfigurationEntry( 323 | Krb5LoginModule.class.getName(), 324 | AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, 325 | kerberosConfigs() 326 | ) 327 | }; 328 | } 329 | }; 330 | 331 | return AccessController.doPrivileged( 332 | (PrivilegedExceptionAction) () -> { 333 | Subject subject = new Subject( 334 | false, 335 | Collections.singleton(new KerberosPrincipal(config.kerberosUserPrincipal())), 336 | new HashSet<>(), 337 | new HashSet<>() 338 | ); 339 | LoginContext loginContext = new LoginContext( 340 | "OpenSearchSinkConnector", 341 | subject, 342 | null, 343 | conf 344 | ); 345 | loginContext.login(); 346 | return loginContext; 347 | } 348 | ); 349 | } 350 | 351 | /** 352 | * Creates the Kerberos configurations. 353 | * 354 | * @return map of kerberos configs 355 | */ 356 | private Map kerberosConfigs() { 357 | Map configs = new HashMap<>(); 358 | configs.put("useTicketCache", "true"); 359 | configs.put("renewTGT", "true"); 360 | configs.put("useKeyTab", "true"); 361 | configs.put("keyTab", config.keytabPath()); 362 | //Krb5 in GSS API needs to be refreshed so it does not throw the error 363 | //Specified version of key is not available 364 | configs.put("refreshKrb5Config", "true"); 365 | configs.put("principal", config.kerberosUserPrincipal()); 366 | configs.put("storeKey", "false"); 367 | configs.put("doNotPrompt", "true"); 368 | return configs; 369 | } 370 | 371 | private static Oid spnegoOid() { 372 | try { 373 | return new Oid("1.3.6.1.5.5.2"); 374 | } catch (GSSException gsse) { 375 | throw new ConnectException(gsse); 376 | } 377 | } 378 | } 379 | -------------------------------------------------------------------------------- /lib/src/main/java/com/dmathieu/kafka/opensearch/DataConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch; 17 | 18 | import com.fasterxml.jackson.core.JsonProcessingException; 19 | import com.fasterxml.jackson.databind.JsonNode; 20 | import com.fasterxml.jackson.databind.ObjectMapper; 21 | import com.fasterxml.jackson.databind.node.ObjectNode; 22 | import com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.BehaviorOnNullValues; 23 | import org.apache.kafka.connect.data.ConnectSchema; 24 | import org.apache.kafka.connect.data.Date; 25 | import org.apache.kafka.connect.data.Decimal; 26 | import org.apache.kafka.connect.data.Field; 27 | import org.apache.kafka.connect.data.Schema; 28 | import org.apache.kafka.connect.data.SchemaBuilder; 29 | import org.apache.kafka.connect.data.Struct; 30 | import org.apache.kafka.connect.data.Time; 31 | import org.apache.kafka.connect.data.Timestamp; 32 | import org.apache.kafka.connect.errors.DataException; 33 | import org.apache.kafka.connect.json.JsonConverter; 34 | import org.apache.kafka.connect.sink.SinkRecord; 35 | import org.apache.kafka.connect.storage.Converter; 36 | import org.elasticsearch.action.DocWriteRequest; 37 | import org.elasticsearch.action.DocWriteRequest.OpType; 38 | import org.elasticsearch.action.delete.DeleteRequest; 39 | import org.elasticsearch.action.index.IndexRequest; 40 | import org.elasticsearch.action.update.UpdateRequest; 41 | import org.elasticsearch.common.xcontent.XContentType; 42 | import org.elasticsearch.index.VersionType; 43 | import org.slf4j.Logger; 44 | import org.slf4j.LoggerFactory; 45 | 46 | import java.math.BigDecimal; 47 | import java.nio.charset.StandardCharsets; 48 | import java.util.ArrayList; 49 | import java.util.Collection; 50 | import java.util.Collections; 51 | import java.util.HashMap; 52 | import java.util.List; 53 | import java.util.Map; 54 | 55 | public class DataConverter { 56 | 57 | private static final Logger log = LoggerFactory.getLogger(DataConverter.class); 58 | 59 | private static final Converter JSON_CONVERTER; 60 | protected static final String MAP_KEY = "key"; 61 | protected static final String MAP_VALUE = "value"; 62 | protected static final String TIMESTAMP_FIELD = "@timestamp"; 63 | 64 | private ObjectMapper objectMapper; 65 | 66 | static { 67 | JSON_CONVERTER = new JsonConverter(); 68 | JSON_CONVERTER.configure(Collections.singletonMap("schemas.enable", "false"), false); 69 | } 70 | 71 | private final OpenSearchSinkConnectorConfig config; 72 | 73 | /** 74 | * Create a DataConverter, specifying how map entries with string keys within record 75 | * values should be written to JSON. Compact map entries are written as 76 | * "entryKey": "entryValue", while the non-compact form are written as a nested 77 | * document such as {"key": "entryKey", "value": "entryValue"}. All map entries 78 | * with non-string keys are always written as nested documents. 79 | * 80 | * @param config connector config 81 | */ 82 | public DataConverter(OpenSearchSinkConnectorConfig config) { 83 | this.config = config; 84 | this.objectMapper = new ObjectMapper(); 85 | } 86 | 87 | private String convertKey(Schema keySchema, Object key) { 88 | if (key == null) { 89 | throw new DataException("Key is used as document id and can not be null."); 90 | } 91 | 92 | final Schema.Type schemaType; 93 | if (keySchema == null) { 94 | schemaType = ConnectSchema.schemaType(key.getClass()); 95 | if (schemaType == null) { 96 | throw new DataException( 97 | "Java class " + key.getClass() + " does not have corresponding schema type." 98 | ); 99 | } 100 | } else { 101 | schemaType = keySchema.type(); 102 | } 103 | 104 | switch (schemaType) { 105 | case INT8: 106 | case INT16: 107 | case INT32: 108 | case INT64: 109 | case STRING: 110 | return String.valueOf(key); 111 | default: 112 | throw new DataException(schemaType.name() + " is not supported as the document id."); 113 | } 114 | } 115 | 116 | public DocWriteRequest convertRecord(SinkRecord record, String index) { 117 | if (record.value() == null) { 118 | switch (config.behaviorOnNullValues()) { 119 | case IGNORE: 120 | log.trace("Ignoring {} with null value.", recordString(record)); 121 | return null; 122 | case DELETE: 123 | if (record.key() == null) { 124 | // Since the record key is used as the ID of the index to delete and we don't have a key 125 | // for this record, we can't delete anything anyways, so we ignore the record. 126 | // We can also disregard the value of the ignoreKey parameter, since even if it's true 127 | // the resulting index we'd try to delete would be based solely off topic/partition/ 128 | // offset information for the SinkRecord. Since that information is guaranteed to be 129 | // unique per message, we can be confident that there wouldn't be any corresponding 130 | // index present in ES to delete anyways. 131 | log.trace( 132 | "Ignoring {} with null key, since the record key is used as the ID of the index", 133 | recordString(record) 134 | ); 135 | return null; 136 | } 137 | // Will proceed as normal, ultimately creating a DeleteRequest 138 | log.trace("Deleting {} from OpenSearch", recordString(record)); 139 | break; 140 | case FAIL: 141 | default: 142 | throw new DataException( 143 | String.format( 144 | "%s with key of %s and null value encountered (to ignore future records like" 145 | + " this change the configuration property '%s' from '%s' to '%s')", 146 | recordString(record), 147 | record.key(), 148 | OpenSearchSinkConnectorConfig.BEHAVIOR_ON_NULL_VALUES_CONFIG, 149 | BehaviorOnNullValues.FAIL, 150 | BehaviorOnNullValues.IGNORE 151 | ) 152 | ); 153 | } 154 | } 155 | 156 | final String id = config.shouldIgnoreKey(record.topic()) 157 | ? String.format("%s+%d+%d", record.topic(), record.kafkaPartition(), record.kafkaOffset()) 158 | : convertKey(record.keySchema(), record.key()); 159 | 160 | // delete 161 | if (record.value() == null) { 162 | return maybeAddExternalVersioning(new DeleteRequest(index).id(id), record); 163 | } 164 | 165 | String payload = getPayload(record); 166 | payload = maybeAddTimestamp(payload, record.timestamp()); 167 | 168 | // index 169 | switch (config.writeMethod()) { 170 | case UPSERT: 171 | return new UpdateRequest(index, id) 172 | .doc(payload, XContentType.JSON) 173 | .upsert(payload, XContentType.JSON) 174 | .retryOnConflict(Math.min(config.maxInFlightRequests(), 5)); 175 | case INSERT: 176 | OpType opType = config.isDataStream() ? OpType.CREATE : OpType.INDEX; 177 | return maybeAddExternalVersioning( 178 | new IndexRequest(index).id(id).source(payload, XContentType.JSON).opType(opType), 179 | record 180 | ); 181 | default: 182 | return null; // shouldn't happen 183 | } 184 | } 185 | 186 | private String getPayload(SinkRecord record) { 187 | if (record.value() == null) { 188 | return null; 189 | } 190 | 191 | Schema schema = config.shouldIgnoreSchema(record.topic()) 192 | ? record.valueSchema() 193 | : preProcessSchema(record.valueSchema()); 194 | Object value = config.shouldIgnoreSchema(record.topic()) 195 | ? record.value() 196 | : preProcessValue(record.value(), record.valueSchema(), schema); 197 | 198 | byte[] rawJsonPayload = JSON_CONVERTER.fromConnectData(record.topic(), schema, value); 199 | return new String(rawJsonPayload, StandardCharsets.UTF_8); 200 | } 201 | 202 | private String maybeAddTimestamp(String payload, Long timestamp) { 203 | if (!config.isDataStream()) { 204 | return payload; 205 | } 206 | try { 207 | JsonNode jsonNode = objectMapper.readTree(payload); 208 | if (!config.dataStreamTimestampField().isEmpty()) { 209 | for (String timestampField : config.dataStreamTimestampField()) { 210 | if (jsonNode.has(timestampField)) { 211 | ((ObjectNode) jsonNode).put(TIMESTAMP_FIELD, jsonNode.get(timestampField).asText()); 212 | return objectMapper.writeValueAsString(jsonNode); 213 | } 214 | } 215 | } else { 216 | ((ObjectNode) jsonNode).put(TIMESTAMP_FIELD, timestamp); 217 | return objectMapper.writeValueAsString(jsonNode); 218 | } 219 | } catch (JsonProcessingException e) { 220 | // Should not happen if the payload was retrieved correctly. 221 | } 222 | return payload; 223 | } 224 | 225 | private DocWriteRequest maybeAddExternalVersioning( 226 | DocWriteRequest request, 227 | SinkRecord record 228 | ) { 229 | if (!config.isDataStream() && !config.shouldIgnoreKey(record.topic())) { 230 | request.versionType(VersionType.EXTERNAL); 231 | request.version(record.kafkaOffset()); 232 | } 233 | 234 | return request; 235 | } 236 | 237 | // We need to pre process the Kafka Connect schema before converting to JSON as OpenSearch 238 | // expects a different JSON format from the current JSON converter provides. Rather than 239 | // completely rewrite a converter for OpenSearch, we will refactor the JSON converter to 240 | // support customized translation. The pre process is no longer needed once we have the JSON 241 | // converter refactored. 242 | // visible for testing 243 | Schema preProcessSchema(Schema schema) { 244 | if (schema == null) { 245 | return null; 246 | } 247 | // Handle logical types 248 | String schemaName = schema.name(); 249 | if (schemaName != null) { 250 | switch (schemaName) { 251 | case Decimal.LOGICAL_NAME: 252 | return copySchemaBasics(schema, SchemaBuilder.float64()).build(); 253 | case Date.LOGICAL_NAME: 254 | case Time.LOGICAL_NAME: 255 | case Timestamp.LOGICAL_NAME: 256 | return schema; 257 | default: 258 | // User type or unknown logical type 259 | break; 260 | } 261 | } 262 | 263 | Schema.Type schemaType = schema.type(); 264 | switch (schemaType) { 265 | case ARRAY: 266 | return preProcessArraySchema(schema); 267 | case MAP: 268 | return preProcessMapSchema(schema); 269 | case STRUCT: 270 | return preProcessStructSchema(schema); 271 | default: 272 | return schema; 273 | } 274 | } 275 | 276 | private Schema preProcessArraySchema(Schema schema) { 277 | Schema valSchema = preProcessSchema(schema.valueSchema()); 278 | return copySchemaBasics(schema, SchemaBuilder.array(valSchema)).build(); 279 | } 280 | 281 | private Schema preProcessMapSchema(Schema schema) { 282 | Schema keySchema = schema.keySchema(); 283 | Schema valueSchema = schema.valueSchema(); 284 | String keyName = keySchema.name() == null ? keySchema.type().name() : keySchema.name(); 285 | String valueName = valueSchema.name() == null ? valueSchema.type().name() : valueSchema.name(); 286 | Schema preprocessedKeySchema = preProcessSchema(keySchema); 287 | Schema preprocessedValueSchema = preProcessSchema(valueSchema); 288 | if (config.useCompactMapEntries() && keySchema.type() == Schema.Type.STRING) { 289 | SchemaBuilder result = SchemaBuilder.map(preprocessedKeySchema, preprocessedValueSchema); 290 | return copySchemaBasics(schema, result).build(); 291 | } 292 | Schema elementSchema = SchemaBuilder.struct().name(keyName + "-" + valueName) 293 | .field(MAP_KEY, preprocessedKeySchema) 294 | .field(MAP_VALUE, preprocessedValueSchema) 295 | .build(); 296 | return copySchemaBasics(schema, SchemaBuilder.array(elementSchema)).build(); 297 | } 298 | 299 | private Schema preProcessStructSchema(Schema schema) { 300 | SchemaBuilder builder = copySchemaBasics(schema, SchemaBuilder.struct().name(schema.name())); 301 | for (Field field : schema.fields()) { 302 | builder.field(field.name(), preProcessSchema(field.schema())); 303 | } 304 | return builder.build(); 305 | } 306 | 307 | private SchemaBuilder copySchemaBasics(Schema source, SchemaBuilder target) { 308 | if (source.isOptional()) { 309 | target.optional(); 310 | } 311 | if (source.defaultValue() != null && source.type() != Schema.Type.STRUCT) { 312 | final Object defaultVal = preProcessValue(source.defaultValue(), source, target); 313 | target.defaultValue(defaultVal); 314 | } 315 | return target; 316 | } 317 | 318 | // visible for testing 319 | Object preProcessValue(Object value, Schema schema, Schema newSchema) { 320 | // Handle missing schemas and acceptable null values 321 | if (schema == null) { 322 | return value; 323 | } 324 | 325 | if (value == null) { 326 | return preProcessNullValue(schema); 327 | } 328 | 329 | // Handle logical types 330 | String schemaName = schema.name(); 331 | if (schemaName != null) { 332 | Object result = preProcessLogicalValue(schemaName, value); 333 | if (result != null) { 334 | return result; 335 | } 336 | } 337 | 338 | Schema.Type schemaType = schema.type(); 339 | switch (schemaType) { 340 | case ARRAY: 341 | return preProcessArrayValue(value, schema, newSchema); 342 | case MAP: 343 | return preProcessMapValue(value, schema, newSchema); 344 | case STRUCT: 345 | return preProcessStructValue(value, schema, newSchema); 346 | default: 347 | return value; 348 | } 349 | } 350 | 351 | private Object preProcessNullValue(Schema schema) { 352 | if (schema.defaultValue() != null) { 353 | return schema.defaultValue(); 354 | } 355 | if (schema.isOptional()) { 356 | return null; 357 | } 358 | throw new DataException("null value for field that is required and has no default value"); 359 | } 360 | 361 | // @returns the decoded logical value or null if this isn't a known logical type 362 | private Object preProcessLogicalValue(String schemaName, Object value) { 363 | switch (schemaName) { 364 | case Decimal.LOGICAL_NAME: 365 | return ((BigDecimal) value).doubleValue(); 366 | case Date.LOGICAL_NAME: 367 | case Time.LOGICAL_NAME: 368 | case Timestamp.LOGICAL_NAME: 369 | return value; 370 | default: 371 | // User-defined type or unknown built-in 372 | return null; 373 | } 374 | } 375 | 376 | private Object preProcessArrayValue(Object value, Schema schema, Schema newSchema) { 377 | Collection collection = (Collection) value; 378 | List result = new ArrayList<>(); 379 | for (Object element: collection) { 380 | result.add(preProcessValue(element, schema.valueSchema(), newSchema.valueSchema())); 381 | } 382 | return result; 383 | } 384 | 385 | private Object preProcessMapValue(Object value, Schema schema, Schema newSchema) { 386 | Schema keySchema = schema.keySchema(); 387 | Schema valueSchema = schema.valueSchema(); 388 | Schema newValueSchema = newSchema.valueSchema(); 389 | Map map = (Map) value; 390 | if (config.useCompactMapEntries() && keySchema.type() == Schema.Type.STRING) { 391 | Map processedMap = new HashMap<>(); 392 | for (Map.Entry entry: map.entrySet()) { 393 | processedMap.put( 394 | preProcessValue(entry.getKey(), keySchema, newSchema.keySchema()), 395 | preProcessValue(entry.getValue(), valueSchema, newValueSchema) 396 | ); 397 | } 398 | return processedMap; 399 | } 400 | List mapStructs = new ArrayList<>(); 401 | for (Map.Entry entry: map.entrySet()) { 402 | Struct mapStruct = new Struct(newValueSchema); 403 | Schema mapKeySchema = newValueSchema.field(MAP_KEY).schema(); 404 | Schema mapValueSchema = newValueSchema.field(MAP_VALUE).schema(); 405 | mapStruct.put(MAP_KEY, preProcessValue(entry.getKey(), keySchema, mapKeySchema)); 406 | mapStruct.put(MAP_VALUE, preProcessValue(entry.getValue(), valueSchema, mapValueSchema)); 407 | mapStructs.add(mapStruct); 408 | } 409 | return mapStructs; 410 | } 411 | 412 | private Object preProcessStructValue(Object value, Schema schema, Schema newSchema) { 413 | Struct struct = (Struct) value; 414 | Struct newStruct = new Struct(newSchema); 415 | for (Field field : schema.fields()) { 416 | Schema newFieldSchema = newSchema.field(field.name()).schema(); 417 | Object converted = preProcessValue(struct.get(field), field.schema(), newFieldSchema); 418 | newStruct.put(field.name(), converted); 419 | } 420 | return newStruct; 421 | } 422 | 423 | private static String recordString(SinkRecord record) { 424 | return String.format( 425 | "record from topic=%s partition=%s offset=%s", 426 | record.topic(), 427 | record.kafkaPartition(), 428 | record.kafkaOffset() 429 | ); 430 | } 431 | } 432 | -------------------------------------------------------------------------------- /lib/src/main/java/com/dmathieu/kafka/opensearch/EnumRecommender.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch; 17 | 18 | import java.util.ArrayList; 19 | import java.util.Collections; 20 | import java.util.LinkedHashSet; 21 | import java.util.List; 22 | import java.util.Map; 23 | import java.util.Set; 24 | 25 | import org.apache.kafka.common.config.ConfigDef; 26 | import org.apache.kafka.common.config.ConfigException; 27 | 28 | class EnumRecommender> implements ConfigDef.Validator, ConfigDef.Recommender { 29 | 30 | private final Set validValues; 31 | private final Class enumClass; 32 | 33 | public EnumRecommender(Class enumClass) { 34 | this.enumClass = enumClass; 35 | Set validEnums = new LinkedHashSet<>(); 36 | for (Object o : enumClass.getEnumConstants()) { 37 | String key = o.toString().toLowerCase(); 38 | validEnums.add(key); 39 | } 40 | 41 | this.validValues = Collections.unmodifiableSet(validEnums); 42 | } 43 | 44 | @Override 45 | public void ensureValid(String key, Object value) { 46 | if (value == null) { 47 | return; 48 | } 49 | String enumValue = value.toString().toLowerCase(); 50 | if (value != null && !validValues.contains(enumValue)) { 51 | throw new ConfigException(key, value, "Value must be one of: " + this); 52 | } 53 | } 54 | 55 | @Override 56 | public String toString() { 57 | return validValues.toString(); 58 | } 59 | 60 | @Override 61 | public List validValues(String name, Map connectorConfigs) { 62 | return Collections.unmodifiableList(new ArrayList<>(validValues)); 63 | } 64 | 65 | @Override 66 | public boolean visible(String name, Map connectorConfigs) { 67 | return true; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /lib/src/main/java/com/dmathieu/kafka/opensearch/Mapping.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch; 17 | 18 | import org.apache.kafka.connect.data.Date; 19 | import org.apache.kafka.connect.data.Decimal; 20 | import org.apache.kafka.connect.data.Field; 21 | import org.apache.kafka.connect.data.Schema; 22 | import org.apache.kafka.connect.data.Time; 23 | import org.apache.kafka.connect.data.Timestamp; 24 | import org.apache.kafka.connect.errors.ConnectException; 25 | import org.apache.kafka.connect.errors.DataException; 26 | import org.elasticsearch.common.xcontent.XContentBuilder; 27 | import org.elasticsearch.common.xcontent.XContentFactory; 28 | 29 | import java.io.IOException; 30 | 31 | public class Mapping { 32 | 33 | // OpenSearch types 34 | public static final String BOOLEAN_TYPE = "boolean"; 35 | public static final String BYTE_TYPE = "byte"; 36 | public static final String BINARY_TYPE = "binary"; 37 | public static final String SHORT_TYPE = "short"; 38 | public static final String INTEGER_TYPE = "integer"; 39 | public static final String LONG_TYPE = "long"; 40 | public static final String FLOAT_TYPE = "float"; 41 | public static final String DOUBLE_TYPE = "double"; 42 | public static final String STRING_TYPE = "string"; 43 | public static final String TEXT_TYPE = "text"; 44 | public static final String KEYWORD_TYPE = "keyword"; 45 | public static final String DATE_TYPE = "date"; 46 | 47 | // OpenSearch mapping fields 48 | private static final String DEFAULT_VALUE_FIELD = "null_value"; 49 | private static final String FIELDS_FIELD = "fields"; 50 | private static final String IGNORE_ABOVE_FIELD = "ignore_above"; 51 | public static final String KEY_FIELD = "key"; 52 | private static final String KEYWORD_FIELD = "keyword"; 53 | private static final String PROPERTIES_FIELD = "properties"; 54 | private static final String TYPE_FIELD = "type"; 55 | public static final String VALUE_FIELD = "value"; 56 | 57 | /** 58 | * Build mapping from the provided schema. 59 | * 60 | * @param schema The schema used to build the mapping. 61 | * @return the schema as a JSON mapping 62 | */ 63 | public static XContentBuilder buildMapping(Schema schema) { 64 | try { 65 | XContentBuilder builder = XContentFactory.jsonBuilder(); 66 | builder.startObject(); 67 | { 68 | buildMapping(schema, builder); 69 | } 70 | builder.endObject(); 71 | return builder; 72 | } catch (IOException e) { 73 | throw new ConnectException("Failed to build mapping for schema " + schema, e); 74 | } 75 | } 76 | 77 | private static XContentBuilder buildMapping(Schema schema, XContentBuilder builder) 78 | throws IOException { 79 | 80 | if (schema == null) { 81 | throw new DataException("Cannot infer mapping without schema."); 82 | } 83 | 84 | // Handle logical types 85 | XContentBuilder logicalConversion = inferLogicalMapping(builder, schema); 86 | if (logicalConversion != null) { 87 | return logicalConversion; 88 | } 89 | 90 | Schema.Type schemaType = schema.type(); 91 | switch (schema.type()) { 92 | case ARRAY: 93 | return buildMapping(schema.valueSchema(), builder); 94 | 95 | case MAP: 96 | return buildMap(schema, builder); 97 | 98 | case STRUCT: 99 | return buildStruct(schema, builder); 100 | 101 | default: 102 | return inferPrimitive(builder, getOpenSearchType(schemaType), schema.defaultValue()); 103 | } 104 | } 105 | 106 | private static void addTextMapping(XContentBuilder builder) throws IOException { 107 | // Add additional mapping for indexing, per https://www.elastic.co/blog/strings-are-dead-long-live-strings 108 | builder.startObject(FIELDS_FIELD); 109 | { 110 | builder.startObject(KEYWORD_FIELD); 111 | { 112 | builder.field(TYPE_FIELD, KEYWORD_TYPE); 113 | builder.field(IGNORE_ABOVE_FIELD, 256); 114 | } 115 | builder.endObject(); 116 | } 117 | builder.endObject(); 118 | } 119 | 120 | private static XContentBuilder buildMap(Schema schema, XContentBuilder builder) 121 | throws IOException { 122 | 123 | builder.startObject(PROPERTIES_FIELD); 124 | { 125 | builder.startObject(KEY_FIELD); 126 | { 127 | buildMapping(schema.keySchema(), builder); 128 | } 129 | builder.endObject(); 130 | builder.startObject(VALUE_FIELD); 131 | { 132 | buildMapping(schema.valueSchema(), builder); 133 | } 134 | builder.endObject(); 135 | } 136 | return builder.endObject(); 137 | } 138 | 139 | private static XContentBuilder buildStruct(Schema schema, XContentBuilder builder) 140 | throws IOException { 141 | 142 | builder.startObject(PROPERTIES_FIELD); 143 | { 144 | for (Field field : schema.fields()) { 145 | builder.startObject(field.name()); 146 | { 147 | buildMapping(field.schema(), builder); 148 | } 149 | builder.endObject(); 150 | } 151 | } 152 | return builder.endObject(); 153 | } 154 | 155 | private static XContentBuilder inferPrimitive( 156 | XContentBuilder builder, 157 | String type, 158 | Object defaultValue 159 | ) throws IOException { 160 | 161 | if (type == null) { 162 | throw new DataException(String.format("Invalid primitive type %s.", type)); 163 | } 164 | 165 | builder.field(TYPE_FIELD, type); 166 | if (type.equals(TEXT_TYPE)) { 167 | addTextMapping(builder); 168 | } 169 | 170 | if (defaultValue == null) { 171 | return builder; 172 | } 173 | 174 | switch (type) { 175 | case BYTE_TYPE: 176 | return builder.field(DEFAULT_VALUE_FIELD, (byte) defaultValue); 177 | case SHORT_TYPE: 178 | return builder.field(DEFAULT_VALUE_FIELD, (short) defaultValue); 179 | case INTEGER_TYPE: 180 | return builder.field(DEFAULT_VALUE_FIELD, (int) defaultValue); 181 | case LONG_TYPE: 182 | return builder.field(DEFAULT_VALUE_FIELD, (long) defaultValue); 183 | case FLOAT_TYPE: 184 | return builder.field(DEFAULT_VALUE_FIELD, (float) defaultValue); 185 | case DOUBLE_TYPE: 186 | return builder.field(DEFAULT_VALUE_FIELD, (double) defaultValue); 187 | case BOOLEAN_TYPE: 188 | return builder.field(DEFAULT_VALUE_FIELD, (boolean) defaultValue); 189 | case DATE_TYPE: 190 | return builder.field(DEFAULT_VALUE_FIELD, ((java.util.Date) defaultValue).getTime()); 191 | /* 192 | * IGNORE default values for text and binary types as this is not supported by ES side. 193 | * see https://www.elastic.co/guide/en/elasticsearch/reference/current/text.html and 194 | * https://www.elastic.co/guide/en/elasticsearch/reference/current/binary.html for details. 195 | */ 196 | case STRING_TYPE: 197 | case TEXT_TYPE: 198 | case BINARY_TYPE: 199 | return builder; 200 | default: 201 | throw new DataException("Invalid primitive type " + type + "."); 202 | } 203 | } 204 | 205 | private static XContentBuilder inferLogicalMapping(XContentBuilder builder, Schema schema) 206 | throws IOException { 207 | 208 | if (schema.name() == null) { 209 | return null; 210 | } 211 | 212 | switch (schema.name()) { 213 | case Date.LOGICAL_NAME: 214 | case Time.LOGICAL_NAME: 215 | case Timestamp.LOGICAL_NAME: 216 | return inferPrimitive(builder, DATE_TYPE, schema.defaultValue()); 217 | case Decimal.LOGICAL_NAME: 218 | return inferPrimitive(builder, DOUBLE_TYPE, schema.defaultValue()); 219 | default: 220 | // User-defined type or unknown built-in 221 | return null; 222 | } 223 | } 224 | 225 | // visible for testing 226 | protected static String getOpenSearchType(Schema.Type schemaType) { 227 | switch (schemaType) { 228 | case BOOLEAN: 229 | return BOOLEAN_TYPE; 230 | case INT8: 231 | return BYTE_TYPE; 232 | case INT16: 233 | return SHORT_TYPE; 234 | case INT32: 235 | return INTEGER_TYPE; 236 | case INT64: 237 | return LONG_TYPE; 238 | case FLOAT32: 239 | return FLOAT_TYPE; 240 | case FLOAT64: 241 | return DOUBLE_TYPE; 242 | case STRING: 243 | return TEXT_TYPE; 244 | case BYTES: 245 | return BINARY_TYPE; 246 | default: 247 | return null; 248 | } 249 | } 250 | } 251 | -------------------------------------------------------------------------------- /lib/src/main/java/com/dmathieu/kafka/opensearch/OffsetTracker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch; 17 | 18 | import org.apache.kafka.clients.consumer.OffsetAndMetadata; 19 | import org.apache.kafka.common.TopicPartition; 20 | import org.apache.kafka.connect.sink.SinkRecord; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | import java.util.Collection; 25 | import java.util.HashMap; 26 | import java.util.Iterator; 27 | import java.util.LinkedHashMap; 28 | import java.util.Map; 29 | import java.util.concurrent.atomic.AtomicLong; 30 | 31 | import static java.util.stream.Collectors.toMap; 32 | 33 | /** 34 | * Tracks processed records to calculate safe offsets to commit. 35 | * 36 | *

Since ElasticsearchClient can potentially process multiple batches asynchronously for the same 37 | * partition, if we don't want to wait for all in-flight batches at the end of the put call 38 | * (or flush/preCommit) we need to keep track of what's the highest offset that is safe to commit. 39 | * For now, we do that at the individual record level because batching is handled by BulkProcessor, 40 | * and we don't have control over grouping/ordering. 41 | */ 42 | class OffsetTracker { 43 | 44 | private static final Logger log = LoggerFactory.getLogger(OffsetTracker.class); 45 | 46 | private final Map> offsetsByPartition = new HashMap<>(); 47 | private final Map maxOffsetByPartition = new HashMap<>(); 48 | 49 | private final AtomicLong numEntries = new AtomicLong(); 50 | 51 | static class OffsetState { 52 | 53 | private final long offset; 54 | private volatile boolean processed; 55 | 56 | OffsetState(long offset) { 57 | this.offset = offset; 58 | } 59 | 60 | /** 61 | * Marks the offset as processed (ready to report to preCommit) 62 | */ 63 | public void markProcessed() { 64 | processed = true; 65 | } 66 | 67 | public boolean isProcessed() { 68 | return processed; 69 | } 70 | } 71 | 72 | /** 73 | * Partitions are no longer owned, we should release all related resources. 74 | * @param topicPartitions partitions to close 75 | */ 76 | public synchronized void closePartitions(Collection topicPartitions) { 77 | topicPartitions.forEach(tp -> { 78 | Map offsets = offsetsByPartition.remove(tp); 79 | if (offsets != null) { 80 | numEntries.getAndAdd(-offsets.size()); 81 | } 82 | maxOffsetByPartition.remove(tp); 83 | }); 84 | } 85 | 86 | /** 87 | * This method assumes that new records are added in offset order. 88 | * Older records can be re-added, and the same Offset object will be return if its 89 | * offset hasn't been reported yet. 90 | * @param sinkRecord record to add 91 | * @return offset state record that can be used to mark the record as processed 92 | */ 93 | public synchronized OffsetState addPendingRecord(SinkRecord sinkRecord) { 94 | log.trace("Adding pending record"); 95 | TopicPartition tp = new TopicPartition(sinkRecord.topic(), sinkRecord.kafkaPartition()); 96 | Long partitionMax = maxOffsetByPartition.get(tp); 97 | if (partitionMax == null || sinkRecord.kafkaOffset() > partitionMax) { 98 | numEntries.incrementAndGet(); 99 | return offsetsByPartition 100 | // Insertion order needs to be maintained 101 | .computeIfAbsent(tp, key -> new LinkedHashMap<>()) 102 | .computeIfAbsent(sinkRecord.kafkaOffset(), OffsetState::new); 103 | } else { 104 | return new OffsetState(sinkRecord.kafkaOffset()); 105 | } 106 | } 107 | 108 | /** 109 | * @return overall number of entries currently in memory. {@link #updateOffsets()} is the one 110 | * cleaning up entries that are not needed anymore (all the contiguos processed entries 111 | * since the last reported offset) 112 | */ 113 | public long numOffsetStateEntries() { 114 | return numEntries.get(); 115 | } 116 | 117 | /** 118 | * Move offsets to the highest we can. 119 | */ 120 | public synchronized void updateOffsets() { 121 | log.trace("Updating offsets"); 122 | offsetsByPartition.forEach(((topicPartition, offsets) -> { 123 | Long max = maxOffsetByPartition.get(topicPartition); 124 | boolean newMaxFound = false; 125 | Iterator iterator = offsets.values().iterator(); 126 | while (iterator.hasNext()) { 127 | OffsetState offsetState = iterator.next(); 128 | if (offsetState.isProcessed()) { 129 | iterator.remove(); 130 | numEntries.decrementAndGet(); 131 | if (max == null || offsetState.offset > max) { 132 | max = offsetState.offset; 133 | newMaxFound = true; 134 | } 135 | } else { 136 | break; 137 | } 138 | } 139 | if (newMaxFound) { 140 | maxOffsetByPartition.put(topicPartition, max); 141 | } 142 | })); 143 | log.trace("Updated offsets, num entries: {}", numEntries); 144 | } 145 | 146 | /** 147 | * @return offsets to commit 148 | */ 149 | public synchronized Map offsets() { 150 | return maxOffsetByPartition.entrySet().stream() 151 | .collect(toMap( 152 | Map.Entry::getKey, 153 | // The offsets you commit are the offsets of the messages you want to read next 154 | // (not the offsets of the messages you did read last) 155 | e -> new OffsetAndMetadata(e.getValue() + 1))); 156 | } 157 | 158 | } 159 | -------------------------------------------------------------------------------- /lib/src/main/java/com/dmathieu/kafka/opensearch/OpenSearchSinkConnector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch; 17 | 18 | import org.apache.kafka.common.config.Config; 19 | import org.apache.kafka.common.config.ConfigDef; 20 | import org.apache.kafka.common.config.ConfigException; 21 | import org.apache.kafka.connect.connector.Task; 22 | import org.apache.kafka.connect.errors.ConnectException; 23 | import org.apache.kafka.connect.sink.SinkConnector; 24 | 25 | import java.util.ArrayList; 26 | import java.util.HashMap; 27 | import java.util.List; 28 | import java.util.Map; 29 | 30 | public class OpenSearchSinkConnector extends SinkConnector { 31 | 32 | private Map configProperties; 33 | 34 | @Override 35 | public String version() { 36 | return Version.getVersion(); 37 | } 38 | 39 | @Override 40 | public void start(Map props) throws ConnectException { 41 | try { 42 | configProperties = props; 43 | // validation 44 | new OpenSearchSinkConnectorConfig(props); 45 | } catch (ConfigException e) { 46 | throw new ConnectException( 47 | "Couldn't start OpenSearchSinkConnector due to configuration error", 48 | e 49 | ); 50 | } 51 | } 52 | 53 | @Override 54 | public Class taskClass() { 55 | return OpenSearchSinkTask.class; 56 | } 57 | 58 | @Override 59 | public List> taskConfigs(int maxTasks) { 60 | List> taskConfigs = new ArrayList<>(); 61 | Map taskProps = new HashMap<>(); 62 | taskProps.putAll(configProperties); 63 | for (int i = 0; i < maxTasks; i++) { 64 | taskConfigs.add(taskProps); 65 | } 66 | return taskConfigs; 67 | } 68 | 69 | @Override 70 | public void stop() throws ConnectException { } 71 | 72 | @Override 73 | public ConfigDef config() { 74 | return OpenSearchSinkConnectorConfig.CONFIG; 75 | } 76 | 77 | @Override 78 | public Config validate(Map connectorConfigs) { 79 | Validator validator = new Validator(connectorConfigs); 80 | return validator.validate(); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /lib/src/main/java/com/dmathieu/kafka/opensearch/OpenSearchSinkTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch; 17 | 18 | import com.dmathieu.kafka.opensearch.OffsetTracker.OffsetState; 19 | import org.apache.http.HttpHost; 20 | import org.apache.kafka.clients.consumer.OffsetAndMetadata; 21 | import org.apache.kafka.common.TopicPartition; 22 | import org.apache.kafka.connect.errors.ConnectException; 23 | import org.apache.kafka.connect.errors.DataException; 24 | import org.apache.kafka.connect.sink.ErrantRecordReporter; 25 | import org.apache.kafka.connect.sink.SinkRecord; 26 | import org.apache.kafka.connect.sink.SinkTask; 27 | import org.apache.kafka.connect.sink.SinkTaskContext; 28 | import org.elasticsearch.action.DocWriteRequest; 29 | import org.elasticsearch.client.RequestOptions; 30 | import org.elasticsearch.client.RestClient; 31 | import org.elasticsearch.client.RestHighLevelClient; 32 | import org.elasticsearch.client.core.MainResponse; 33 | import org.slf4j.Logger; 34 | import org.slf4j.LoggerFactory; 35 | 36 | import java.util.Collection; 37 | import java.util.Collections; 38 | import java.util.HashSet; 39 | import java.util.Map; 40 | import java.util.Set; 41 | import java.util.function.BooleanSupplier; 42 | import java.util.stream.Collectors; 43 | 44 | import com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.BehaviorOnNullValues; 45 | 46 | @SuppressWarnings("checkstyle:ClassDataAbstractionCoupling") 47 | public class OpenSearchSinkTask extends SinkTask { 48 | 49 | private static final Logger log = LoggerFactory.getLogger(OpenSearchSinkTask.class); 50 | 51 | private DataConverter converter; 52 | private OpenSearchClient client; 53 | private OpenSearchSinkConnectorConfig config; 54 | private ErrantRecordReporter reporter; 55 | private Set existingMappings; 56 | private Set indexCache; 57 | private OffsetTracker offsetTracker; 58 | private PartitionPauser partitionPauser; 59 | 60 | @Override 61 | public void start(Map props) { 62 | start(props, null); 63 | } 64 | 65 | // visible for testing 66 | protected void start(Map props, OpenSearchClient client) { 67 | log.info("Starting OpenSearchSinkTask."); 68 | 69 | this.config = new OpenSearchSinkConnectorConfig(props); 70 | this.converter = new DataConverter(config); 71 | this.existingMappings = new HashSet<>(); 72 | this.indexCache = new HashSet<>(); 73 | this.offsetTracker = new OffsetTracker(); 74 | 75 | int offsetHighWaterMark = config.maxBufferedRecords() * 10; 76 | int offsetLowWaterMark = config.maxBufferedRecords() * 5; 77 | this.partitionPauser = new PartitionPauser(context, 78 | () -> offsetTracker.numOffsetStateEntries() > offsetHighWaterMark, 79 | () -> offsetTracker.numOffsetStateEntries() <= offsetLowWaterMark); 80 | 81 | this.reporter = null; 82 | try { 83 | if (context.errantRecordReporter() == null) { 84 | log.info("Errant record reporter not configured."); 85 | } 86 | // may be null if DLQ not enabled 87 | reporter = context.errantRecordReporter(); 88 | } catch (NoClassDefFoundError | NoSuchMethodError e) { 89 | // Will occur in Connect runtimes earlier than 2.6 90 | log.warn("AK versions prior to 2.6 do not support the errant record reporter."); 91 | } 92 | 93 | this.client = client != null ? client 94 | : new OpenSearchClient(config, reporter, offsetTracker); 95 | 96 | log.info("Started OpenSearchSinkTask. Connecting to ES server version: {}", 97 | getServerVersion()); 98 | } 99 | 100 | @Override 101 | public void put(Collection records) throws ConnectException { 102 | log.debug("Putting {} records to OpenSsearch.", records.size()); 103 | 104 | client.throwIfFailed(); 105 | partitionPauser.maybeResumePartitions(); 106 | 107 | for (SinkRecord record : records) { 108 | OffsetState offsetState = offsetTracker.addPendingRecord(record); 109 | 110 | if (shouldSkipRecord(record)) { 111 | logTrace("Ignoring {} with null value.", record); 112 | offsetState.markProcessed(); 113 | reportBadRecord(record, new ConnectException("Cannot write null valued record.")); 114 | continue; 115 | } 116 | 117 | logTrace("Writing {} to OpenSearch.", record); 118 | 119 | tryWriteRecord(record, offsetState); 120 | } 121 | partitionPauser.maybePausePartitions(); 122 | } 123 | 124 | @Override 125 | public Map preCommit( 126 | Map currentOffsets) { 127 | try { 128 | // This will just trigger an asynchronous execution of any buffered records 129 | client.flush(); 130 | } catch (IllegalStateException e) { 131 | log.debug("Tried to flush data to OpenSearch, but BulkProcessor is already closed.", e); 132 | } 133 | return super.preCommit(currentOffsets); 134 | } 135 | 136 | @Override 137 | public void stop() { 138 | log.debug("Stopping OpenSearch client."); 139 | client.close(); 140 | } 141 | 142 | @Override 143 | public String version() { 144 | return Version.getVersion(); 145 | } 146 | 147 | private void checkMapping(String index, SinkRecord record) { 148 | if (!config.shouldIgnoreSchema(record.topic()) && !existingMappings.contains(index)) { 149 | if (!client.hasMapping(index)) { 150 | client.createMapping(index, record.valueSchema()); 151 | } 152 | log.debug("Caching mapping for index '{}' locally.", index); 153 | existingMappings.add(index); 154 | } 155 | } 156 | 157 | private String getServerVersion() { 158 | ConfigCallbackHandler configCallbackHandler = new ConfigCallbackHandler(config); 159 | RestHighLevelClient highLevelClient = new RestHighLevelClient( 160 | RestClient 161 | .builder( 162 | config.connectionUrls() 163 | .stream() 164 | .map(HttpHost::create) 165 | .collect(Collectors.toList()) 166 | .toArray(new HttpHost[config.connectionUrls().size()]) 167 | ) 168 | .setHttpClientConfigCallback(configCallbackHandler) 169 | ); 170 | MainResponse response; 171 | String esVersionNumber = "Unknown"; 172 | try { 173 | response = highLevelClient.info(RequestOptions.DEFAULT); 174 | esVersionNumber = response.getVersion().getNumber(); 175 | } catch (Exception e) { 176 | // Same error messages as from validating the connection for IOException. 177 | // Insufficient privileges to validate the version number if caught 178 | // ElasticsearchStatusException. 179 | log.warn("Failed to get ES server version", e); 180 | } finally { 181 | try { 182 | highLevelClient.close(); 183 | } catch (Exception e) { 184 | log.warn("Failed to close high level client", e); 185 | } 186 | } 187 | return esVersionNumber; 188 | } 189 | 190 | /** 191 | * Returns the converted index name from a given topic name. OpenSsearch accepts: 192 | *

    193 | *
  • all lowercase
    • 194 | *
  • less than 256 bytes
    • 195 | *
  • does not start with - or _
    • 196 | *
  • is not . or ..
    • 197 | *
198 | * (ref_.) 199 | */ 200 | private String convertTopicToIndexName(String topic) { 201 | String index = topic.toLowerCase(); 202 | if (index.length() > 255) { 203 | index = index.substring(0, 255); 204 | } 205 | 206 | if (index.startsWith("-") || index.startsWith("_")) { 207 | index = index.substring(1); 208 | } 209 | 210 | if (index.equals(".") || index.equals("..")) { 211 | index = index.replace(".", "dot"); 212 | log.warn("OpenSearch cannot have indices named {}. Index will be named {}.", topic, index); 213 | } 214 | 215 | if (!topic.equals(index)) { 216 | log.trace("Topic '{}' was translated to index '{}'.", topic, index); 217 | } 218 | 219 | return index; 220 | } 221 | 222 | /** 223 | * Returns the converted index name from a given topic name in the form {type}-{dataset}-{topic}. 224 | * For the topic, OpenSsearch accepts: 225 | *
    226 | *
  • all lowercase
    • 227 | *
  • no longer than 100 bytes
    • 228 | *
229 | * (ref_.) 230 | */ 231 | private String convertTopicToDataStreamName(String topic) { 232 | topic = topic.toLowerCase(); 233 | if (topic.length() > 100) { 234 | topic = topic.substring(0, 100); 235 | } 236 | String dataStream = String.format( 237 | "%s-%s-%s", 238 | config.dataStreamType().name().toLowerCase(), 239 | config.dataStreamDataset(), 240 | topic 241 | ); 242 | return dataStream; 243 | } 244 | 245 | /** 246 | * Returns the converted index name from a given topic name. If writing to a data stream, 247 | * returns the index name in the form {type}-{dataset}-{topic}. For both cases, OpenSearch 248 | * accepts: 249 | *
    250 | *
  • all lowercase
    • 251 | *
  • less than 256 bytes
    • 252 | *
  • does not start with - or _
    • 253 | *
  • is not . or ..
    • 254 | *
255 | * (ref_.) 256 | */ 257 | private String createIndexName(String topic) { 258 | return config.isDataStream() 259 | ? convertTopicToDataStreamName(topic) 260 | : convertTopicToIndexName(topic); 261 | } 262 | 263 | private void ensureIndexExists(String index) { 264 | if (!indexCache.contains(index)) { 265 | log.info("Creating index {}.", index); 266 | client.createIndexOrDataStream(index); 267 | indexCache.add(index); 268 | } 269 | } 270 | 271 | private void logTrace(String formatMsg, SinkRecord record) { 272 | if (log.isTraceEnabled()) { 273 | log.trace(formatMsg, recordString(record)); 274 | } 275 | } 276 | 277 | private void reportBadRecord(SinkRecord record, Throwable error) { 278 | if (reporter != null) { 279 | // No need to wait for the futures (synchronously or async), the framework will wait for 280 | // all these futures before calling preCommit 281 | reporter.report(record, error); 282 | } 283 | } 284 | 285 | private boolean shouldSkipRecord(SinkRecord record) { 286 | return record.value() == null && config.behaviorOnNullValues() == BehaviorOnNullValues.IGNORE; 287 | } 288 | 289 | private void tryWriteRecord(SinkRecord sinkRecord, OffsetState offsetState) { 290 | String indexName = createIndexName(sinkRecord.topic()); 291 | 292 | ensureIndexExists(indexName); 293 | checkMapping(indexName, sinkRecord); 294 | 295 | DocWriteRequest docWriteRequest = null; 296 | try { 297 | docWriteRequest = converter.convertRecord(sinkRecord, indexName); 298 | } catch (DataException convertException) { 299 | reportBadRecord(sinkRecord, convertException); 300 | 301 | if (config.dropInvalidMessage()) { 302 | log.error("Can't convert {}.", recordString(sinkRecord), convertException); 303 | offsetState.markProcessed(); 304 | } else { 305 | throw convertException; 306 | } 307 | } 308 | 309 | if (docWriteRequest != null) { 310 | logTrace("Adding {} to bulk processor.", sinkRecord); 311 | client.index(sinkRecord, docWriteRequest, offsetState); 312 | } 313 | } 314 | 315 | private static String recordString(SinkRecord record) { 316 | return String.format( 317 | "record from topic=%s partition=%s offset=%s", 318 | record.topic(), 319 | record.kafkaPartition(), 320 | record.kafkaOffset() 321 | ); 322 | } 323 | 324 | @Override 325 | public void close(Collection partitions) { 326 | offsetTracker.closePartitions(partitions); 327 | } 328 | 329 | // Visible for testing 330 | static class PartitionPauser { 331 | 332 | // Kafka consumer poll timeout to set when partitions are paused, to avoid waiting for a long 333 | // time (default poll timeout) to resume it. 334 | private static final long PAUSE_POLL_TIMEOUT_MS = 100; 335 | 336 | private final SinkTaskContext context; 337 | private final BooleanSupplier pauseCondition; 338 | private final BooleanSupplier resumeCondition; 339 | private boolean partitionsPaused; 340 | 341 | public PartitionPauser(SinkTaskContext context, 342 | BooleanSupplier pauseCondition, 343 | BooleanSupplier resumeCondition) { 344 | this.context = context; 345 | this.pauseCondition = pauseCondition; 346 | this.resumeCondition = resumeCondition; 347 | } 348 | 349 | /** 350 | * Resume partitions if they are paused and resume condition is met. 351 | * Has to be run in the task thread. 352 | */ 353 | void maybeResumePartitions() { 354 | if (partitionsPaused) { 355 | if (resumeCondition.getAsBoolean()) { 356 | log.debug("Resuming all partitions"); 357 | context.resume(context.assignment().toArray(new TopicPartition[0])); 358 | partitionsPaused = false; 359 | } else { 360 | context.timeout(PAUSE_POLL_TIMEOUT_MS); 361 | } 362 | } 363 | } 364 | 365 | /** 366 | * Pause partitions if they are not paused and pause condition is met. 367 | * Has to be run in the task thread. 368 | */ 369 | void maybePausePartitions() { 370 | if (!partitionsPaused && pauseCondition.getAsBoolean()) { 371 | log.debug("Pausing all partitions"); 372 | context.pause(context.assignment().toArray(new TopicPartition[0])); 373 | context.timeout(PAUSE_POLL_TIMEOUT_MS); 374 | partitionsPaused = true; 375 | } 376 | } 377 | } 378 | } 379 | -------------------------------------------------------------------------------- /lib/src/main/java/com/dmathieu/kafka/opensearch/RetryUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch; 17 | 18 | import java.util.concurrent.Callable; 19 | import java.util.concurrent.ThreadLocalRandom; 20 | import java.util.concurrent.TimeUnit; 21 | 22 | import org.apache.kafka.common.utils.Time; 23 | import org.apache.kafka.connect.errors.ConnectException; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | /** 28 | * Utility to compute the retry times for a given attempt, using exponential backoff. 29 | * 30 | *

The purposes of using exponential backoff is to give the ES service time to recover when it 31 | * becomes overwhelmed. Adding jitter attempts to prevent a thundering herd, where large numbers 32 | * of requests from many tasks overwhelm the ES service, and without randomization all tasks 33 | * retry at the same time. Randomization should spread the retries out and should reduce the 34 | * overall time required to complete all attempts. 35 | * See this blog post 36 | * for details. 37 | */ 38 | public class RetryUtil { 39 | 40 | private static final Logger log = LoggerFactory.getLogger(RetryUtil.class); 41 | 42 | /** 43 | * An arbitrary absolute maximum practical retry time. 44 | */ 45 | public static final long MAX_RETRY_TIME_MS = TimeUnit.HOURS.toMillis(24); 46 | 47 | /** 48 | * Compute the time to sleep using exponential backoff with jitter. This method computes the 49 | * normal exponential backoff as {@code initialRetryBackoffMs << retryAttempt}, and then 50 | * chooses a random value between 0 and that value. 51 | * 52 | * @param retryAttempts the number of previous retry attempts; must be non-negative 53 | * @param initialRetryBackoffMs the initial time to wait before retrying; assumed to 54 | * be 0 if value is negative 55 | * @return the non-negative time in milliseconds to wait before the next retry attempt, 56 | * or 0 if {@code initialRetryBackoffMs} is negative 57 | */ 58 | public static long computeRandomRetryWaitTimeInMillis(int retryAttempts, 59 | long initialRetryBackoffMs) { 60 | if (initialRetryBackoffMs < 0) { 61 | return 0; 62 | } 63 | if (retryAttempts < 0) { 64 | return initialRetryBackoffMs; 65 | } 66 | long maxRetryTime = computeRetryWaitTimeInMillis(retryAttempts, initialRetryBackoffMs); 67 | return ThreadLocalRandom.current().nextLong(0, maxRetryTime); 68 | } 69 | 70 | /** 71 | * Compute the time to sleep using exponential backoff. This method computes the normal 72 | * exponential backoff as {@code initialRetryBackoffMs << retryAttempt}, bounded to always 73 | * be less than {@link #MAX_RETRY_TIME_MS}. 74 | * 75 | * @param retryAttempts the number of previous retry attempts; must be non-negative 76 | * @param initialRetryBackoffMs the initial time to wait before retrying; assumed to be 0 77 | * if value is negative 78 | * @return the non-negative time in milliseconds to wait before the next retry attempt, 79 | * or 0 if {@code initialRetryBackoffMs} is negative 80 | */ 81 | public static long computeRetryWaitTimeInMillis(int retryAttempts, 82 | long initialRetryBackoffMs) { 83 | if (initialRetryBackoffMs < 0) { 84 | return 0; 85 | } 86 | if (retryAttempts <= 0) { 87 | return initialRetryBackoffMs; 88 | } 89 | if (retryAttempts > 32) { 90 | // This would overflow the exponential algorithm ... 91 | return MAX_RETRY_TIME_MS; 92 | } 93 | long result = initialRetryBackoffMs << retryAttempts; 94 | return result < 0L ? MAX_RETRY_TIME_MS : Math.min(MAX_RETRY_TIME_MS, result); 95 | } 96 | 97 | /** 98 | * Call the supplied function up to the {@code maxTotalAttempts}. 99 | * 100 | *

The description of the function should be a succinct, human-readable present tense phrase 101 | * that summarizes the function, such as "read tables" or "connect to database" or 102 | * "make remote request". This description will be used within exception and log messages. 103 | * 104 | * @param description present tense description of the action, used to create the error 105 | * message; may not be null 106 | * @param function the function to call; may not be null 107 | * @param maxTotalAttempts maximum number of total attempts, including the first call 108 | * @param initialBackoff the initial backoff in ms before retrying 109 | * @param the return type of the function to retry 110 | * @return the function's return value 111 | * @throws ConnectException if the function failed after retries 112 | */ 113 | public static T callWithRetries( 114 | String description, 115 | Callable function, 116 | int maxTotalAttempts, 117 | long initialBackoff 118 | ) { 119 | return callWithRetries(description, function, maxTotalAttempts, initialBackoff, Time.SYSTEM); 120 | } 121 | 122 | /** 123 | * Call the supplied function up to the {@code maxTotalAttempts}. 124 | * 125 | *

The description of the function should be a succinct, human-readable present tense phrase 126 | * that summarizes the function, such as "read tables" or "connect to database" or 127 | * "make remote request". This description will be used within exception and log messages. 128 | * 129 | * @param description present tense description of the action, used to create the error 130 | * message; may not be null 131 | * @param function the function to call; may not be null 132 | * @param maxTotalAttempts maximum number of attempts 133 | * @param initialBackoff the initial backoff in ms before retrying 134 | * @param clock the clock to use for waiting 135 | * @param the return type of the function to retry 136 | * @return the function's return value 137 | * @throws ConnectException if the function failed after retries 138 | */ 139 | protected static T callWithRetries( 140 | String description, 141 | Callable function, 142 | int maxTotalAttempts, 143 | long initialBackoff, 144 | Time clock 145 | ) { 146 | assert description != null; 147 | assert function != null; 148 | int attempt = 0; 149 | while (true) { 150 | ++attempt; 151 | try { 152 | log.trace( 153 | "Try {} (attempt {} of {})", 154 | description, 155 | attempt, 156 | maxTotalAttempts 157 | ); 158 | T call = function.call(); 159 | return call; 160 | } catch (Exception e) { 161 | if (attempt >= maxTotalAttempts) { 162 | String msg = String.format("Failed to %s due to '%s' after %d attempt(s)", 163 | description, e, attempt); 164 | log.error(msg, e); 165 | throw new ConnectException(msg, e); 166 | } 167 | 168 | // Otherwise it is retriable and we should retry 169 | long backoff = computeRandomRetryWaitTimeInMillis(attempt, initialBackoff); 170 | 171 | log.warn("Failed to {} due to {}. Retrying attempt ({}/{}) after backoff of {} ms", 172 | description, e.getCause(), attempt, maxTotalAttempts, backoff); 173 | clock.sleep(backoff); 174 | } 175 | } 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /lib/src/main/java/com/dmathieu/kafka/opensearch/UnsafeX509ExtendedTrustManager.java: -------------------------------------------------------------------------------- 1 | package com.dmathieu.kafka.opensearch; 2 | 3 | import javax.net.ssl.SSLEngine; 4 | import javax.net.ssl.X509ExtendedTrustManager; 5 | import java.net.Socket; 6 | import java.security.cert.X509Certificate; 7 | 8 | public final class UnsafeX509ExtendedTrustManager extends X509ExtendedTrustManager { 9 | private static final X509ExtendedTrustManager INSTANCE = new UnsafeX509ExtendedTrustManager(); 10 | private static final X509Certificate[] EMPTY_CERTIFICATES = new X509Certificate[0]; 11 | 12 | private UnsafeX509ExtendedTrustManager() {} 13 | 14 | public static X509ExtendedTrustManager getInstance() { 15 | return INSTANCE; 16 | } 17 | 18 | @Override 19 | public void checkClientTrusted(X509Certificate[] certificates, String authType) { 20 | // ignore certificate validation 21 | } 22 | 23 | @Override 24 | public void checkClientTrusted(X509Certificate[] certificates, String authType, Socket socket) { 25 | // ignore certificate validation 26 | } 27 | 28 | @Override 29 | public void checkClientTrusted(X509Certificate[] certificates, String authType, SSLEngine sslEngine) { 30 | // ignore certificate validation 31 | } 32 | 33 | @Override 34 | public void checkServerTrusted(X509Certificate[] certificates, String authType) { 35 | // ignore certificate validation 36 | } 37 | 38 | @Override 39 | public void checkServerTrusted(X509Certificate[] certificates, String authType, Socket socket) { 40 | // ignore certificate validation 41 | } 42 | 43 | @Override 44 | public void checkServerTrusted(X509Certificate[] certificates, String authType, SSLEngine sslEngine) { 45 | // ignore certificate validation 46 | } 47 | 48 | @Override 49 | public X509Certificate[] getAcceptedIssuers() { 50 | return EMPTY_CERTIFICATES; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /lib/src/main/java/com/dmathieu/kafka/opensearch/Version.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch; 17 | 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | 21 | import java.io.InputStream; 22 | import java.util.Properties; 23 | 24 | public class Version { 25 | private static final Logger log = LoggerFactory.getLogger(Version.class); 26 | private static String version = "unknown"; 27 | 28 | private static final String VERSION_FILE = "/kafka-connect-opensearch-version.properties"; 29 | 30 | static { 31 | try { 32 | Properties props = new Properties(); 33 | try (InputStream versionFileStream = Version.class.getResourceAsStream(VERSION_FILE)) { 34 | props.load(versionFileStream); 35 | version = props.getProperty("version", version).trim(); 36 | } 37 | } catch (Exception e) { 38 | log.warn("Error while loading version:", e); 39 | version = e.toString(); 40 | } 41 | } 42 | 43 | public static String getVersion() { 44 | return version; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /lib/src/main/resources/kafka-connect-opensearch-version.properties: -------------------------------------------------------------------------------- 1 | # DO NOT UPDATE THIS FILE 2 | # The version is set by the Action which builds the project and uploads the assets to GitHub. 3 | 4 | version=DEV 5 | -------------------------------------------------------------------------------- /lib/src/test/java/com/dmathieu/kafka/opensearch/MappingTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch; 17 | 18 | import com.google.gson.JsonObject; 19 | import com.google.gson.JsonParser; 20 | import java.io.ByteArrayOutputStream; 21 | import java.io.IOException; 22 | import java.util.HashMap; 23 | import org.apache.kafka.connect.data.Date; 24 | import org.apache.kafka.connect.data.Decimal; 25 | import org.apache.kafka.connect.data.Field; 26 | import org.apache.kafka.connect.data.Schema; 27 | import org.apache.kafka.connect.data.SchemaBuilder; 28 | import org.apache.kafka.connect.data.Time; 29 | import org.apache.kafka.connect.data.Timestamp; 30 | import org.apache.kafka.connect.errors.DataException; 31 | import org.elasticsearch.common.xcontent.XContentBuilder; 32 | 33 | import static com.dmathieu.kafka.opensearch.Mapping.KEYWORD_TYPE; 34 | import static com.dmathieu.kafka.opensearch.Mapping.KEY_FIELD; 35 | import static com.dmathieu.kafka.opensearch.Mapping.TEXT_TYPE; 36 | import static com.dmathieu.kafka.opensearch.Mapping.VALUE_FIELD; 37 | 38 | import org.junit.jupiter.api.Test; 39 | import static org.junit.jupiter.api.Assertions.*; 40 | 41 | public class MappingTest { 42 | 43 | @Test 44 | public void testBuildMappingWithNullSchema() { 45 | assertThrows(DataException.class, () -> { 46 | XContentBuilder builder = Mapping.buildMapping(null); 47 | }); 48 | } 49 | 50 | @Test 51 | public void testBuildMapping() throws IOException { 52 | JsonObject result = runTest(createSchema()); 53 | verifyMapping(createSchema(), result); 54 | } 55 | 56 | @Test 57 | public void testBuildMappingForString() throws IOException { 58 | Schema schema = SchemaBuilder.struct() 59 | .name("record") 60 | .field("string", Schema.STRING_SCHEMA) 61 | .build(); 62 | 63 | JsonObject result = runTest(schema); 64 | JsonObject string = result.getAsJsonObject("properties").getAsJsonObject("string"); 65 | JsonObject keyword = string.getAsJsonObject("fields").getAsJsonObject("keyword"); 66 | 67 | assertEquals(TEXT_TYPE, string.get("type").getAsString()); 68 | assertEquals(KEYWORD_TYPE, keyword.get("type").getAsString()); 69 | assertEquals(256, keyword.get("ignore_above").getAsInt()); 70 | } 71 | 72 | @Test 73 | public void testBuildMappingSetsDefaultValue() throws IOException { 74 | Schema schema = SchemaBuilder 75 | .struct() 76 | .name("record") 77 | .field("boolean", SchemaBuilder.bool().defaultValue(true).build()) 78 | .field("int8", SchemaBuilder.int8().defaultValue((byte) 1).build()) 79 | .field("int16", SchemaBuilder.int16().defaultValue((short) 1).build()) 80 | .field("int32", SchemaBuilder.int32().defaultValue(1).build()) 81 | .field("int64", SchemaBuilder.int64().defaultValue((long) 1).build()) 82 | .field("float32", SchemaBuilder.float32().defaultValue((float) 1).build()) 83 | .field("float64", SchemaBuilder.float64().defaultValue((double) 1).build()) 84 | .build(); 85 | 86 | JsonObject properties = runTest(schema).getAsJsonObject("properties"); 87 | assertEquals(1, properties.getAsJsonObject("int8").get("null_value").getAsInt()); 88 | assertEquals(1, properties.getAsJsonObject("int16").get("null_value").getAsInt()); 89 | assertEquals(1, properties.getAsJsonObject("int32").get("null_value").getAsInt()); 90 | assertEquals(1, properties.getAsJsonObject("int64").get("null_value").getAsInt()); 91 | assertEquals(1, properties.getAsJsonObject("float32").get("null_value").getAsInt()); 92 | assertEquals(1, properties.getAsJsonObject("float64").get("null_value").getAsInt()); 93 | assertEquals(true, properties.getAsJsonObject("boolean").get("null_value").getAsBoolean()); 94 | } 95 | 96 | @Test 97 | public void testBuildMappingSetsDefaultValueForDate() throws IOException { 98 | java.util.Date expected = new java.util.Date(); 99 | Schema schema = SchemaBuilder 100 | .struct() 101 | .name("record") 102 | .field("date", Date.builder().defaultValue(expected).build()) 103 | .build(); 104 | 105 | JsonObject result = runTest(schema); 106 | 107 | assertEquals( 108 | expected.getTime(), 109 | result.getAsJsonObject("properties").getAsJsonObject("date").get("null_value").getAsLong() 110 | ); 111 | } 112 | 113 | @Test 114 | public void testBuildMappingSetsNoDefaultValueForStrings() throws IOException { 115 | Schema schema = SchemaBuilder 116 | .struct() 117 | .name("record") 118 | .field("string", SchemaBuilder.string().defaultValue("0").build()) 119 | .build(); 120 | 121 | JsonObject result = runTest(schema); 122 | 123 | assertNull(result.getAsJsonObject("properties").getAsJsonObject("string").get("null_value")); 124 | } 125 | 126 | private Schema createSchema() { 127 | return createSchemaBuilder("record") 128 | .field("struct", createSchemaBuilder("inner").build()) 129 | .build(); 130 | } 131 | 132 | private SchemaBuilder createSchemaBuilder(String name) { 133 | return SchemaBuilder.struct().name(name) 134 | .field("boolean", Schema.BOOLEAN_SCHEMA) 135 | .field("bytes", Schema.BYTES_SCHEMA) 136 | .field("int8", Schema.INT8_SCHEMA) 137 | .field("int16", Schema.INT16_SCHEMA) 138 | .field("int32", Schema.INT32_SCHEMA) 139 | .field("int64", Schema.INT64_SCHEMA) 140 | .field("float32", Schema.FLOAT32_SCHEMA) 141 | .field("float64", Schema.FLOAT64_SCHEMA) 142 | .field("string", Schema.STRING_SCHEMA) 143 | .field("array", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) 144 | .field("map", SchemaBuilder.map(Schema.STRING_SCHEMA, Schema.INT32_SCHEMA).build()) 145 | .field("decimal", Decimal.schema(2)) 146 | .field("date", Date.SCHEMA) 147 | .field("time", Time.SCHEMA) 148 | .field("timestamp", Timestamp.SCHEMA); 149 | } 150 | 151 | private static JsonObject runTest(Schema schema) throws IOException { 152 | XContentBuilder builder = Mapping.buildMapping(schema); 153 | builder.flush(); 154 | ByteArrayOutputStream stream = (ByteArrayOutputStream) builder.getOutputStream(); 155 | return (JsonObject) JsonParser.parseString(stream.toString()); 156 | } 157 | 158 | private void verifyMapping(Schema schema, JsonObject mapping) { 159 | String schemaName = schema.name(); 160 | Object type = mapping.get("type"); 161 | if (schemaName != null) { 162 | switch (schemaName) { 163 | case Date.LOGICAL_NAME: 164 | case Time.LOGICAL_NAME: 165 | case Timestamp.LOGICAL_NAME: 166 | assertEquals("\"" + Mapping.DATE_TYPE + "\"", type.toString()); 167 | return; 168 | case Decimal.LOGICAL_NAME: 169 | assertEquals("\"" + Mapping.DOUBLE_TYPE + "\"", type.toString()); 170 | return; 171 | } 172 | } 173 | 174 | DataConverter converter = new DataConverter(new OpenSearchSinkConnectorConfig(OpenSearchSinkConnectorConfigTest.addNecessaryProps(new HashMap<>()))); 175 | Schema.Type schemaType = schema.type(); 176 | switch (schemaType) { 177 | case ARRAY: 178 | verifyMapping(schema.valueSchema(), mapping); 179 | break; 180 | case MAP: 181 | Schema newSchema = converter.preProcessSchema(schema); 182 | JsonObject mapProperties = mapping.get("properties").getAsJsonObject(); 183 | verifyMapping(newSchema.keySchema(), mapProperties.get(KEY_FIELD).getAsJsonObject()); 184 | verifyMapping(newSchema.valueSchema(), mapProperties.get(VALUE_FIELD).getAsJsonObject()); 185 | break; 186 | case STRUCT: 187 | JsonObject properties = mapping.get("properties").getAsJsonObject(); 188 | for (Field field: schema.fields()) { 189 | verifyMapping(field.schema(), properties.get(field.name()).getAsJsonObject()); 190 | } 191 | break; 192 | default: 193 | assertEquals("\"" + Mapping.getOpenSearchType(schemaType) + "\"", type.toString()); 194 | } 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /lib/src/test/java/com/dmathieu/kafka/opensearch/OffsetTrackerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch; 17 | 18 | import com.dmathieu.kafka.opensearch.OffsetTracker.OffsetState; 19 | import org.apache.kafka.clients.consumer.OffsetAndMetadata; 20 | import org.apache.kafka.common.TopicPartition; 21 | import org.apache.kafka.connect.sink.SinkRecord; 22 | import org.testcontainers.shaded.com.google.common.collect.ImmutableList; 23 | 24 | import org.junit.jupiter.api.Test; 25 | 26 | import java.util.Map; 27 | 28 | import static org.assertj.core.api.Assertions.assertThat; 29 | 30 | public class OffsetTrackerTest { 31 | 32 | @Test 33 | public void testHappyPath() { 34 | OffsetTracker offsetTracker = new OffsetTracker(); 35 | 36 | TopicPartition tp = new TopicPartition("t1", 0); 37 | 38 | SinkRecord record1 = sinkRecord(tp, 0); 39 | SinkRecord record2 = sinkRecord(tp, 1); 40 | SinkRecord record3 = sinkRecord(tp, 2); 41 | 42 | OffsetState offsetState1 = offsetTracker.addPendingRecord(record1); 43 | OffsetState offsetState2 = offsetTracker.addPendingRecord(record2); 44 | OffsetState offsetState3 = offsetTracker.addPendingRecord(record3); 45 | 46 | assertThat(offsetTracker.offsets()).isEmpty(); 47 | 48 | offsetState2.markProcessed(); 49 | assertThat(offsetTracker.offsets()).isEmpty(); 50 | 51 | offsetState1.markProcessed(); 52 | 53 | offsetTracker.updateOffsets(); 54 | Map offsetMap = offsetTracker.offsets(); 55 | assertThat(offsetMap).hasSize(1); 56 | assertThat(offsetMap.get(tp).offset()).isEqualTo(2); 57 | 58 | offsetState3.markProcessed(); 59 | offsetTracker.updateOffsets(); 60 | offsetMap = offsetTracker.offsets(); 61 | assertThat(offsetMap).hasSize(1); 62 | assertThat(offsetMap.get(tp).offset()).isEqualTo(3); 63 | 64 | offsetTracker.updateOffsets(); 65 | assertThat(offsetMap.get(tp).offset()).isEqualTo(3); 66 | } 67 | 68 | /** 69 | * Verify that if we receive records that are below the already committed offset for partition 70 | * (e.g. after a RetriableException), the offset reporting is not affected. 71 | */ 72 | @Test 73 | public void testBelowWatermark() { 74 | OffsetTracker offsetTracker = new OffsetTracker(); 75 | 76 | TopicPartition tp = new TopicPartition("t1", 0); 77 | 78 | SinkRecord record1 = sinkRecord(tp, 0); 79 | SinkRecord record2 = sinkRecord(tp, 1); 80 | 81 | OffsetState offsetState1 = offsetTracker.addPendingRecord(record1); 82 | OffsetState offsetState2 = offsetTracker.addPendingRecord(record2); 83 | 84 | offsetState1.markProcessed(); 85 | offsetState2.markProcessed(); 86 | offsetTracker.updateOffsets(); 87 | assertThat(offsetTracker.offsets().get(tp).offset()).isEqualTo(2); 88 | 89 | offsetState2 = offsetTracker.addPendingRecord(record2); 90 | offsetTracker.updateOffsets(); 91 | assertThat(offsetTracker.offsets().get(tp).offset()).isEqualTo(2); 92 | 93 | offsetState2.markProcessed(); 94 | offsetTracker.updateOffsets(); 95 | assertThat(offsetTracker.offsets().get(tp).offset()).isEqualTo(2); 96 | } 97 | 98 | @Test 99 | public void testBatchRetry() { 100 | OffsetTracker offsetTracker = new OffsetTracker(); 101 | 102 | TopicPartition tp = new TopicPartition("t1", 0); 103 | 104 | SinkRecord record1 = sinkRecord(tp, 0); 105 | SinkRecord record2 = sinkRecord(tp, 1); 106 | 107 | OffsetState offsetState1A = offsetTracker.addPendingRecord(record1); 108 | OffsetState offsetState2A = offsetTracker.addPendingRecord(record2); 109 | 110 | // first fails but second succeeds 111 | offsetState2A.markProcessed(); 112 | offsetTracker.updateOffsets(); 113 | assertThat(offsetTracker.offsets()).isEmpty(); 114 | 115 | // now simulate the batch being retried by the framework (e.g. after a RetriableException) 116 | OffsetState offsetState1B = offsetTracker.addPendingRecord(record1); 117 | OffsetState offsetState2B = offsetTracker.addPendingRecord(record2); 118 | 119 | offsetState2B.markProcessed(); 120 | offsetState1B.markProcessed(); 121 | offsetTracker.updateOffsets(); 122 | assertThat(offsetTracker.offsets().get(tp).offset()).isEqualTo(2); 123 | } 124 | 125 | @Test 126 | public void testRebalance() { 127 | OffsetTracker offsetTracker = new OffsetTracker(); 128 | 129 | TopicPartition tp1 = new TopicPartition("t1", 0); 130 | TopicPartition tp2 = new TopicPartition("t2", 0); 131 | TopicPartition tp3 = new TopicPartition("t3", 0); 132 | 133 | offsetTracker.addPendingRecord(sinkRecord(tp1, 0)).markProcessed(); 134 | offsetTracker.addPendingRecord(sinkRecord(tp1, 1)); 135 | offsetTracker.addPendingRecord(sinkRecord(tp2, 0)).markProcessed(); 136 | assertThat(offsetTracker.numOffsetStateEntries()).isEqualTo(3); 137 | 138 | offsetTracker.updateOffsets(); 139 | assertThat(offsetTracker.offsets().size()).isEqualTo(2); 140 | assertThat(offsetTracker.numOffsetStateEntries()).isEqualTo(1); 141 | 142 | offsetTracker.closePartitions(ImmutableList.of(tp1, tp3)); 143 | assertThat(offsetTracker.offsets().keySet()).containsExactly(tp2); 144 | assertThat(offsetTracker.numOffsetStateEntries()).isEqualTo(0); 145 | } 146 | 147 | private SinkRecord sinkRecord(TopicPartition tp, long offset) { 148 | return sinkRecord(tp.topic(), tp.partition(), offset); 149 | } 150 | 151 | private SinkRecord sinkRecord(String topic, int partition, long offset) { 152 | return new SinkRecord(topic, 153 | partition, 154 | null, 155 | "testKey", 156 | null, 157 | "testValue" + offset, 158 | offset); 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /lib/src/test/java/com/dmathieu/kafka/opensearch/OpenSearchSinkConnectorConfigTest.java: -------------------------------------------------------------------------------- 1 | package com.dmathieu.kafka.opensearch; 2 | 3 | 4 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.CONNECTION_PASSWORD_CONFIG; 5 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.CONNECTION_USERNAME_CONFIG; 6 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.CONNECTION_TIMEOUT_MS_CONFIG; 7 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.CONNECTION_URL_CONFIG; 8 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.DATA_STREAM_DATASET_CONFIG; 9 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.DATA_STREAM_TYPE_CONFIG; 10 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.KERBEROS_KEYTAB_PATH_CONFIG; 11 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.PROXY_HOST_CONFIG; 12 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.PROXY_PASSWORD_CONFIG; 13 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.PROXY_PORT_CONFIG; 14 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.PROXY_USERNAME_CONFIG; 15 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.READ_TIMEOUT_MS_CONFIG; 16 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.SECURITY_PROTOCOL_CONFIG; 17 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.SSL_CONFIG_PREFIX; 18 | 19 | import static com.dmathieu.kafka.opensearch.helper.OpenSearchContainer.OPENSEARCH_USER_NAME; 20 | import static com.dmathieu.kafka.opensearch.helper.OpenSearchContainer.OPENSEARCH_USER_PASSWORD; 21 | 22 | import static org.apache.kafka.common.config.SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG; 23 | 24 | import com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.SecurityProtocol; 25 | import java.io.IOException; 26 | import java.nio.file.Files; 27 | import java.nio.file.Path; 28 | import java.util.HashMap; 29 | import java.util.Map; 30 | import org.apache.kafka.common.config.ConfigException; 31 | import org.apache.kafka.common.config.SslConfigs; 32 | import org.apache.kafka.common.config.types.Password; 33 | 34 | import org.junit.jupiter.api.Test; 35 | import org.junit.jupiter.api.BeforeEach; 36 | import static org.junit.jupiter.api.Assertions.*; 37 | 38 | public class OpenSearchSinkConnectorConfigTest { 39 | 40 | private Map props; 41 | 42 | @BeforeEach 43 | public void setup() { 44 | props = addNecessaryProps(new HashMap<>()); 45 | } 46 | 47 | @Test 48 | public void testDefaultHttpTimeoutsConfig() { 49 | OpenSearchSinkConnectorConfig config = new OpenSearchSinkConnectorConfig(props); 50 | assertEquals(config.readTimeoutMs(), 3000); 51 | assertEquals(config.connectionTimeoutMs(), 1000); 52 | } 53 | 54 | @Test 55 | public void testSetHttpTimeoutsConfig() { 56 | props.put(READ_TIMEOUT_MS_CONFIG, "10000"); 57 | props.put(CONNECTION_TIMEOUT_MS_CONFIG, "15000"); 58 | OpenSearchSinkConnectorConfig config = new OpenSearchSinkConnectorConfig(props); 59 | 60 | assertEquals(config.readTimeoutMs(), 10000); 61 | assertEquals(config.connectionTimeoutMs(), 15000); 62 | } 63 | 64 | @Test 65 | public void shouldAllowValidChractersDataStreamDataset() { 66 | props.put(DATA_STREAM_DATASET_CONFIG, "a_valid.dataset123"); 67 | new OpenSearchSinkConnectorConfig(props); 68 | } 69 | 70 | @Test 71 | public void shouldAllowValidDataStreamType() { 72 | props.put(DATA_STREAM_TYPE_CONFIG, "metrics"); 73 | new OpenSearchSinkConnectorConfig(props); 74 | } 75 | 76 | @Test 77 | public void shouldAllowValidDataStreamTypeCaseInsensitive() { 78 | props.put(DATA_STREAM_TYPE_CONFIG, "mEtRICS"); 79 | new OpenSearchSinkConnectorConfig(props); 80 | } 81 | 82 | @Test 83 | public void shouldNotAllowInvalidCaseDataStreamDataset() { 84 | assertThrows(ConfigException.class, () -> { 85 | props.put(DATA_STREAM_DATASET_CONFIG, "AN_INVALID.dataset123"); 86 | new OpenSearchSinkConnectorConfig(props); 87 | }); 88 | } 89 | 90 | @Test 91 | public void shouldNotAllowInvalidCharactersDataStreamDataset() { 92 | assertThrows(ConfigException.class, () -> { 93 | props.put(DATA_STREAM_DATASET_CONFIG, "not-valid?"); 94 | new OpenSearchSinkConnectorConfig(props); 95 | }); 96 | } 97 | 98 | @Test 99 | public void shouldNotAllowInvalidDataStreamType() { 100 | assertThrows(ConfigException.class, () -> { 101 | props.put(DATA_STREAM_TYPE_CONFIG, "notLogOrMetrics"); 102 | new OpenSearchSinkConnectorConfig(props); 103 | }); 104 | } 105 | 106 | @Test 107 | public void shouldNotAllowLongDataStreamDataset() { 108 | assertThrows(ConfigException.class, () -> { 109 | props.put(DATA_STREAM_DATASET_CONFIG, String.format("%d%100d", 1, 1)); 110 | new OpenSearchSinkConnectorConfig(props); 111 | }); 112 | } 113 | 114 | @Test 115 | public void shouldNotAllowNullUrlList(){ 116 | assertThrows(ConfigException.class, () -> { 117 | props.put(CONNECTION_URL_CONFIG, null); 118 | new OpenSearchSinkConnectorConfig(props); 119 | }); 120 | } 121 | 122 | @Test 123 | public void testSslConfigs() { 124 | props.put(SSL_CONFIG_PREFIX + SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, "/path"); 125 | props.put(SSL_CONFIG_PREFIX + SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, "opensesame"); 126 | props.put(SSL_CONFIG_PREFIX + SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, "/path2"); 127 | props.put(SSL_CONFIG_PREFIX + SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, "opensesame2"); 128 | OpenSearchSinkConnectorConfig config = new OpenSearchSinkConnectorConfig(props); 129 | 130 | Map sslConfigs = config.sslConfigs(); 131 | assertTrue(sslConfigs.size() > 0); 132 | assertEquals( 133 | new Password("opensesame"), 134 | sslConfigs.get(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG) 135 | ); 136 | assertEquals( 137 | new Password("opensesame2"), 138 | sslConfigs.get(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG) 139 | ); 140 | assertEquals("/path", sslConfigs.get(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)); 141 | assertEquals("/path2", sslConfigs.get(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)); 142 | } 143 | 144 | @Test 145 | public void shouldAcceptValidBasicProxy() { 146 | props.put(PROXY_HOST_CONFIG, "proxy host"); 147 | OpenSearchSinkConnectorConfig config = new OpenSearchSinkConnectorConfig(props); 148 | 149 | assertNotNull(config); 150 | assertTrue(config.isBasicProxyConfigured()); 151 | assertFalse(config.isProxyWithAuthenticationConfigured()); 152 | } 153 | 154 | @Test 155 | public void shouldAcceptValidProxyWithAuthentication() { 156 | props.put(PROXY_HOST_CONFIG, "proxy host"); 157 | props.put(PROXY_PORT_CONFIG, "1010"); 158 | props.put(PROXY_USERNAME_CONFIG, "username"); 159 | props.put(PROXY_PASSWORD_CONFIG, "password"); 160 | OpenSearchSinkConnectorConfig config = new OpenSearchSinkConnectorConfig(props); 161 | 162 | assertNotNull(config); 163 | assertTrue(config.isBasicProxyConfigured()); 164 | assertTrue(config.isProxyWithAuthenticationConfigured()); 165 | assertEquals("proxy host", config.proxyHost()); 166 | assertEquals(1010, config.proxyPort()); 167 | assertEquals("username", config.proxyUsername()); 168 | assertEquals("password", config.proxyPassword().value()); 169 | } 170 | 171 | @Test 172 | public void shouldNotAllowInvalidProxyPort() { 173 | assertThrows(ConfigException.class, () -> { 174 | props.put(PROXY_PORT_CONFIG, "-666"); 175 | new OpenSearchSinkConnectorConfig(props); 176 | }); 177 | } 178 | 179 | @Test 180 | public void shouldNotAllowInvalidUrl() { 181 | assertThrows(ConfigException.class, () -> { 182 | props.put(CONNECTION_URL_CONFIG, ".com:/bbb/dfs,http://valid.com"); 183 | new OpenSearchSinkConnectorConfig(props); 184 | }); 185 | } 186 | 187 | @Test 188 | public void shouldNotAllowInvalidSecurityProtocol() { 189 | assertThrows(ConfigException.class, () -> { 190 | props.put(SECURITY_PROTOCOL_CONFIG, "unsecure"); 191 | new OpenSearchSinkConnectorConfig(props); 192 | }); 193 | } 194 | 195 | @Test 196 | public void shouldDisableHostnameVerification() { 197 | props.put(SSL_CONFIG_PREFIX + SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, "https"); 198 | OpenSearchSinkConnectorConfig config = new OpenSearchSinkConnectorConfig(props); 199 | assertFalse(config.shouldDisableHostnameVerification()); 200 | 201 | props.put(SSL_CONFIG_PREFIX + SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, ""); 202 | config = new OpenSearchSinkConnectorConfig(props); 203 | assertTrue(config.shouldDisableHostnameVerification()); 204 | 205 | props.put(SSL_CONFIG_PREFIX + SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, null); 206 | config = new OpenSearchSinkConnectorConfig(props); 207 | assertFalse(config.shouldDisableHostnameVerification()); 208 | } 209 | 210 | @Test 211 | public void shouldNotAllowInvalidExtensionKeytab() { 212 | assertThrows(ConfigException.class, () -> { 213 | props.put(KERBEROS_KEYTAB_PATH_CONFIG, "keytab.wrongextension"); 214 | new OpenSearchSinkConnectorConfig(props); 215 | }); 216 | } 217 | 218 | @Test 219 | public void shouldNotAllowNonExistingKeytab() { 220 | assertThrows(ConfigException.class, () -> { 221 | props.put(KERBEROS_KEYTAB_PATH_CONFIG, "idontexist.keytab"); 222 | new OpenSearchSinkConnectorConfig(props); 223 | }); 224 | } 225 | 226 | @Test 227 | public void shouldAllowValidKeytab() throws IOException { 228 | Path keytab = Files.createTempFile("iexist", ".keytab"); 229 | props.put(KERBEROS_KEYTAB_PATH_CONFIG, keytab.toString()); 230 | 231 | new OpenSearchSinkConnectorConfig(props); 232 | 233 | keytab.toFile().delete(); 234 | } 235 | 236 | public static Map addNecessaryProps(Map props) { 237 | if (props == null) { 238 | props = new HashMap<>(); 239 | } 240 | props.put(OpenSearchSinkConnectorConfig.CONNECTION_URL_CONFIG, "http://localhost:8080"); 241 | props.put(SSL_CONFIG_PREFIX + SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, ""); 242 | props.put(CONNECTION_USERNAME_CONFIG, OPENSEARCH_USER_NAME); 243 | props.put(CONNECTION_PASSWORD_CONFIG, OPENSEARCH_USER_PASSWORD); 244 | 245 | return props; 246 | } 247 | } 248 | -------------------------------------------------------------------------------- /lib/src/test/java/com/dmathieu/kafka/opensearch/OpenSearchSinkConnectorTest.java: -------------------------------------------------------------------------------- 1 | package com.dmathieu.kafka.opensearch; 2 | 3 | import java.util.HashMap; 4 | import java.util.List; 5 | import java.util.Map; 6 | import org.apache.kafka.connect.errors.ConnectException; 7 | import org.apache.kafka.connect.sink.SinkConnector; 8 | 9 | import org.junit.jupiter.api.Test; 10 | import org.junit.jupiter.api.BeforeEach; 11 | import static org.junit.jupiter.api.Assertions.*; 12 | 13 | public class OpenSearchSinkConnectorTest { 14 | 15 | private OpenSearchSinkConnector connector; 16 | private Map settings; 17 | 18 | @BeforeEach 19 | public void before() { 20 | settings = OpenSearchSinkConnectorConfigTest.addNecessaryProps(new HashMap<>()); 21 | connector = new OpenSearchSinkConnector(); 22 | } 23 | 24 | @Test 25 | public void shouldCatchInvalidConfigs() { 26 | assertThrows(ConnectException.class, () -> { 27 | connector.start(new HashMap<>()); 28 | }); 29 | } 30 | 31 | @Test 32 | public void shouldGenerateValidTaskConfigs() { 33 | connector.start(settings); 34 | List> taskConfigs = connector.taskConfigs(2); 35 | assertFalse(taskConfigs.isEmpty(), "zero task configs provided"); 36 | for (Map taskConfig : taskConfigs) { 37 | assertEquals(settings, taskConfig); 38 | } 39 | } 40 | 41 | @Test 42 | public void shouldNotHaveNullConfigDef() { 43 | // ConfigDef objects don't have an overridden equals() method; just make sure it's non-null 44 | assertNotNull(connector.config()); 45 | } 46 | 47 | @Test 48 | public void shouldReturnConnectorType() { 49 | assertTrue(SinkConnector.class.isAssignableFrom(connector.getClass())); 50 | } 51 | 52 | @Test 53 | public void shouldReturnSinkTask() { 54 | assertEquals(OpenSearchSinkTask.class, connector.taskClass()); 55 | } 56 | 57 | @Test 58 | public void shouldStartAndStop() { 59 | connector.start(settings); 60 | connector.stop(); 61 | } 62 | 63 | @Test 64 | public void testVersion() { 65 | assertNotNull(connector.version()); 66 | assertEquals("DEV", connector.version()); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /lib/src/test/java/com/dmathieu/kafka/opensearch/OpenSearchSinkTaskTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch; 17 | 18 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.BEHAVIOR_ON_NULL_VALUES_CONFIG; 19 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.DATA_STREAM_DATASET_CONFIG; 20 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.DATA_STREAM_TYPE_CONFIG; 21 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.DROP_INVALID_MESSAGE_CONFIG; 22 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.IGNORE_KEY_CONFIG; 23 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.IGNORE_SCHEMA_CONFIG; 24 | import static org.mockito.ArgumentMatchers.any; 25 | import static org.mockito.ArgumentMatchers.eq; 26 | import static org.mockito.Mockito.doThrow; 27 | import static org.mockito.Mockito.mock; 28 | import static org.mockito.Mockito.never; 29 | import static org.mockito.Mockito.times; 30 | import static org.mockito.Mockito.verify; 31 | import static org.mockito.Mockito.when; 32 | 33 | import com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.BehaviorOnNullValues; 34 | import com.dmathieu.kafka.opensearch.OffsetTracker.OffsetState; 35 | import org.apache.kafka.common.record.TimestampType; 36 | import org.apache.kafka.connect.data.Schema; 37 | import org.apache.kafka.connect.data.SchemaBuilder; 38 | import org.apache.kafka.connect.data.Struct; 39 | import org.apache.kafka.connect.errors.ConnectException; 40 | import org.apache.kafka.connect.errors.DataException; 41 | import org.apache.kafka.connect.sink.ErrantRecordReporter; 42 | import org.apache.kafka.connect.sink.SinkRecord; 43 | import org.apache.kafka.connect.sink.SinkTaskContext; 44 | import org.elasticsearch.action.DocWriteRequest; 45 | 46 | import org.junit.jupiter.api.Test; 47 | import org.junit.jupiter.api.BeforeEach; 48 | import static org.junit.jupiter.api.Assertions.*; 49 | 50 | import java.util.HashMap; 51 | import java.util.Map; 52 | import java.util.Collections; 53 | 54 | public class OpenSearchSinkTaskTest { 55 | 56 | protected static final String TOPIC = "topic"; 57 | 58 | protected OpenSearchClient client; 59 | private OpenSearchSinkTask task; 60 | private Map props; 61 | private SinkTaskContext context; 62 | 63 | private void setUpTask() { 64 | task = new OpenSearchSinkTask(); 65 | task.initialize(context); 66 | task.start(props, client); 67 | } 68 | 69 | @BeforeEach 70 | public void setUp() { 71 | props = OpenSearchSinkConnectorConfigTest.addNecessaryProps(new HashMap<>()); 72 | props.put(IGNORE_KEY_CONFIG, "true"); 73 | 74 | client = mock(OpenSearchClient.class); 75 | context = mock(SinkTaskContext.class); 76 | 77 | setUpTask(); 78 | } 79 | 80 | @Test 81 | public void testPutSkipNullRecords() { 82 | props.put(BEHAVIOR_ON_NULL_VALUES_CONFIG, BehaviorOnNullValues.IGNORE.name()); 83 | setUpTask(); 84 | 85 | // skip null 86 | SinkRecord nullRecord = record(true, true, 0); 87 | task.put(Collections.singletonList(nullRecord)); 88 | verify(client, never()).index(eq(nullRecord), any(DocWriteRequest.class), any(OffsetState.class)); 89 | 90 | // don't skip non-null 91 | SinkRecord notNullRecord = record(true, false,1); 92 | task.put(Collections.singletonList(notNullRecord)); 93 | verify(client, times(1)).index(eq(notNullRecord), any(DocWriteRequest.class), any(OffsetState.class)); 94 | } 95 | 96 | @Test 97 | public void testReportNullRecords() { 98 | ErrantRecordReporter mockReporter = mock(ErrantRecordReporter.class); 99 | when(context.errantRecordReporter()).thenReturn(mockReporter); 100 | 101 | props.put(BEHAVIOR_ON_NULL_VALUES_CONFIG, BehaviorOnNullValues.IGNORE.name()); 102 | setUpTask(); 103 | 104 | // report null 105 | SinkRecord nullRecord = record(true, true, 0); 106 | task.put(Collections.singletonList(nullRecord)); 107 | verify(client, never()).index(eq(nullRecord), any(), any()); 108 | verify(mockReporter, times(1)).report(eq(nullRecord), any(ConnectException.class)); 109 | 110 | // don't report 111 | SinkRecord notNullRecord = record(true, false,1); 112 | task.put(Collections.singletonList(notNullRecord)); 113 | verify(client, times(1)).index(eq(notNullRecord), any(), any()); 114 | verify(mockReporter, never()).report(eq(notNullRecord), any(ConnectException.class)); 115 | } 116 | 117 | @Test 118 | public void testPutFailNullRecords() { 119 | props.put(BEHAVIOR_ON_NULL_VALUES_CONFIG, BehaviorOnNullValues.FAIL.name()); 120 | setUpTask(); 121 | 122 | assertThrows(DataException.class, () -> { 123 | // fail null 124 | SinkRecord nullRecord = record(true, true, 0); 125 | task.put(Collections.singletonList(nullRecord)); 126 | }); 127 | } 128 | 129 | @Test 130 | public void testCreateIndex() { 131 | task.put(Collections.singletonList(record())); 132 | verify(client, times(1)).createIndexOrDataStream(eq(TOPIC)); 133 | } 134 | 135 | @Test 136 | public void testCreateUpperCaseIndex() { 137 | task.put(Collections.singletonList(record())); 138 | verify(client, times(1)).createIndexOrDataStream(eq(TOPIC.toLowerCase())); 139 | } 140 | 141 | @Test 142 | public void testDoNotCreateCachedIndex() { 143 | task.put(Collections.singletonList(record())); 144 | verify(client, times(1)).createIndexOrDataStream(eq(TOPIC)); 145 | 146 | task.put(Collections.singletonList(record())); 147 | verify(client, times(1)).createIndexOrDataStream(eq(TOPIC)); 148 | } 149 | 150 | @Test 151 | public void testIgnoreSchema() { 152 | props.put(IGNORE_SCHEMA_CONFIG, "true"); 153 | setUpTask(); 154 | 155 | SinkRecord record = record(); 156 | task.put(Collections.singletonList(record)); 157 | verify(client, never()).hasMapping(eq(TOPIC)); 158 | verify(client, never()).createMapping(eq(TOPIC), eq(record.valueSchema())); 159 | } 160 | 161 | @Test 162 | public void testCheckMapping() { 163 | when(client.hasMapping(TOPIC)).thenReturn(true); 164 | 165 | SinkRecord record = record(); 166 | task.put(Collections.singletonList(record)); 167 | verify(client, times(1)).hasMapping(eq(TOPIC)); 168 | verify(client, never()).createMapping(eq(TOPIC), eq(record.valueSchema())); 169 | } 170 | 171 | @Test 172 | public void testAddMapping() { 173 | SinkRecord record = record(); 174 | task.put(Collections.singletonList(record)); 175 | verify(client, times(1)).hasMapping(eq(TOPIC)); 176 | verify(client, times(1)).createMapping(eq(TOPIC), eq(record.valueSchema())); 177 | } 178 | 179 | @Test 180 | public void testDoNotAddCachedMapping() { 181 | SinkRecord record = record(); 182 | task.put(Collections.singletonList(record)); 183 | verify(client, times(1)).hasMapping(eq(TOPIC)); 184 | verify(client, times(1)).createMapping(eq(TOPIC), eq(record.valueSchema())); 185 | 186 | task.put(Collections.singletonList(record)); 187 | verify(client, times(1)).hasMapping(eq(TOPIC)); 188 | verify(client, times(1)).createMapping(eq(TOPIC), eq(record.valueSchema())); 189 | } 190 | 191 | @Test 192 | public void testPut() { 193 | SinkRecord record = record(); 194 | task.put(Collections.singletonList(record)); 195 | verify(client, times(1)).index(eq(record), any(), any()); 196 | } 197 | 198 | @Test 199 | public void testPutSkipInvalidRecord() { 200 | props.put(DROP_INVALID_MESSAGE_CONFIG, "true"); 201 | props.put(IGNORE_KEY_CONFIG, "false"); 202 | setUpTask(); 203 | 204 | // skip invalid 205 | SinkRecord invalidRecord = record(true, 0); 206 | task.put(Collections.singletonList(invalidRecord)); 207 | verify(client, never()).index(eq(invalidRecord), any(), any()); 208 | 209 | // don't skip valid 210 | SinkRecord validRecord = record(false, 1); 211 | task.put(Collections.singletonList(validRecord)); 212 | verify(client, times(1)).index(eq(validRecord), any(), any()); 213 | } 214 | 215 | @Test 216 | public void testPutReportInvalidRecord() { 217 | ErrantRecordReporter mockReporter = mock(ErrantRecordReporter.class); 218 | when(context.errantRecordReporter()).thenReturn(mockReporter); 219 | 220 | props.put(DROP_INVALID_MESSAGE_CONFIG, "true"); 221 | props.put(IGNORE_KEY_CONFIG, "false"); 222 | setUpTask(); 223 | 224 | // report invalid 225 | SinkRecord invalidRecord = record(true, 0); 226 | task.put(Collections.singletonList(invalidRecord)); 227 | verify(client, never()).index(eq(invalidRecord), any(), any()); 228 | verify(mockReporter, times(1)).report(eq(invalidRecord), any(DataException.class)); 229 | 230 | // don't report valid 231 | SinkRecord validRecord = record(false, 1); 232 | task.put(Collections.singletonList(validRecord)); 233 | verify(client, times(1)).index(eq(validRecord), any(), any()); 234 | verify(mockReporter, never()).report(eq(validRecord), any(DataException.class)); 235 | } 236 | 237 | @Test 238 | public void testPutFailsOnInvalidRecord() { 239 | props.put(DROP_INVALID_MESSAGE_CONFIG, "false"); 240 | props.put(IGNORE_KEY_CONFIG, "false"); 241 | setUpTask(); 242 | 243 | assertThrows(DataException.class, () -> { 244 | SinkRecord invalidRecord = record(); 245 | task.put(Collections.singletonList(invalidRecord)); 246 | }); 247 | } 248 | 249 | @Test 250 | public void testFlush() { 251 | setUpTask(); 252 | task.preCommit(null); 253 | verify(client, times(1)).flush(); 254 | } 255 | 256 | @Test 257 | public void testFlushDoesNotThrow() { 258 | setUpTask(); 259 | doThrow(new IllegalStateException("already closed")).when(client).flush(); 260 | 261 | // should not throw 262 | task.preCommit(null); 263 | verify(client, times(1)).flush(); 264 | } 265 | 266 | @Test 267 | public void testStartAndStop() { 268 | task = new OpenSearchSinkTask(); 269 | task.initialize(context); 270 | task.start(props); 271 | task.stop(); 272 | } 273 | 274 | @Test 275 | public void testVersion() { 276 | setUpTask(); 277 | assertEquals("DEV", task.version()); 278 | } 279 | 280 | @Test 281 | public void testConvertTopicToDataStreamAllowsDashes() { 282 | String type = "logs"; 283 | String dataset = "a_valid_dataset"; 284 | props.put(DATA_STREAM_TYPE_CONFIG, type); 285 | props.put(DATA_STREAM_DATASET_CONFIG, dataset); 286 | setUpTask(); 287 | 288 | String topic = "-dash"; 289 | task.put(Collections.singletonList(record(topic, true, false, 0))); 290 | String indexName = dataStreamName(type, dataset, topic); 291 | verify(client, times(1)).createIndexOrDataStream(eq(indexName)); 292 | } 293 | 294 | @Test 295 | public void testConvertTopicToDataStreamAllowUnderscores() { 296 | String type = "logs"; 297 | String dataset = "a_valid_dataset"; 298 | props.put(DATA_STREAM_TYPE_CONFIG, type); 299 | props.put(DATA_STREAM_DATASET_CONFIG, dataset); 300 | setUpTask(); 301 | 302 | String topic = "_underscore"; 303 | task.put(Collections.singletonList(record(topic, true, false, 0))); 304 | String indexName = dataStreamName(type, dataset, topic); 305 | verify(client, times(1)).createIndexOrDataStream(eq(indexName)); 306 | } 307 | 308 | @Test 309 | public void testConvertTopicToDataStreamTooLong() { 310 | String type = "logs"; 311 | String dataset = "a_valid_dataset"; 312 | props.put(DATA_STREAM_TYPE_CONFIG, type); 313 | props.put(DATA_STREAM_DATASET_CONFIG, dataset); 314 | setUpTask(); 315 | 316 | String topic = String.format("%0101d", 1); 317 | task.put(Collections.singletonList(record(topic, true, false, 0))); 318 | String indexName = dataStreamName(type, dataset, topic.substring(0, 100)); 319 | verify(client, times(1)).createIndexOrDataStream(eq(indexName)); 320 | } 321 | 322 | @Test 323 | public void testConvertTopicToDataStreamUpperCase() { 324 | String type = "logs"; 325 | String dataset = "a_valid_dataset"; 326 | props.put(DATA_STREAM_TYPE_CONFIG, type); 327 | props.put(DATA_STREAM_DATASET_CONFIG, dataset); 328 | setUpTask(); 329 | 330 | String topic = "UPPERCASE"; 331 | task.put(Collections.singletonList(record(topic, true, false, 0))); 332 | String indexName = dataStreamName(type, dataset, topic.toLowerCase()); 333 | verify(client, times(1)).createIndexOrDataStream(eq(indexName)); 334 | } 335 | 336 | @Test 337 | public void testConvertTopicToIndexName() { 338 | setUpTask(); 339 | 340 | String upperCaseTopic = "UPPERCASE"; 341 | SinkRecord record = record(upperCaseTopic, true, false, 0); 342 | task.put(Collections.singletonList(record)); 343 | verify(client, times(1)).createIndexOrDataStream(eq(upperCaseTopic.toLowerCase())); 344 | 345 | String tooLongTopic = String.format("%0256d", 1); 346 | record = record(tooLongTopic, true, false, 0); 347 | task.put(Collections.singletonList(record)); 348 | verify(client, times(1)).createIndexOrDataStream(eq(tooLongTopic.substring(0, 255))); 349 | 350 | String startsWithDash = "-dash"; 351 | record = record(startsWithDash, true, false, 0); 352 | task.put(Collections.singletonList(record)); 353 | verify(client, times(1)).createIndexOrDataStream(eq("dash")); 354 | 355 | String startsWithUnderscore = "_underscore"; 356 | record = record(startsWithUnderscore, true, false, 0); 357 | task.put(Collections.singletonList(record)); 358 | verify(client, times(1)).createIndexOrDataStream(eq("underscore")); 359 | 360 | String dot = "."; 361 | record = record(dot, true, false, 0); 362 | task.put(Collections.singletonList(record)); 363 | verify(client, times(1)).createIndexOrDataStream(eq("dot")); 364 | 365 | String dots = ".."; 366 | record = record(dots, true, false, 0); 367 | task.put(Collections.singletonList(record)); 368 | verify(client, times(1)).createIndexOrDataStream(eq("dotdot")); 369 | } 370 | 371 | @Test 372 | public void testShouldNotThrowIfReporterDoesNotExist() { 373 | when(context.errantRecordReporter()) 374 | .thenThrow(new NoSuchMethodError("what are you doing")) 375 | .thenThrow(new NoClassDefFoundError("i no exist")); 376 | 377 | // call start twice for both exceptions 378 | setUpTask(); 379 | setUpTask(); 380 | } 381 | 382 | private String dataStreamName(String type, String dataset, String namespace) { 383 | return String.format("%s-%s-%s", type, dataset, namespace); 384 | } 385 | 386 | private SinkRecord record() { 387 | return record(true, false,0); 388 | } 389 | 390 | private SinkRecord record(boolean nullKey, long offset) { 391 | return record(nullKey, false, offset); 392 | } 393 | 394 | private SinkRecord record(boolean nullKey, boolean nullValue, long offset) { 395 | return record(TOPIC, nullKey, nullValue, offset); 396 | } 397 | 398 | private SinkRecord record(String topic, boolean nullKey, boolean nullValue, long offset) { 399 | Schema schema = SchemaBuilder.struct().name("struct") 400 | .field("user", Schema.STRING_SCHEMA) 401 | .field("message", Schema.STRING_SCHEMA) 402 | .build(); 403 | 404 | Struct struct = new Struct(schema); 405 | struct.put("user", "Liquan"); 406 | struct.put("message", "trying out Elastic Search."); 407 | 408 | return new SinkRecord( 409 | topic, 410 | 1, 411 | Schema.STRING_SCHEMA, 412 | nullKey ? null : "key", 413 | schema, 414 | nullValue ? null : struct, 415 | offset, 416 | System.currentTimeMillis(), 417 | TimestampType.CREATE_TIME 418 | ); 419 | } 420 | } 421 | -------------------------------------------------------------------------------- /lib/src/test/java/com/dmathieu/kafka/opensearch/PartitionPauserTest.java: -------------------------------------------------------------------------------- 1 | package com.dmathieu.kafka.opensearch; 2 | 3 | import com.google.common.collect.ImmutableSet; 4 | import com.dmathieu.kafka.opensearch.OpenSearchSinkTask.PartitionPauser; 5 | import org.apache.kafka.common.TopicPartition; 6 | import org.apache.kafka.connect.sink.SinkTaskContext; 7 | 8 | import org.junit.jupiter.api.Test; 9 | 10 | import java.util.concurrent.atomic.AtomicBoolean; 11 | 12 | import static org.mockito.Mockito.clearInvocations; 13 | import static org.mockito.Mockito.mock; 14 | import static org.mockito.Mockito.verify; 15 | import static org.mockito.Mockito.verifyNoMoreInteractions; 16 | import static org.mockito.Mockito.when; 17 | 18 | public class PartitionPauserTest { 19 | 20 | @Test 21 | public void partitionPauserTest() { 22 | SinkTaskContext context = mock(SinkTaskContext.class); 23 | AtomicBoolean pauseCondition = new AtomicBoolean(); 24 | AtomicBoolean resumeCondition = new AtomicBoolean(); 25 | PartitionPauser partitionPauser = new PartitionPauser(context, 26 | pauseCondition::get, 27 | resumeCondition::get); 28 | 29 | TopicPartition tp = new TopicPartition("test-topic", 0); 30 | when(context.assignment()).thenReturn(ImmutableSet.of(tp)); 31 | 32 | partitionPauser.maybePausePartitions(); 33 | verifyNoMoreInteractions(context); 34 | 35 | partitionPauser.maybeResumePartitions(); 36 | verifyNoMoreInteractions(context); 37 | 38 | pauseCondition.set(true); 39 | partitionPauser.maybePausePartitions(); 40 | verify(context).assignment(); 41 | verify(context).pause(tp); 42 | verify(context).timeout(100); 43 | verifyNoMoreInteractions(context); 44 | 45 | clearInvocations(context); 46 | partitionPauser.maybePausePartitions(); 47 | verifyNoMoreInteractions(context); 48 | 49 | partitionPauser.maybeResumePartitions(); 50 | verify(context).timeout(100); 51 | verifyNoMoreInteractions(context); 52 | 53 | resumeCondition.set(true); 54 | clearInvocations(context); 55 | partitionPauser.maybeResumePartitions(); 56 | verify(context).assignment(); 57 | verify(context).resume(tp); 58 | verifyNoMoreInteractions(context); 59 | 60 | clearInvocations(context); 61 | partitionPauser.maybeResumePartitions(); 62 | verifyNoMoreInteractions(context); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /lib/src/test/java/com/dmathieu/kafka/opensearch/RetryUtilTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | package com.dmathieu.kafka.opensearch; 16 | 17 | import java.io.IOException; 18 | import org.apache.kafka.common.utils.MockTime; 19 | import org.apache.kafka.connect.errors.ConnectException; 20 | 21 | 22 | import org.junit.jupiter.api.BeforeEach; 23 | import org.junit.jupiter.api.Test; 24 | import static org.junit.jupiter.api.Assertions.*; 25 | 26 | import static org.mockito.ArgumentMatchers.anyLong; 27 | import static org.mockito.Mockito.spy; 28 | import static org.mockito.Mockito.times; 29 | import static org.mockito.Mockito.verify; 30 | 31 | public class RetryUtilTest { 32 | 33 | private int timesThrown; 34 | 35 | @BeforeEach 36 | public void setup() { 37 | timesThrown = 0; 38 | } 39 | 40 | @Test 41 | public void computeRetryBackoffForNegativeAttempts() { 42 | assertComputeRetryInRange(0, 10L); 43 | assertEquals(10L, RetryUtil.computeRandomRetryWaitTimeInMillis(-1, 10L)); 44 | } 45 | 46 | @Test 47 | public void computeRetryBackoffForValidRanges() { 48 | assertComputeRetryInRange(10, 10L); 49 | assertComputeRetryInRange(10, 100L); 50 | assertComputeRetryInRange(10, 1000L); 51 | assertComputeRetryInRange(100, 1000L); 52 | } 53 | 54 | @Test 55 | public void computeRetryBackoffForNegativeRetryTimes() { 56 | assertComputeRetryInRange(1, -100L); 57 | assertComputeRetryInRange(10, -100L); 58 | assertComputeRetryInRange(100, -100L); 59 | } 60 | 61 | @Test 62 | public void computeNonRandomRetryTimes() { 63 | assertEquals(100L, RetryUtil.computeRetryWaitTimeInMillis(0, 100L)); 64 | assertEquals(200L, RetryUtil.computeRetryWaitTimeInMillis(1, 100L)); 65 | assertEquals(400L, RetryUtil.computeRetryWaitTimeInMillis(2, 100L)); 66 | assertEquals(800L, RetryUtil.computeRetryWaitTimeInMillis(3, 100L)); 67 | assertEquals(1600L, RetryUtil.computeRetryWaitTimeInMillis(4, 100L)); 68 | assertEquals(3200L, RetryUtil.computeRetryWaitTimeInMillis(5, 100L)); 69 | } 70 | 71 | @Test 72 | public void testCallWithRetriesNoRetries() throws Exception { 73 | MockTime mockClock = new MockTime(); 74 | long expectedTime = mockClock.milliseconds(); 75 | 76 | assertTrue(RetryUtil.callWithRetries("test", () -> testFunction(0), 3, 100, mockClock)); 77 | assertEquals(expectedTime, mockClock.milliseconds()); 78 | } 79 | 80 | @Test 81 | public void testCallWithRetriesSomeRetries() throws Exception { 82 | MockTime mockClock = spy(new MockTime()); 83 | 84 | assertTrue(RetryUtil.callWithRetries("test", () -> testFunction(2), 3, 100, mockClock)); 85 | verify(mockClock, times(2)).sleep(anyLong()); 86 | } 87 | 88 | @Test 89 | public void testCallWithRetriesExhaustedRetries() throws Exception { 90 | MockTime mockClock = new MockTime(); 91 | 92 | assertThrows(ConnectException.class, () -> { 93 | assertTrue(RetryUtil.callWithRetries("test", () -> testFunction(4), 3, 100, mockClock)); 94 | verify(mockClock, times(3)).sleep(anyLong()); 95 | }); 96 | } 97 | 98 | private boolean testFunction(int timesToThrow) throws IOException { 99 | if (timesThrown < timesToThrow) { 100 | timesThrown++; 101 | throw new IOException("oh no i iz borke, plz retry"); 102 | } 103 | 104 | return true; 105 | } 106 | 107 | protected void assertComputeRetryInRange(int retryAttempts, long retryBackoffMs) { 108 | for (int i = 0; i != 20; ++i) { 109 | for (int retries = 0; retries <= retryAttempts; ++retries) { 110 | long maxResult = RetryUtil.computeRetryWaitTimeInMillis(retries, retryBackoffMs); 111 | long result = RetryUtil.computeRandomRetryWaitTimeInMillis(retries, retryBackoffMs); 112 | if (retryBackoffMs < 0) { 113 | assertEquals(0, result); 114 | } else { 115 | assertTrue(result >= 0L); 116 | assertTrue(result <= maxResult); 117 | } 118 | } 119 | } 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /lib/src/test/java/com/dmathieu/kafka/opensearch/helper/NetworkErrorContainer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch.helper; 17 | 18 | import org.testcontainers.containers.BindMode; 19 | import org.testcontainers.containers.GenericContainer; 20 | import org.testcontainers.containers.wait.strategy.Wait; 21 | 22 | public class NetworkErrorContainer extends GenericContainer { 23 | 24 | private static final String DEFAULT_DOCKER_IMAGE = "gaiaadm/pumba:latest"; 25 | 26 | private static final String PUMBA_PAUSE_COMMAND = "--log-level info --interval 120s pause --duration 10s "; 27 | private static final String DOCKER_SOCK = "/var/run/docker.sock"; 28 | 29 | public NetworkErrorContainer(String containerToInterrupt) { 30 | this(DEFAULT_DOCKER_IMAGE, containerToInterrupt); 31 | } 32 | 33 | public NetworkErrorContainer( 34 | String dockerImageName, 35 | String containerToInterrupt 36 | ) { 37 | super(dockerImageName); 38 | 39 | setCommand(PUMBA_PAUSE_COMMAND + containerToInterrupt); 40 | addFileSystemBind(DOCKER_SOCK, DOCKER_SOCK, BindMode.READ_WRITE); 41 | setWaitStrategy(Wait.forLogMessage(".*pausing container.*", 1)); 42 | withLogConsumer(l -> System.out.print(l.getUtf8String())); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /lib/src/test/java/com/dmathieu/kafka/opensearch/helper/OpenSearchContainer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch.helper; 17 | 18 | import org.apache.kafka.common.config.SslConfigs; 19 | import org.elasticsearch.client.security.user.User; 20 | import org.elasticsearch.client.security.user.privileges.Role; 21 | import org.elasticsearch.client.security.user.privileges.IndicesPrivileges; 22 | import org.elasticsearch.client.security.user.privileges.Role.Builder; 23 | 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | import org.testcontainers.containers.ContainerLaunchException; 27 | import org.testcontainers.containers.output.OutputFrame; 28 | import org.testcontainers.containers.wait.strategy.Wait; 29 | import org.testcontainers.images.RemoteDockerImage; 30 | import org.testcontainers.images.builder.ImageFromDockerfile; 31 | import org.testcontainers.images.builder.dockerfile.DockerfileBuilder; 32 | import org.testcontainers.shaded.org.apache.commons.io.IOUtils; 33 | import org.testcontainers.utility.DockerImageName; 34 | 35 | import java.io.File; 36 | import java.io.FileOutputStream; 37 | import java.io.IOException; 38 | import java.io.InputStream; 39 | import java.net.InetAddress; 40 | import java.net.URI; 41 | import java.net.URISyntaxException; 42 | import java.time.Duration; 43 | import java.util.HashMap; 44 | import java.util.List; 45 | import java.util.Map; 46 | import java.util.ArrayList; 47 | import java.util.Collections; 48 | 49 | import com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig; 50 | import com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.SecurityProtocol; 51 | 52 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.CONNECTION_PASSWORD_CONFIG; 53 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.CONNECTION_URL_CONFIG; 54 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.CONNECTION_USERNAME_CONFIG; 55 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.SECURITY_PROTOCOL_CONFIG; 56 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.SSL_CONFIG_PREFIX; 57 | 58 | /** 59 | * A specialized TestContainer container for testing OpenSearch. 60 | */ 61 | public class OpenSearchContainer 62 | extends org.testcontainers.elasticsearch.ElasticsearchContainer { 63 | 64 | private static final Logger log = LoggerFactory.getLogger(OpenSearchContainer.class); 65 | 66 | /** 67 | * Default OpenSearch Docker image name. 68 | */ 69 | public static final String DEFAULT_DOCKER_IMAGE_NAME = 70 | "opensearchproject/opensearch"; 71 | 72 | /** 73 | * Default OpenSearch version. 74 | */ 75 | public static final String DEFAULT_OS_VERSION = "1.1.0"; 76 | 77 | /** 78 | * Default OpenSearch port. 79 | */ 80 | public static final int OPENSEARCH_DEFAULT_PORT = 9200; 81 | 82 | /** 83 | * Path to the OpenSearch configuration directory. 84 | */ 85 | public static String CONFIG_PATH = "/usr/share/opensearch/config"; 86 | 87 | /** 88 | * Create an {@link OpenSearchContainer} using the image name specified in the 89 | * {@code opensearch.image} system property or {@code OPENSEARCH_IMAGE} environment 90 | * variable, or defaulting to {@link #DEFAULT_DOCKER_IMAGE_NAME}, and the version specified in 91 | * the {@code opensearch.version} system property, {@code OPENSEARCH_VERSION} environment 92 | * variable, or defaulting to {@link #DEFAULT_ES_VERSION}. 93 | * 94 | * @return the unstarted container; never null 95 | */ 96 | public static OpenSearchContainer fromSystemProperties() { 97 | String imageName = getSystemOrEnvProperty( 98 | "opensearch.image", 99 | "OPENSEARCH_IMAGE", 100 | DEFAULT_DOCKER_IMAGE_NAME 101 | ); 102 | String version = getSystemOrEnvProperty( 103 | "opensearch.version", 104 | "OPENSEARCH_VERSION", 105 | DEFAULT_OS_VERSION 106 | ); 107 | return new OpenSearchContainer(imageName + ":" + version); 108 | } 109 | 110 | public static OpenSearchContainer withESVersion(String ESVersion) { 111 | String imageName = getSystemOrEnvProperty( 112 | "opensearch.image", 113 | "OPENSEARCH_IMAGE", 114 | DEFAULT_DOCKER_IMAGE_NAME 115 | ); 116 | return new OpenSearchContainer(imageName + ":" + ESVersion); 117 | } 118 | 119 | // Super user that has superuser role. Should not be used by connector 120 | private static final String OPENSEARCH_SUPERUSER_NAME = "admin"; 121 | private static final String OPENSEARCH_SUPERUSER_PASSWORD = "admin"; 122 | 123 | public static final String OPENSEARCH_USER_NAME = "admin"; 124 | public static final String OPENSEARCH_USER_PASSWORD = "admin"; 125 | private static final String OS_SINK_CONNECTOR_ROLE = "os_sink_connector_role"; 126 | 127 | private static final long TWO_GIGABYTES = 2L * 1024 * 1024 * 1024; 128 | 129 | private final String imageName; 130 | private String keytabPath; 131 | 132 | /** 133 | * Create an OpenSearch container with the given image name with version qualifier. 134 | * 135 | * @param imageName the image name 136 | */ 137 | public OpenSearchContainer(String imageName) { 138 | super(DockerImageName.parse(imageName).asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch")); 139 | this.imageName = imageName; 140 | withSharedMemorySize(TWO_GIGABYTES); 141 | withLogConsumer(this::containerLog); 142 | } 143 | 144 | @Override 145 | public void start() { 146 | super.start(); 147 | 148 | Map props = new HashMap<>(); 149 | props.put(CONNECTION_USERNAME_CONFIG, OPENSEARCH_SUPERUSER_NAME); 150 | props.put(CONNECTION_PASSWORD_CONFIG, OPENSEARCH_SUPERUSER_PASSWORD); 151 | props.put(CONNECTION_URL_CONFIG, this.getConnectionUrl(false)); 152 | props.put(SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SSL.name()); 153 | props.put(SSL_CONFIG_PREFIX + SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, ""); 154 | 155 | OpenSearchHelperClient helperClient = getHelperClient(props); 156 | //createUsersAndRoles(helperClient); 157 | } 158 | 159 | private void createUsersAndRoles(OpenSearchHelperClient helperClient ) { 160 | Map users = getUsers(); 161 | List roles = getRoles(); 162 | 163 | try { 164 | for (Role role: roles) { 165 | helperClient.createRole(role); 166 | } 167 | for (Map.Entry userToPassword: users.entrySet()) { 168 | helperClient.createUser(userToPassword); 169 | } 170 | } catch (IOException e) { 171 | throw new ContainerLaunchException("Container startup failed", e); 172 | } 173 | } 174 | 175 | public OpenSearchContainer withKerberosEnabled(String keytab) { 176 | enableKerberos(keytab); 177 | return this; 178 | } 179 | 180 | /** 181 | * Set whether the OpenSearch instance should use Kerberos. 182 | * 183 | *

This can only be called before the container is started. 184 | * 185 | * @param keytab non-null keytab path if Kerberos is enabled 186 | */ 187 | public void enableKerberos(String keytab) { 188 | if (isCreated()) { 189 | throw new IllegalStateException( 190 | "enableKerberos can only be used before the container is created." 191 | ); 192 | } 193 | keytabPath = keytab; 194 | } 195 | 196 | /** 197 | * Get whether the OpenSearch instance is configured to use Kerberos. 198 | * 199 | * @return true if Kerberos is enabled, or false otherwise 200 | */ 201 | public boolean isKerberosEnabled() { 202 | return keytabPath != null; 203 | } 204 | 205 | private String getFullResourcePath(String resourceName) { 206 | if (isKerberosEnabled()) { 207 | return "/kerberos/" + resourceName; 208 | } 209 | return "/default/" + resourceName; 210 | } 211 | 212 | @Override 213 | protected void configure() { 214 | super.configure(); 215 | 216 | waitingFor( 217 | Wait.forLogMessage(".*Node '.*' initialized.*", 1) 218 | .withStartupTimeout(Duration.ofMinutes(5)) 219 | ); 220 | 221 | ImageFromDockerfile image = new ImageFromDockerfile() 222 | .withFileFromClasspath("opensearch.yml", getFullResourcePath("opensearch.yml")) 223 | .withFileFromClasspath("instances.yml", getFullResourcePath("instances.yml")) 224 | .withDockerfileFromBuilder(this::buildImage); 225 | 226 | if (isKerberosEnabled()) { 227 | log.info("Creating Kerberized OpenSearch image."); 228 | image.withFileFromFile("es.keytab", new File(keytabPath)); 229 | } else { 230 | log.info("Using basic authentication"); 231 | withEnv("OPENSEARCH_USERNAME", OPENSEARCH_SUPERUSER_NAME); 232 | withEnv("OPENSEARCH_PASSWORD", OPENSEARCH_SUPERUSER_PASSWORD); 233 | } 234 | 235 | log.info("Extending Docker image to generate certs and enable SSL"); 236 | withEnv("OPENSEARCH_PASSWORD", OPENSEARCH_SUPERUSER_PASSWORD); 237 | withEnv("IP_ADDRESS", hostMachineIpAddress()); 238 | 239 | image 240 | // Copy the script to generate the certs and start OpenSearch 241 | .withFileFromClasspath("start-opensearch.sh", 242 | getFullResourcePath("start-opensearch.sh")); 243 | setImage(image); 244 | } 245 | 246 | private void buildImage(DockerfileBuilder builder) { 247 | builder 248 | .from(imageName) 249 | .copy("opensearch.yml", CONFIG_PATH + "/opensearch.yml"); 250 | 251 | log.info("Building OpenSearch image with SSL configuration"); 252 | builder 253 | .copy("instances.yml", CONFIG_PATH + "/instances.yml") 254 | .copy("start-opensearch.sh", CONFIG_PATH + "/start-opensearch.sh") 255 | 256 | .user("root") 257 | .run("chown opensearch:opensearch " + CONFIG_PATH + "/opensearch.yml") 258 | .run("chown opensearch:opensearch " + CONFIG_PATH + "/instances.yml") 259 | .run("chown opensearch:opensearch " + CONFIG_PATH + "/start-opensearch.sh") 260 | .user("opensearch") 261 | 262 | .entryPoint(CONFIG_PATH + "/start-opensearch.sh"); 263 | 264 | if (isKerberosEnabled()) { 265 | log.info("Building OpenSearch image with Kerberos configuration."); 266 | builder.copy("es.keytab", CONFIG_PATH + "/es.keytab"); 267 | } 268 | } 269 | 270 | public String hostMachineIpAddress() { 271 | String dockerHost = System.getenv("DOCKER_HOST"); 272 | if (dockerHost != null && !dockerHost.trim().isEmpty()) { 273 | try { 274 | URI url = new URI(dockerHost); 275 | dockerHost = url.getHost(); 276 | log.info("Including DOCKER_HOST address {} in OpenSearch certs", dockerHost); 277 | return dockerHost; 278 | } catch (URISyntaxException e) { 279 | log.info("DOCKER_HOST={} could not be parsed into a URL: {}", dockerHost, e.getMessage(), e); 280 | } 281 | } 282 | try { 283 | String hostAddress = InetAddress.getLocalHost().getHostAddress(); 284 | log.info("Including test machine address {} in OpenSearch certs", hostAddress); 285 | return hostAddress; 286 | } catch (IOException e) { 287 | return ""; 288 | } 289 | } 290 | 291 | /** 292 | * @see OpenSearchContainer#getConnectionUrl(boolean) 293 | */ 294 | public String getConnectionUrl() { 295 | return getConnectionUrl(true); 296 | } 297 | 298 | /** 299 | * Get the OpenSearch connection URL. 300 | * 301 | *

This can only be called once the container is started. 302 | * 303 | * @param useContainerIpAddress use container IP if true, host machine's IP otherwise 304 | * 305 | * @return the connection URL; never null 306 | */ 307 | public String getConnectionUrl(boolean useContainerIpAddress) { 308 | return String.format( 309 | "https://%s:%d", 310 | useContainerIpAddress ? getContainerIpAddress() : hostMachineIpAddress(), 311 | getMappedPort(OPENSEARCH_DEFAULT_PORT) 312 | ); 313 | } 314 | 315 | protected String generateTemporaryFile(InputStream inputStream) throws IOException { 316 | File file = File.createTempFile("OpenSearchTestContainer", ""); 317 | try (FileOutputStream outputStream = new FileOutputStream(file)) { 318 | IOUtils.copy(inputStream, outputStream); 319 | } 320 | return file.getAbsolutePath(); 321 | } 322 | 323 | private static String getSystemOrEnvProperty(String sysPropName, String envPropName, String defaultValue) { 324 | String propertyValue = System.getProperty(sysPropName); 325 | if (null == propertyValue) { 326 | propertyValue = System.getenv(envPropName); 327 | if (null == propertyValue) { 328 | propertyValue = defaultValue; 329 | } 330 | } 331 | return propertyValue; 332 | } 333 | 334 | /** 335 | * Capture the container log by writing the container's standard output 336 | * to {@link System#out} (in yellow) and standard error to {@link System#err} (in red). 337 | * 338 | * @param logMessage the container log message 339 | */ 340 | protected void containerLog(OutputFrame logMessage) { 341 | switch (logMessage.getType()) { 342 | case STDOUT: 343 | // Normal output in yellow 344 | System.out.print((char)27 + "[33m" + logMessage.getUtf8String()); 345 | System.out.print((char)27 + "[0m"); // reset 346 | break; 347 | case STDERR: 348 | // Error output in red 349 | System.err.print((char)27 + "[31m" + logMessage.getUtf8String()); 350 | System.out.print((char)27 + "[0m"); // reset 351 | break; 352 | case END: 353 | // End output in green 354 | System.err.print((char)27 + "[32m" + logMessage.getUtf8String()); 355 | System.out.print((char)27 + "[0m"); // reset 356 | break; 357 | default: 358 | break; 359 | } 360 | } 361 | 362 | public OpenSearchHelperClient getHelperClient(Map props) { 363 | // copy properties so that original properties are not affected 364 | Map superUserProps = new HashMap<>(props); 365 | superUserProps.put(CONNECTION_USERNAME_CONFIG, OPENSEARCH_SUPERUSER_NAME); 366 | superUserProps.put(CONNECTION_PASSWORD_CONFIG, OPENSEARCH_SUPERUSER_PASSWORD); 367 | OpenSearchSinkConnectorConfig config = new OpenSearchSinkConnectorConfig(superUserProps); 368 | OpenSearchHelperClient client = new OpenSearchHelperClient(props.get(CONNECTION_URL_CONFIG), config); 369 | return client; 370 | } 371 | 372 | protected static List getRoles() { 373 | List roles = new ArrayList<>(); 374 | roles.add(getMinimalPrivilegesRole()); 375 | return roles; 376 | } 377 | 378 | protected static Map getUsers() { 379 | Map users = new HashMap<>(); 380 | users.put(getMinimalPrivilegesUser(), getMinimalPrivilegesPassword()); 381 | return users; 382 | } 383 | 384 | private static Role getMinimalPrivilegesRole() { 385 | IndicesPrivileges.Builder indicesPrivilegesBuilder = IndicesPrivileges.builder(); 386 | IndicesPrivileges indicesPrivileges = indicesPrivilegesBuilder 387 | .indices("*") 388 | .privileges("create_index", "read", "write", "view_index_metadata") 389 | .build(); 390 | Builder builder = Role.builder(); 391 | builder = builder.clusterPrivileges("monitor"); 392 | Role role = builder 393 | .name(OS_SINK_CONNECTOR_ROLE) 394 | .indicesPrivileges(indicesPrivileges) 395 | .build(); 396 | return role; 397 | } 398 | 399 | private static User getMinimalPrivilegesUser() { 400 | return new User(OPENSEARCH_USER_NAME, 401 | Collections.singletonList(OS_SINK_CONNECTOR_ROLE)); 402 | } 403 | 404 | private static String getMinimalPrivilegesPassword() { 405 | return OPENSEARCH_USER_PASSWORD; 406 | } 407 | } 408 | -------------------------------------------------------------------------------- /lib/src/test/java/com/dmathieu/kafka/opensearch/helper/OpenSearchHelperClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch.helper; 17 | 18 | import org.apache.http.HttpHost; 19 | import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; 20 | import org.elasticsearch.action.search.SearchRequest; 21 | import org.elasticsearch.client.RequestOptions; 22 | import org.elasticsearch.client.RestClient; 23 | import org.elasticsearch.client.RestHighLevelClient; 24 | import org.elasticsearch.client.core.CountRequest; 25 | import org.elasticsearch.client.indices.CreateIndexRequest; 26 | import org.elasticsearch.client.indices.DataStream; 27 | import org.elasticsearch.client.indices.DeleteDataStreamRequest; 28 | import org.elasticsearch.client.indices.GetDataStreamRequest; 29 | import org.elasticsearch.client.indices.GetIndexRequest; 30 | import org.elasticsearch.client.indices.GetMappingsRequest; 31 | import org.elasticsearch.client.indices.GetMappingsResponse; 32 | import org.elasticsearch.client.security.PutRoleRequest; 33 | import org.elasticsearch.client.security.PutRoleResponse; 34 | import org.elasticsearch.client.security.PutUserRequest; 35 | import org.elasticsearch.client.security.PutUserResponse; 36 | import org.elasticsearch.client.security.RefreshPolicy; 37 | import org.elasticsearch.client.security.user.User; 38 | import org.elasticsearch.client.security.user.privileges.Role; 39 | import org.elasticsearch.cluster.metadata.MappingMetadata; 40 | import org.elasticsearch.common.xcontent.XContentType; 41 | import org.elasticsearch.search.SearchHits; 42 | import org.slf4j.Logger; 43 | import org.slf4j.LoggerFactory; 44 | 45 | import java.io.IOException; 46 | import java.util.List; 47 | import java.util.Map.Entry; 48 | 49 | import com.dmathieu.kafka.opensearch.ConfigCallbackHandler; 50 | import com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig; 51 | 52 | public class OpenSearchHelperClient { 53 | 54 | private static final Logger log = LoggerFactory.getLogger(OpenSearchHelperClient.class); 55 | 56 | private RestHighLevelClient client; 57 | 58 | public OpenSearchHelperClient(String url, OpenSearchSinkConnectorConfig config) { 59 | ConfigCallbackHandler configCallbackHandler = new ConfigCallbackHandler(config); 60 | this.client = new RestHighLevelClient( 61 | RestClient 62 | .builder(HttpHost.create(url)) 63 | .setHttpClientConfigCallback(configCallbackHandler) 64 | ); 65 | } 66 | 67 | public void deleteIndex(String index, boolean isDataStream) throws IOException { 68 | if (isDataStream) { 69 | DeleteDataStreamRequest request = new DeleteDataStreamRequest(index); 70 | client.indices().deleteDataStream(request, RequestOptions.DEFAULT); 71 | return; 72 | } 73 | DeleteIndexRequest request = new DeleteIndexRequest(index); 74 | client.indices().delete(request, RequestOptions.DEFAULT); 75 | } 76 | 77 | public DataStream getDataStream(String dataStream) throws IOException { 78 | GetDataStreamRequest request = new GetDataStreamRequest(dataStream); 79 | List datastreams = client.indices() 80 | .getDataStream(request, RequestOptions.DEFAULT) 81 | .getDataStreams(); 82 | return datastreams.size() == 0 ? null : datastreams.get(0); 83 | } 84 | 85 | public long getDocCount(String index) throws IOException { 86 | CountRequest request = new CountRequest(index); 87 | return client.count(request, RequestOptions.DEFAULT).getCount(); 88 | } 89 | 90 | public MappingMetadata getMapping(String index) throws IOException { 91 | GetMappingsRequest request = new GetMappingsRequest().indices(index); 92 | GetMappingsResponse response = client.indices().getMapping(request, RequestOptions.DEFAULT); 93 | return response.mappings().get(index); 94 | } 95 | 96 | public boolean indexExists(String index) throws IOException { 97 | GetIndexRequest request = new GetIndexRequest(index); 98 | return client.indices().exists(request, RequestOptions.DEFAULT); 99 | } 100 | 101 | public void createIndex(String index, String jsonMappings) throws IOException { 102 | CreateIndexRequest createIndexRequest = new CreateIndexRequest(index).mapping(jsonMappings, XContentType.JSON); 103 | client.indices().create(createIndexRequest, RequestOptions.DEFAULT); 104 | } 105 | 106 | public SearchHits search(String index) throws IOException { 107 | SearchRequest request = new SearchRequest(index); 108 | return client.search(request, RequestOptions.DEFAULT).getHits(); 109 | } 110 | 111 | public void createRole(Role role) throws IOException { 112 | PutRoleRequest putRoleRequest = new PutRoleRequest(role, RefreshPolicy.IMMEDIATE); 113 | PutRoleResponse putRoleResponse = client.security().putRole(putRoleRequest, RequestOptions.DEFAULT); 114 | if (!putRoleResponse.isCreated()) { 115 | throw new RuntimeException(String.format("Failed to create a role %s", role.getName())); 116 | } 117 | } 118 | 119 | public void createUser(Entry userToPassword) throws IOException { 120 | PutUserRequest putUserRequest = PutUserRequest.withPassword( 121 | userToPassword.getKey(), 122 | userToPassword.getValue().toCharArray(), 123 | true, 124 | RefreshPolicy.IMMEDIATE 125 | ); 126 | PutUserResponse putUserResponse = client.security().putUser(putUserRequest, RequestOptions.DEFAULT); 127 | if (!putUserResponse.isCreated()) { 128 | throw new RuntimeException(String.format("Failed to create a user %s", userToPassword.getKey().getUsername())); 129 | } 130 | } 131 | 132 | public void close() { 133 | try { 134 | client.close(); 135 | } catch (IOException e) { 136 | log.error("Error closing client.", e); 137 | } 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /lib/src/test/java/com/dmathieu/kafka/opensearch/integration/BaseConnectorIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch.integration; 17 | 18 | import java.util.Optional; 19 | import java.util.concurrent.TimeUnit; 20 | 21 | import org.apache.kafka.connect.runtime.AbstractStatus; 22 | import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo; 23 | import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster; 24 | import org.apache.kafka.test.IntegrationTest; 25 | import org.apache.kafka.test.TestUtils; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | import org.junit.jupiter.api.*; 30 | 31 | public abstract class BaseConnectorIT { 32 | 33 | private static final Logger log = LoggerFactory.getLogger(BaseConnectorIT.class); 34 | 35 | protected static final long CONSUME_MAX_DURATION_MS = TimeUnit.MINUTES.toMillis(1); 36 | protected static final long CONNECTOR_STARTUP_DURATION_MS = TimeUnit.MINUTES.toMillis(60); 37 | 38 | protected EmbeddedConnectCluster connect; 39 | 40 | protected void startConnect() { 41 | connect = new EmbeddedConnectCluster.Builder() 42 | .name("elasticsearch-it-connect-cluster") 43 | .build(); 44 | 45 | // start the clusters 46 | connect.start(); 47 | } 48 | 49 | protected void stopConnect() { 50 | // stop all Connect, Kafka and Zk threads. 51 | connect.stop(); 52 | } 53 | 54 | /** 55 | * Wait up to {@link #CONNECTOR_STARTUP_DURATION_MS maximum time limit} for the connector with the given 56 | * name to start the specified number of tasks. 57 | * 58 | * @param name the name of the connector 59 | * @param numTasks the minimum number of tasks that are expected 60 | * @return the time this method discovered the connector has started, in milliseconds past epoch 61 | * @throws InterruptedException if this was interrupted 62 | */ 63 | protected long waitForConnectorToStart(String name, int numTasks) throws InterruptedException { 64 | TestUtils.waitForCondition( 65 | () -> assertConnectorAndTasksRunning(name, numTasks).orElse(false), 66 | CONNECTOR_STARTUP_DURATION_MS, 67 | "Connector tasks did not start in time." 68 | ); 69 | return System.currentTimeMillis(); 70 | } 71 | 72 | /** 73 | * Confirm that a connector with an exact number of tasks is running. 74 | * 75 | * @param connectorName the connector 76 | * @param numTasks the minimum number of tasks 77 | * @return true if the connector and tasks are in RUNNING state; false otherwise 78 | */ 79 | protected Optional assertConnectorAndTasksRunning(String connectorName, int numTasks) { 80 | try { 81 | ConnectorStateInfo info = connect.connectorStatus(connectorName); 82 | boolean result = info != null 83 | && info.tasks().size() >= numTasks 84 | && info.connector().state().equals(AbstractStatus.State.RUNNING.toString()) 85 | && info.tasks().stream().allMatch(s -> s.state().equals(AbstractStatus.State.RUNNING.toString())); 86 | return Optional.of(result); 87 | } catch (Exception e) { 88 | log.error("Could not check connector state info."); 89 | return Optional.empty(); 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /lib/src/test/java/com/dmathieu/kafka/opensearch/integration/BlockingTransformer.java: -------------------------------------------------------------------------------- 1 | package com.dmathieu.kafka.opensearch.integration; 2 | 3 | import com.github.tomakehurst.wiremock.common.FileSource; 4 | import com.github.tomakehurst.wiremock.extension.Parameters; 5 | import com.github.tomakehurst.wiremock.extension.ResponseTransformer; 6 | import com.github.tomakehurst.wiremock.http.Request; 7 | import com.github.tomakehurst.wiremock.http.Response; 8 | import com.github.tomakehurst.wiremock.junit.WireMockRule; 9 | import org.apache.kafka.connect.errors.ConnectException; 10 | 11 | import java.util.concurrent.Semaphore; 12 | import java.util.concurrent.atomic.AtomicInteger; 13 | 14 | /** 15 | * Transformer that blocks all incoming requests until {@link #release(int)} is called 16 | * to fairly unblock a given number of requests. 17 | */ 18 | public class BlockingTransformer extends ResponseTransformer { 19 | 20 | private final Semaphore s = new Semaphore(0, true); 21 | private final AtomicInteger requestCount = new AtomicInteger(); 22 | 23 | public static final String NAME = "blockingTransformer"; 24 | 25 | @Override 26 | public Response transform(Request request, Response response, FileSource files, Parameters parameters) { 27 | try { 28 | s.acquire(); 29 | } catch (InterruptedException e) { 30 | throw new ConnectException(e); 31 | } finally { 32 | s.release(); 33 | } 34 | requestCount.incrementAndGet(); 35 | return response; 36 | } 37 | 38 | @Override 39 | public String getName() { 40 | return NAME; 41 | } 42 | 43 | public void release(int permits) { 44 | s.release(permits); 45 | } 46 | 47 | /** 48 | * How many requests are currently blocked 49 | */ 50 | public int queueLength() { 51 | return s.getQueueLength(); 52 | } 53 | 54 | /** 55 | * How many requests have been processed 56 | */ 57 | public int requestCount() { 58 | return requestCount.get(); 59 | } 60 | 61 | @Override 62 | public boolean applyGlobally() { 63 | return false; 64 | } 65 | 66 | public static BlockingTransformer getInstance(WireMockRule wireMockRule) { 67 | return wireMockRule.getOptions() 68 | .extensionsOfType(BlockingTransformer.class) 69 | .get(NAME); 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /lib/src/test/java/com/dmathieu/kafka/opensearch/integration/OpenSearchConnectorBaseIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch.integration; 17 | 18 | import org.apache.kafka.connect.json.JsonConverter; 19 | import org.apache.kafka.connect.storage.StringConverter; 20 | import org.apache.kafka.test.TestUtils; 21 | import org.elasticsearch.ElasticsearchStatusException; 22 | import org.elasticsearch.client.security.user.User; 23 | import org.elasticsearch.client.security.user.privileges.IndicesPrivileges; 24 | import org.elasticsearch.client.security.user.privileges.Role; 25 | import org.elasticsearch.client.security.user.privileges.Role.Builder; 26 | import org.elasticsearch.search.SearchHit; 27 | 28 | import java.io.IOException; 29 | import java.util.ArrayList; 30 | import java.net.ConnectException; 31 | import java.util.Collections; 32 | import java.util.HashMap; 33 | import java.util.List; 34 | import java.util.Map; 35 | 36 | import com.dmathieu.kafka.opensearch.OpenSearchSinkConnector; 37 | import com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig; 38 | import com.dmathieu.kafka.opensearch.helper.OpenSearchContainer; 39 | import com.dmathieu.kafka.opensearch.helper.OpenSearchHelperClient; 40 | 41 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.CONNECTION_PASSWORD_CONFIG; 42 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.CONNECTION_URL_CONFIG; 43 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.CONNECTION_USERNAME_CONFIG; 44 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.IGNORE_KEY_CONFIG; 45 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.IGNORE_SCHEMA_CONFIG; 46 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.DATA_STREAM_DATASET_CONFIG; 47 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.DATA_STREAM_TYPE_CONFIG; 48 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.SSL_CONFIG_PREFIX; 49 | import static org.apache.kafka.connect.json.JsonConverterConfig.SCHEMAS_ENABLE_CONFIG; 50 | import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG; 51 | import static org.apache.kafka.connect.runtime.ConnectorConfig.KEY_CONVERTER_CLASS_CONFIG; 52 | import static org.apache.kafka.connect.runtime.ConnectorConfig.TASKS_MAX_CONFIG; 53 | import static org.apache.kafka.connect.runtime.ConnectorConfig.VALUE_CONVERTER_CLASS_CONFIG; 54 | import static org.apache.kafka.connect.runtime.SinkConnectorConfig.TOPICS_CONFIG; 55 | import org.apache.kafka.common.config.SslConfigs; 56 | 57 | import org.junit.jupiter.api.*; 58 | import static org.junit.jupiter.api.Assertions.*; 59 | 60 | public class OpenSearchConnectorBaseIT extends BaseConnectorIT { 61 | 62 | protected static final int NUM_RECORDS = 5; 63 | protected static final int TASKS_MAX = 1; 64 | protected static final String CONNECTOR_NAME = "es-connector"; 65 | protected static final String TOPIC = "test"; 66 | 67 | protected static OpenSearchContainer container; 68 | 69 | protected boolean isDataStream; 70 | protected OpenSearchHelperClient helperClient; 71 | protected Map props; 72 | protected String index; 73 | 74 | @AfterAll 75 | public static void cleanupAfterAll() { 76 | container.close(); 77 | } 78 | 79 | @BeforeEach 80 | public void setup() { 81 | index = TOPIC; 82 | isDataStream = false; 83 | 84 | startConnect(); 85 | connect.kafka().createTopic(TOPIC); 86 | 87 | props = createProps(); 88 | helperClient = container.getHelperClient(props); 89 | } 90 | 91 | @AfterEach 92 | public void cleanup() throws IOException { 93 | stopConnect(); 94 | 95 | if (container.isRunning()) { 96 | if (helperClient != null) { 97 | try { 98 | helperClient.deleteIndex(index, isDataStream); 99 | helperClient.close(); 100 | } catch (ConnectException e) { 101 | // Server is already down. No need to close 102 | } 103 | } 104 | } 105 | } 106 | 107 | protected Map createProps() { 108 | Map props = new HashMap<>(); 109 | 110 | // generic configs 111 | props.put(CONNECTOR_CLASS_CONFIG, OpenSearchSinkConnector.class.getName()); 112 | props.put(TOPICS_CONFIG, TOPIC); 113 | props.put(TASKS_MAX_CONFIG, Integer.toString(TASKS_MAX)); 114 | props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); 115 | props.put(VALUE_CONVERTER_CLASS_CONFIG, JsonConverter.class.getName()); 116 | props.put("value.converter." + SCHEMAS_ENABLE_CONFIG, "false"); 117 | 118 | // connectors specific 119 | props.put(CONNECTION_URL_CONFIG, container.getConnectionUrl()); 120 | props.put(IGNORE_KEY_CONFIG, "true"); 121 | props.put(IGNORE_SCHEMA_CONFIG, "true"); 122 | 123 | props.put(SSL_CONFIG_PREFIX + SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, ""); 124 | props.put(CONNECTION_USERNAME_CONFIG, "admin"); 125 | props.put(CONNECTION_PASSWORD_CONFIG, "admin"); 126 | 127 | return props; 128 | } 129 | 130 | protected void runSimpleTest(Map props) throws Exception { 131 | // start the connector 132 | connect.configureConnector(CONNECTOR_NAME, props); 133 | 134 | // wait for tasks to spin up 135 | waitForConnectorToStart(CONNECTOR_NAME, TASKS_MAX); 136 | 137 | writeRecords(NUM_RECORDS); 138 | 139 | verifySearchResults(NUM_RECORDS); 140 | } 141 | 142 | protected void setDataStream() { 143 | isDataStream = true; 144 | props.put(DATA_STREAM_TYPE_CONFIG, "logs"); 145 | props.put(DATA_STREAM_DATASET_CONFIG, "dataset"); 146 | index = "logs-dataset-" + TOPIC; 147 | } 148 | 149 | protected void setupFromContainer() { 150 | String address = container.getConnectionUrl(); 151 | props.put(CONNECTION_URL_CONFIG, address); 152 | helperClient = new OpenSearchHelperClient( 153 | props.get(CONNECTION_URL_CONFIG), 154 | new OpenSearchSinkConnectorConfig(props) 155 | ); 156 | } 157 | 158 | protected void verifySearchResults(int numRecords) throws Exception { 159 | waitForRecords(numRecords); 160 | 161 | for (SearchHit hit : helperClient.search(index)) { 162 | int id = (Integer) hit.getSourceAsMap().get("doc_num"); 163 | assertNotNull(id); 164 | assertTrue(id < numRecords); 165 | 166 | if (isDataStream) { 167 | assertTrue(hit.getIndex().contains(index)); 168 | } else { 169 | assertEquals(index, hit.getIndex()); 170 | } 171 | } 172 | } 173 | 174 | protected void waitForRecords(int numRecords) throws InterruptedException { 175 | TestUtils.waitForCondition( 176 | () -> { 177 | try { 178 | return helperClient.getDocCount(index) == numRecords; 179 | } catch (ElasticsearchStatusException e) { 180 | if (e.getMessage().contains("index_not_found_exception")) { 181 | return false; 182 | } 183 | 184 | throw e; 185 | } 186 | }, 187 | CONSUME_MAX_DURATION_MS, 188 | "Sufficient amount of document were not found in ES on time." 189 | ); 190 | } 191 | 192 | protected void writeRecords(int numRecords) { 193 | writeRecordsFromStartIndex(0, numRecords); 194 | } 195 | 196 | protected void writeRecordsFromStartIndex(int start, int numRecords) { 197 | for (int i = start; i < start + numRecords; i++) { 198 | connect.kafka().produce( 199 | TOPIC, 200 | String.valueOf(i), 201 | String.format("{\"doc_num\":%d,\"@timestamp\":\"2021-04-28T11:11:22.%03dZ\"}", i, i) 202 | ); 203 | } 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /lib/src/test/java/com/dmathieu/kafka/opensearch/integration/OpenSearchConnectorIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch.integration; 17 | 18 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.BATCH_SIZE_CONFIG; 19 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.BEHAVIOR_ON_NULL_VALUES_CONFIG; 20 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.CONNECTION_PASSWORD_CONFIG; 21 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.CONNECTION_USERNAME_CONFIG; 22 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.IGNORE_KEY_CONFIG; 23 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.LINGER_MS_CONFIG; 24 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.BULK_SIZE_BYTES_CONFIG; 25 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.WRITE_METHOD_CONFIG; 26 | import static org.apache.kafka.connect.runtime.ConnectorConfig.VALUE_CONVERTER_CLASS_CONFIG; 27 | import static org.assertj.core.api.Assertions.assertThat; 28 | import static org.awaitility.Awaitility.await; 29 | 30 | import com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig; 31 | import com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.BehaviorOnNullValues; 32 | import com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.WriteMethod; 33 | import com.dmathieu.kafka.opensearch.helper.OpenSearchContainer; 34 | 35 | import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsResult; 36 | import org.apache.kafka.clients.consumer.OffsetAndMetadata; 37 | import org.apache.kafka.common.TopicPartition; 38 | import org.apache.kafka.connect.storage.StringConverter; 39 | import org.apache.kafka.test.TestUtils; 40 | 41 | import io.confluent.common.utils.IntegrationTest; 42 | import org.elasticsearch.client.security.user.User; 43 | import org.elasticsearch.client.security.user.privileges.Role; 44 | import org.elasticsearch.search.SearchHit; 45 | 46 | import java.time.Duration; 47 | import java.util.List; 48 | import java.util.Map; 49 | import java.text.SimpleDateFormat; 50 | import java.util.Date; 51 | import java.util.concurrent.TimeUnit; 52 | 53 | import org.junit.jupiter.api.*; 54 | import static org.junit.jupiter.api.Assertions.*; 55 | 56 | @Tag("Integration") 57 | public class OpenSearchConnectorIT extends OpenSearchConnectorBaseIT { 58 | 59 | protected static final long COMMIT_MAX_DURATION_MS = TimeUnit.MINUTES.toMillis(1); 60 | 61 | // TODO: test compatibility 62 | 63 | @BeforeAll 64 | public static void setupBeforeAll() { 65 | container = OpenSearchContainer.fromSystemProperties(); 66 | container.start(); 67 | } 68 | 69 | @Override @BeforeEach 70 | public void setup() { 71 | if (!container.isRunning()) { 72 | setupBeforeAll(); 73 | } 74 | super.setup(); 75 | } 76 | 77 | /** 78 | * Verify that mapping errors when an index has strict mapping is handled correctly 79 | */ 80 | @Test 81 | public void testStrictMappings() throws Exception { 82 | helperClient.createIndex(TOPIC, "{ \"dynamic\" : \"strict\", " + 83 | " \"properties\": { \"longProp\": { \"type\": \"long\" } } } }"); 84 | 85 | props.put(OpenSearchSinkConnectorConfig.BATCH_SIZE_CONFIG, "1"); 86 | props.put(OpenSearchSinkConnectorConfig.MAX_RETRIES_CONFIG, "1"); 87 | props.put(OpenSearchSinkConnectorConfig.RETRY_BACKOFF_MS_CONFIG, "10"); 88 | props.put(OpenSearchSinkConnectorConfig.MAX_IN_FLIGHT_REQUESTS_CONFIG, "2"); 89 | connect.configureConnector(CONNECTOR_NAME, props); 90 | waitForConnectorToStart(CONNECTOR_NAME, TASKS_MAX); 91 | 92 | assertThat(getConnectorOffset(CONNECTOR_NAME, TOPIC, 0)).isEqualTo(0); 93 | connect.kafka().produce(TOPIC, "key1", "{\"longProp\":1}"); 94 | connect.kafka().produce(TOPIC, "key2", "{\"any-prop\":1}"); 95 | connect.kafka().produce(TOPIC, "key3", "{\"any-prop\":1}"); 96 | connect.kafka().produce(TOPIC, "key4", "{\"any-prop\":1}"); 97 | 98 | await().atMost(Duration.ofMinutes(1)).untilAsserted(() -> 99 | assertThat(connect.connectorStatus(CONNECTOR_NAME).tasks().get(0).state()) 100 | .isEqualTo("FAILED")); 101 | 102 | assertThat(connect.connectorStatus(CONNECTOR_NAME).tasks().get(0).trace()) 103 | .contains("ElasticsearchException[Elasticsearch exception " + 104 | "[type=strict_dynamic_mapping_exception," + 105 | " reason=mapping set to strict, dynamic introduction of"); 106 | 107 | // The framework commits offsets right before failing the task, verify the failed record's 108 | // offset is properly included and we can move on 109 | TestUtils.waitForCondition( 110 | () -> getConnectorOffset(CONNECTOR_NAME, TOPIC, 0) == 2, 111 | COMMIT_MAX_DURATION_MS, 112 | "Connector tasks did store offsets in time." 113 | ); 114 | } 115 | 116 | @Test 117 | public void testTopicChangingSMT() throws Exception { 118 | props.put("transforms", "TimestampRouter"); 119 | props.put("transforms.TimestampRouter.type", "org.apache.kafka.connect.transforms.TimestampRouter"); 120 | String timestampFormat = "YYYYMM"; 121 | SimpleDateFormat formatter = new SimpleDateFormat(timestampFormat); 122 | Date date = new Date(System.currentTimeMillis()); 123 | props.put("transforms.TimestampRouter.topic.format", "route-it-to-here-${topic}-at-${timestamp}"); 124 | props.put("transforms.TimestampRouter.timestamp.format", timestampFormat); 125 | index = String.format("route-it-to-here-%s-at-%s", TOPIC, formatter.format(date)); 126 | runSimpleTest(props); 127 | 128 | TestUtils.waitForCondition( 129 | () -> getConnectorOffset(CONNECTOR_NAME, TOPIC, 0) == NUM_RECORDS, 130 | COMMIT_MAX_DURATION_MS, 131 | "Connector tasks did store offsets in time." 132 | ); 133 | } 134 | 135 | private long getConnectorOffset(String connectorName, String topic, int partition) throws Exception { 136 | String cGroupName = "connect-" + connectorName; 137 | ListConsumerGroupOffsetsResult offsetsResult = connect.kafka().createAdminClient() 138 | .listConsumerGroupOffsets(cGroupName); 139 | OffsetAndMetadata offsetAndMetadata = offsetsResult.partitionsToOffsetAndMetadata().get() 140 | .get(new TopicPartition(topic, partition)); 141 | return offsetAndMetadata == null ? 0 : offsetAndMetadata.offset(); 142 | } 143 | 144 | @Test 145 | public void testBatchByByteSize() throws Exception { 146 | // Based on the size of the topic, key, and value strings in JSON format. 147 | int approximateRecordByteSize = 60; 148 | props.put(BULK_SIZE_BYTES_CONFIG, Integer.toString(approximateRecordByteSize * 2)); 149 | props.put(LINGER_MS_CONFIG, "180000"); 150 | 151 | connect.configureConnector(CONNECTOR_NAME, props); 152 | waitForConnectorToStart(CONNECTOR_NAME, TASKS_MAX); 153 | 154 | writeRecords(3); 155 | // Only 2 records fit in 1 batch. The other record is sent once another record is written. 156 | verifySearchResults(2); 157 | 158 | writeRecords(1); 159 | verifySearchResults(4); 160 | } 161 | 162 | @Test 163 | public void testStopESContainer() throws Exception { 164 | props.put(OpenSearchSinkConnectorConfig.MAX_RETRIES_CONFIG, "2"); 165 | props.put(OpenSearchSinkConnectorConfig.RETRY_BACKOFF_MS_CONFIG, "10"); 166 | props.put(OpenSearchSinkConnectorConfig.BATCH_SIZE_CONFIG, "1"); 167 | props.put(OpenSearchSinkConnectorConfig.MAX_IN_FLIGHT_REQUESTS_CONFIG, 168 | Integer.toString(NUM_RECORDS - 1)); 169 | 170 | // run connector and write 171 | runSimpleTest(props); 172 | 173 | // stop ES, for all following requests to fail with "connection refused" 174 | container.stop(); 175 | 176 | // try to write some more 177 | writeRecords(NUM_RECORDS); 178 | 179 | // Connector should fail since the server is down 180 | await().atMost(Duration.ofMinutes(1)).untilAsserted(() -> 181 | assertThat(connect.connectorStatus(CONNECTOR_NAME).tasks().get(0).state()) 182 | .isEqualTo("FAILED")); 183 | 184 | assertThat(connect.connectorStatus(CONNECTOR_NAME).tasks().get(0).trace()) 185 | .contains("'java.net.ConnectException: Connection refused' after 3 attempt(s)"); 186 | } 187 | 188 | @Test 189 | public void testChangeConfigsAndRestart() throws Exception { 190 | // run connector and write 191 | runSimpleTest(props); 192 | 193 | // restart 194 | props.put(BATCH_SIZE_CONFIG, "10"); 195 | props.put(LINGER_MS_CONFIG, "1000"); 196 | connect.configureConnector(CONNECTOR_NAME, props); 197 | 198 | // write some more 199 | writeRecords(NUM_RECORDS); 200 | verifySearchResults(NUM_RECORDS * 2); 201 | } 202 | 203 | @Test 204 | public void testDelete() throws Exception { 205 | props.put(BEHAVIOR_ON_NULL_VALUES_CONFIG, BehaviorOnNullValues.DELETE.name()); 206 | props.put(IGNORE_KEY_CONFIG, "false"); 207 | runSimpleTest(props); 208 | 209 | // should have 5 records at this point 210 | // try deleting last one 211 | int lastRecord = NUM_RECORDS - 1; 212 | connect.kafka().produce(TOPIC, String.valueOf(lastRecord), null); 213 | 214 | // should have one less records 215 | verifySearchResults(NUM_RECORDS - 1); 216 | } 217 | 218 | @Test 219 | public void testHappyPath() throws Exception { 220 | runSimpleTest(props); 221 | } 222 | 223 | @Test @Disabled 224 | public void testHappyPathDataStream() throws Exception { 225 | setDataStream(); 226 | 227 | runSimpleTest(props); 228 | 229 | assertEquals(index, helperClient.getDataStream(index).getName()); 230 | } 231 | 232 | @Test 233 | public void testNullValue() throws Exception { 234 | runSimpleTest(props); 235 | 236 | // should have 5 records at this point 237 | // try writing null value 238 | connect.kafka().produce(TOPIC, String.valueOf(NUM_RECORDS), null); 239 | 240 | // should still have 5 records 241 | verifySearchResults(NUM_RECORDS); 242 | } 243 | 244 | /* 245 | * Currently writing primitives to ES fails because ES expects a JSON document and the connector 246 | * does not wrap primitives in any way into a JSON document. 247 | */ 248 | @Test 249 | public void testPrimitive() throws Exception { 250 | props.put(VALUE_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); 251 | 252 | connect.configureConnector(CONNECTOR_NAME, props); 253 | 254 | // wait for tasks to spin up 255 | waitForConnectorToStart(CONNECTOR_NAME, TASKS_MAX); 256 | 257 | for (int i = 0; i < NUM_RECORDS; i++) { 258 | connect.kafka().produce(TOPIC, String.valueOf(i), String.valueOf(i)); 259 | } 260 | 261 | waitForRecords(0); 262 | } 263 | 264 | @Test 265 | public void testUpsert() throws Exception { 266 | props.put(WRITE_METHOD_CONFIG, WriteMethod.UPSERT.toString()); 267 | props.put(IGNORE_KEY_CONFIG, "false"); 268 | runSimpleTest(props); 269 | 270 | // should have 10 records at this point 271 | // try updating last one 272 | int lastRecord = NUM_RECORDS - 1; 273 | connect.kafka().produce(TOPIC, String.valueOf(lastRecord), String.format("{\"doc_num\":%d}", 0)); 274 | writeRecordsFromStartIndex(NUM_RECORDS, NUM_RECORDS); 275 | 276 | // should have double number of records 277 | verifySearchResults(NUM_RECORDS * 2); 278 | 279 | for (SearchHit hit : helperClient.search(TOPIC)) { 280 | if (Integer.parseInt(hit.getId()) == lastRecord) { 281 | // last record should be updated 282 | int docNum = (Integer) hit.getSourceAsMap().get("doc_num"); 283 | assertEquals(0, docNum); 284 | } 285 | } 286 | } 287 | } 288 | -------------------------------------------------------------------------------- /lib/src/test/java/com/dmathieu/kafka/opensearch/integration/OpenSearchConnectorKerberosIT.java: -------------------------------------------------------------------------------- 1 | package com.dmathieu.kafka.opensearch.integration; 2 | 3 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.KERBEROS_KEYTAB_PATH_CONFIG; 4 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.KERBEROS_PRINCIPAL_CONFIG; 5 | 6 | import com.dmathieu.kafka.opensearch.helper.OpenSearchContainer; 7 | import java.io.File; 8 | import java.io.IOException; 9 | import java.nio.file.Files; 10 | import java.nio.file.Path; 11 | import java.util.Comparator; 12 | import java.util.Map; 13 | import java.util.Properties; 14 | import org.apache.hadoop.minikdc.MiniKdc; 15 | import org.apache.kafka.connect.errors.ConnectException; 16 | import io.confluent.common.utils.IntegrationTest; 17 | 18 | import org.junit.jupiter.api.*; 19 | 20 | @Tag("Integration") @Disabled 21 | public class OpenSearchConnectorKerberosIT extends OpenSearchConnectorBaseIT { 22 | 23 | private static File baseDir; 24 | private static MiniKdc kdc; 25 | private static String esPrincipal; 26 | protected static String esKeytab; 27 | private static String userPrincipal; 28 | private static String userKeytab; 29 | 30 | @BeforeAll 31 | public static void setupBeforeAll() throws Exception { 32 | initKdc(); 33 | 34 | container = OpenSearchContainer.fromSystemProperties().withKerberosEnabled(esKeytab); 35 | container.start(); 36 | } 37 | 38 | /** 39 | * Shuts down the KDC and cleans up files. 40 | */ 41 | @AfterAll 42 | public static void cleanupAfterAll() { 43 | container.close(); 44 | closeKdc(); 45 | } 46 | 47 | @Test 48 | public void testKerberos() throws Exception { 49 | 50 | addKerberosConfigs(props); 51 | helperClient = container.getHelperClient(props); 52 | runSimpleTest(props); 53 | } 54 | 55 | protected static void initKdc() throws Exception { 56 | baseDir = new File(System.getProperty("test.build.dir", "target/test-dir")); 57 | if (baseDir.exists()) { 58 | deleteDirectory(baseDir.toPath()); 59 | } 60 | 61 | Properties kdcConf = MiniKdc.createConf(); 62 | kdc = new MiniKdc(kdcConf, baseDir); 63 | kdc.start(); 64 | 65 | String es = "es"; 66 | File keytabFile = new File(baseDir, es + ".keytab"); 67 | esKeytab = keytabFile.getAbsolutePath(); 68 | kdc.createPrincipal(keytabFile, es + "/localhost", "HTTP/localhost"); 69 | esPrincipal = es + "/localhost@" + kdc.getRealm(); 70 | 71 | String user = "connect-es"; 72 | keytabFile = new File(baseDir, user + ".keytab"); 73 | userKeytab = keytabFile.getAbsolutePath(); 74 | kdc.createPrincipal(keytabFile, user + "/localhost"); 75 | userPrincipal = user + "/localhost@" + kdc.getRealm(); 76 | } 77 | 78 | protected static void addKerberosConfigs(Map props) { 79 | props.put(KERBEROS_PRINCIPAL_CONFIG, userPrincipal); 80 | props.put(KERBEROS_KEYTAB_PATH_CONFIG, userKeytab); 81 | } 82 | 83 | private static void closeKdc() { 84 | if (kdc != null) { 85 | kdc.stop(); 86 | } 87 | 88 | if (baseDir.exists()) { 89 | deleteDirectory(baseDir.toPath()); 90 | } 91 | } 92 | 93 | private static void deleteDirectory(Path directoryPath) { 94 | try { 95 | Files.walk(directoryPath) 96 | .sorted(Comparator.reverseOrder()) 97 | .map(Path::toFile) 98 | .forEach(File::delete); 99 | } catch (IOException e) { 100 | throw new ConnectException(e); 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /lib/src/test/java/com/dmathieu/kafka/opensearch/integration/OpenSearchSinkTaskIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Confluent Inc. 3 | * 4 | * Licensed under the Confluent Community License (the "License"); you may not use 5 | * this file except in compliance with the License. You may obtain a copy of the 6 | * License at 7 | * 8 | * http://www.confluent.io/confluent-community-license 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | * specific language governing permissions and limitations under the License. 14 | */ 15 | 16 | package com.dmathieu.kafka.opensearch.integration; 17 | 18 | import com.fasterxml.jackson.core.JsonProcessingException; 19 | import com.github.tomakehurst.wiremock.client.WireMock; 20 | import com.github.tomakehurst.wiremock.junit.WireMockRule; 21 | import com.dmathieu.kafka.opensearch.OpenSearchSinkConnector; 22 | import com.dmathieu.kafka.opensearch.OpenSearchSinkTask; 23 | import org.apache.kafka.clients.consumer.OffsetAndMetadata; 24 | import org.apache.kafka.common.TopicPartition; 25 | import org.apache.kafka.connect.errors.ConnectException; 26 | import org.apache.kafka.connect.errors.DataException; 27 | import org.apache.kafka.connect.json.JsonConverter; 28 | import org.apache.kafka.connect.sink.SinkRecord; 29 | import org.apache.kafka.connect.sink.SinkTaskContext; 30 | import org.apache.kafka.connect.storage.StringConverter; 31 | import org.testcontainers.shaded.com.google.common.collect.ImmutableList; 32 | import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; 33 | import org.testcontainers.shaded.com.google.common.collect.ImmutableSet; 34 | 35 | import java.util.ArrayList; 36 | import java.util.Collections; 37 | import java.util.HashMap; 38 | import java.util.List; 39 | import java.util.Map; 40 | import java.util.stream.IntStream; 41 | 42 | import com.github.tomakehurst.wiremock.WireMockServer; 43 | import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; 44 | import static com.github.tomakehurst.wiremock.client.WireMock.any; 45 | import static com.github.tomakehurst.wiremock.client.WireMock.anyUrl; 46 | import static com.github.tomakehurst.wiremock.client.WireMock.ok; 47 | import static com.github.tomakehurst.wiremock.client.WireMock.okJson; 48 | import static com.github.tomakehurst.wiremock.client.WireMock.post; 49 | import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; 50 | import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; 51 | 52 | import static com.dmathieu.kafka.opensearch.OpenSearchSinkConnectorConfig.*; 53 | import static com.dmathieu.kafka.opensearch.integration.OpenSearchConnectorNetworkIT.errorBulkResponse; 54 | import static java.util.stream.Collectors.toList; 55 | import static org.apache.kafka.connect.json.JsonConverterConfig.SCHEMAS_ENABLE_CONFIG; 56 | import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG; 57 | import static org.apache.kafka.connect.runtime.ConnectorConfig.KEY_CONVERTER_CLASS_CONFIG; 58 | import static org.apache.kafka.connect.runtime.ConnectorConfig.TASKS_MAX_CONFIG; 59 | import static org.apache.kafka.connect.runtime.ConnectorConfig.VALUE_CONVERTER_CLASS_CONFIG; 60 | import static org.apache.kafka.connect.runtime.SinkConnectorConfig.TOPICS_CONFIG; 61 | import static org.assertj.core.api.Assertions.assertThat; 62 | import static org.assertj.core.api.Assertions.assertThatThrownBy; 63 | import static org.awaitility.Awaitility.await; 64 | import static org.mockito.Mockito.mock; 65 | import static org.mockito.Mockito.verify; 66 | import static org.mockito.Mockito.when; 67 | 68 | import org.junit.jupiter.api.*; 69 | import static org.junit.jupiter.api.Assertions.*; 70 | 71 | public class OpenSearchSinkTaskIT { 72 | 73 | protected static final String TOPIC = "test"; 74 | protected static final int TASKS_MAX = 1; 75 | protected WireMockServer wireMockServer; 76 | 77 | @BeforeEach 78 | public void setup() { 79 | wireMockServer = new WireMockServer(options().dynamicPort()); 80 | wireMockServer.start(); 81 | wireMockServer.stubFor(any(anyUrl()).atPriority(10).willReturn(ok())); 82 | } 83 | 84 | @AfterEach 85 | public void cleanup() { 86 | wireMockServer.stop(); 87 | } 88 | 89 | @Test 90 | public void testOffsetCommit() { 91 | wireMockServer.stubFor(post(urlPathEqualTo("/_bulk")) 92 | .willReturn(ok().withFixedDelay(60_000))); 93 | 94 | Map props = createProps(); 95 | props.put(READ_TIMEOUT_MS_CONFIG, "1000"); 96 | props.put(MAX_RETRIES_CONFIG, "2"); 97 | props.put(RETRY_BACKOFF_MS_CONFIG, "10"); 98 | props.put(BATCH_SIZE_CONFIG, "1"); 99 | 100 | OpenSearchSinkTask task = new OpenSearchSinkTask(); 101 | TopicPartition tp = new TopicPartition(TOPIC, 0); 102 | 103 | SinkTaskContext context = mock(SinkTaskContext.class); 104 | when(context.assignment()).thenReturn(ImmutableSet.of(tp)); 105 | task.initialize(context); 106 | task.start(props); 107 | 108 | List records = ImmutableList.of( 109 | sinkRecord(tp, 0), 110 | sinkRecord(tp, 1)); 111 | task.put(records); 112 | 113 | // Nothing should be committed at this point 114 | Map currentOffsets = 115 | ImmutableMap.of(tp, new OffsetAndMetadata(2)); 116 | assertThat(task.preCommit(currentOffsets)).isEqualTo(currentOffsets); 117 | } 118 | 119 | @Test 120 | public void testIndividualFailure() throws JsonProcessingException { 121 | wireMockServer.stubFor(post(urlPathEqualTo("/_bulk")) 122 | .willReturn(okJson(OpenSearchConnectorNetworkIT.errorBulkResponse(3, 123 | "strict_dynamic_mapping_exception", 1)))); 124 | 125 | Map props = createProps(); 126 | props.put(READ_TIMEOUT_MS_CONFIG, "1000"); 127 | props.put(MAX_RETRIES_CONFIG, "2"); 128 | props.put(RETRY_BACKOFF_MS_CONFIG, "10"); 129 | props.put(MAX_IN_FLIGHT_REQUESTS_CONFIG, "1"); 130 | props.put(BATCH_SIZE_CONFIG, "3"); 131 | props.put(LINGER_MS_CONFIG, "10000"); 132 | props.put(BEHAVIOR_ON_MALFORMED_DOCS_CONFIG, "ignore"); 133 | 134 | final OpenSearchSinkTask task = new OpenSearchSinkTask(); 135 | TopicPartition tp = new TopicPartition(TOPIC, 0); 136 | 137 | SinkTaskContext context = mock(SinkTaskContext.class); 138 | when(context.assignment()).thenReturn(ImmutableSet.of(tp)); 139 | task.initialize(context); 140 | task.start(props); 141 | 142 | List records = IntStream.range(0, 6).boxed() 143 | .map(offset -> sinkRecord(tp, offset)) 144 | .collect(toList()); 145 | task.put(records); 146 | 147 | // All is safe to commit 148 | Map currentOffsets = 149 | ImmutableMap.of(tp, new OffsetAndMetadata(6)); 150 | assertThat(task.preCommit(currentOffsets)) 151 | .isEqualTo(currentOffsets); 152 | 153 | // Now check that we actually fail and offsets are not past the failed record 154 | props.put(BEHAVIOR_ON_MALFORMED_DOCS_CONFIG, "fail"); 155 | task.initialize(context); 156 | task.start(props); 157 | 158 | assertThatThrownBy(() -> task.put(records)) 159 | .isInstanceOf(ConnectException.class) 160 | .hasMessageContaining("Indexing record failed"); 161 | 162 | currentOffsets = ImmutableMap.of(tp, new OffsetAndMetadata(0)); 163 | assertThat(getOffsetOrZero(task.preCommit(currentOffsets), tp)) 164 | .isLessThanOrEqualTo(1); 165 | } 166 | 167 | @Test 168 | public void testConvertDataException() throws JsonProcessingException { 169 | wireMockServer.stubFor(post(urlPathEqualTo("/_bulk")) 170 | .willReturn(okJson(errorBulkResponse(3)))); 171 | 172 | Map props = createProps(); 173 | props.put(READ_TIMEOUT_MS_CONFIG, "1000"); 174 | props.put(MAX_RETRIES_CONFIG, "2"); 175 | props.put(RETRY_BACKOFF_MS_CONFIG, "10"); 176 | props.put(MAX_IN_FLIGHT_REQUESTS_CONFIG, "1"); 177 | props.put(BATCH_SIZE_CONFIG, "10"); 178 | props.put(LINGER_MS_CONFIG, "10000"); 179 | props.put(IGNORE_KEY_CONFIG, "false"); 180 | props.put(DROP_INVALID_MESSAGE_CONFIG, "true"); 181 | 182 | final OpenSearchSinkTask task = new OpenSearchSinkTask(); 183 | TopicPartition tp = new TopicPartition(TOPIC, 0); 184 | 185 | SinkTaskContext context = mock(SinkTaskContext.class); 186 | when(context.assignment()).thenReturn(ImmutableSet.of(tp)); 187 | task.initialize(context); 188 | task.start(props); 189 | 190 | List records = ImmutableList.of( 191 | sinkRecord(tp, 0), 192 | sinkRecord(tp, 1, null, "value"), // this should throw a DataException 193 | sinkRecord(tp, 2)); 194 | task.put(records); 195 | 196 | // All is safe to commit 197 | Map currentOffsets = 198 | ImmutableMap.of(tp, new OffsetAndMetadata(3)); 199 | assertThat(task.preCommit(currentOffsets)) 200 | .isEqualTo(currentOffsets); 201 | 202 | // Now check that we actually fail and offsets are not past the failed record 203 | props.put(DROP_INVALID_MESSAGE_CONFIG, "false"); 204 | task.initialize(context); 205 | task.start(props); 206 | 207 | assertThatThrownBy(() -> task.put(records)) 208 | .isInstanceOf(DataException.class) 209 | .hasMessageContaining("Key is used as document id and can not be null"); 210 | 211 | currentOffsets = ImmutableMap.of(tp, new OffsetAndMetadata(0)); 212 | assertThat(task.preCommit(currentOffsets).get(tp).offset()) 213 | .isLessThanOrEqualTo(1); 214 | } 215 | 216 | @Test 217 | public void testNullValue() throws JsonProcessingException { 218 | wireMockServer.stubFor(post(urlPathEqualTo("/_bulk")) 219 | .willReturn(okJson(errorBulkResponse(3)))); 220 | 221 | Map props = createProps(); 222 | props.put(READ_TIMEOUT_MS_CONFIG, "1000"); 223 | props.put(MAX_RETRIES_CONFIG, "2"); 224 | props.put(RETRY_BACKOFF_MS_CONFIG, "10"); 225 | props.put(MAX_IN_FLIGHT_REQUESTS_CONFIG, "1"); 226 | props.put(BATCH_SIZE_CONFIG, "10"); 227 | props.put(LINGER_MS_CONFIG, "10000"); 228 | props.put(BEHAVIOR_ON_NULL_VALUES_CONFIG, "ignore"); 229 | 230 | final OpenSearchSinkTask task = new OpenSearchSinkTask(); 231 | TopicPartition tp = new TopicPartition(TOPIC, 0); 232 | 233 | SinkTaskContext context = mock(SinkTaskContext.class); 234 | when(context.assignment()).thenReturn(ImmutableSet.of(tp)); 235 | task.initialize(context); 236 | task.start(props); 237 | 238 | List records = ImmutableList.of( 239 | sinkRecord(tp, 0), 240 | sinkRecord(tp, 1, "testKey", null), 241 | sinkRecord(tp, 2)); 242 | task.put(records); 243 | 244 | // All is safe to commit 245 | Map currentOffsets = 246 | ImmutableMap.of(tp, new OffsetAndMetadata(3)); 247 | assertThat(task.preCommit(currentOffsets)) 248 | .isEqualTo(currentOffsets); 249 | 250 | // Now check that we actually fail and offsets are not past the failed record 251 | props.put(BEHAVIOR_ON_NULL_VALUES_CONFIG, "fail"); 252 | task.initialize(context); 253 | task.start(props); 254 | 255 | assertThatThrownBy(() -> task.put(records)) 256 | .isInstanceOf(DataException.class) 257 | .hasMessageContaining("null value encountered"); 258 | 259 | currentOffsets = ImmutableMap.of(tp, new OffsetAndMetadata(0)); 260 | assertThat(task.preCommit(currentOffsets).get(tp).offset()) 261 | .isLessThanOrEqualTo(1); 262 | } 263 | 264 | /** 265 | * Verify things are handled correctly when we receive the same records in a new put call 266 | * (e.g. after a RetriableException) 267 | */ 268 | @Test 269 | public void testPutRetry() throws Exception { 270 | wireMockServer.stubFor(post(urlPathEqualTo("/_bulk")) 271 | .withRequestBody(WireMock.containing("{\"doc_num\":0}")) 272 | .willReturn(okJson(OpenSearchConnectorNetworkIT.errorBulkResponse()))); 273 | 274 | wireMockServer.stubFor(post(urlPathEqualTo("/_bulk")) 275 | .withRequestBody(WireMock.containing("{\"doc_num\":1}")) 276 | .willReturn(aResponse().withFixedDelay(60_000))); 277 | 278 | Map props = createProps(); 279 | props.put(READ_TIMEOUT_MS_CONFIG, "1000"); 280 | props.put(MAX_RETRIES_CONFIG, "2"); 281 | props.put(RETRY_BACKOFF_MS_CONFIG, "10"); 282 | props.put(BATCH_SIZE_CONFIG, "1"); 283 | 284 | OpenSearchSinkTask task = new OpenSearchSinkTask(); 285 | TopicPartition tp = new TopicPartition(TOPIC, 0); 286 | 287 | SinkTaskContext context = mock(SinkTaskContext.class); 288 | when(context.assignment()).thenReturn(ImmutableSet.of(tp)); 289 | task.initialize(context); 290 | task.start(props); 291 | 292 | List records = ImmutableList.of( 293 | sinkRecord(tp, 0), 294 | sinkRecord(tp, 1)); 295 | task.open(ImmutableList.of(tp)); 296 | task.put(records); 297 | 298 | Map currentOffsets = 299 | ImmutableMap.of(tp, new OffsetAndMetadata(2)); 300 | 301 | wireMockServer.stubFor(post(urlPathEqualTo("/_bulk")) 302 | .willReturn(okJson(OpenSearchConnectorNetworkIT.errorBulkResponse()))); 303 | 304 | task.put(records); 305 | await().untilAsserted(() -> 306 | assertThat(task.preCommit(currentOffsets)) 307 | .isEqualTo(currentOffsets)); 308 | } 309 | 310 | /** 311 | * Verify partitions are paused and resumed 312 | */ 313 | /*@Test @Disabled 314 | public void testOffsetsBackpressure() throws Exception { 315 | wireMockServer.stubFor(post(urlPathEqualTo("/_bulk")) 316 | .willReturn(okJson(OpenSearchConnectorNetworkIT.errorBulkResponse()))); 317 | 318 | wireMockServer.stubFor(post(urlPathEqualTo("/_bulk")) 319 | .withRequestBody(WireMock.containing("{\"doc_num\":0}")) 320 | .willReturn(okJson(OpenSearchConnectorNetworkIT.errorBulkResponse()) 321 | .withTransformers(BlockingTransformer.NAME))); 322 | 323 | Map props = createProps(); 324 | props.put(READ_TIMEOUT_MS_CONFIG, "1000"); 325 | props.put(MAX_RETRIES_CONFIG, "2"); 326 | props.put(RETRY_BACKOFF_MS_CONFIG, "10"); 327 | props.put(BATCH_SIZE_CONFIG, "1"); 328 | props.put(MAX_BUFFERED_RECORDS_CONFIG, "2"); 329 | 330 | OpenSearchSinkTask task = new OpenSearchSinkTask(); 331 | 332 | TopicPartition tp1 = new TopicPartition(TOPIC, 0); 333 | 334 | SinkTaskContext context = mock(SinkTaskContext.class); 335 | when(context.assignment()).thenReturn(ImmutableSet.of(tp1)); 336 | task.initialize(context); 337 | task.start(props); 338 | 339 | List records = new ArrayList<>(); 340 | for (int i = 0; i < 100; i++) { 341 | records.add(sinkRecord(tp1, i)); 342 | } 343 | task.put(records); 344 | 345 | verify(context).pause(tp1); 346 | 347 | BlockingTransformer.getInstance(wireMockServer).release(1); 348 | 349 | await().untilAsserted(() -> { 350 | task.put(Collections.emptyList()); 351 | verify(context).resume(tp1); 352 | }); 353 | }*/ 354 | 355 | /** 356 | * Verify offsets are updated when partitions are closed/open 357 | */ 358 | @Test @Disabled("Reenable when/if we reenable async flushing") 359 | public void testRebalance() throws Exception { 360 | wireMockServer.stubFor(post(urlPathEqualTo("/_bulk")) 361 | .withRequestBody(WireMock.containing("{\"doc_num\":0}")) 362 | .willReturn(okJson(OpenSearchConnectorNetworkIT.errorBulkResponse()))); 363 | 364 | wireMockServer.stubFor(post(urlPathEqualTo("/_bulk")) 365 | .withRequestBody(WireMock.containing("{\"doc_num\":1}")) 366 | .willReturn(aResponse().withFixedDelay(60_000))); 367 | 368 | Map props = createProps(); 369 | props.put(READ_TIMEOUT_MS_CONFIG, "1000"); 370 | props.put(MAX_RETRIES_CONFIG, "2"); 371 | props.put(RETRY_BACKOFF_MS_CONFIG, "10"); 372 | props.put(BATCH_SIZE_CONFIG, "1"); 373 | 374 | OpenSearchSinkTask task = new OpenSearchSinkTask(); 375 | 376 | SinkTaskContext context = mock(SinkTaskContext.class); 377 | task.initialize(context); 378 | task.start(props); 379 | 380 | TopicPartition tp1 = new TopicPartition(TOPIC, 0); 381 | TopicPartition tp2 = new TopicPartition(TOPIC, 1); 382 | List records = ImmutableList.of( 383 | sinkRecord(tp1, 0), 384 | sinkRecord(tp1, 1), 385 | sinkRecord(tp2, 0)); 386 | task.put(records); 387 | 388 | Map currentOffsets = 389 | ImmutableMap.of( 390 | tp1, new OffsetAndMetadata(2), 391 | tp2, new OffsetAndMetadata(1)); 392 | await().untilAsserted(() -> 393 | assertThat(task.preCommit(currentOffsets)) 394 | .isEqualTo(ImmutableMap.of(tp1, new OffsetAndMetadata(1), 395 | tp2, new OffsetAndMetadata(1)))); 396 | 397 | task.close(ImmutableList.of(tp1)); 398 | task.open(ImmutableList.of(new TopicPartition(TOPIC, 2))); 399 | await().untilAsserted(() -> 400 | assertThat(task.preCommit(currentOffsets)) 401 | .isEqualTo(ImmutableMap.of(tp2, new OffsetAndMetadata(1)))); 402 | } 403 | 404 | private SinkRecord sinkRecord(TopicPartition tp, long offset) { 405 | return sinkRecord(tp.topic(), tp.partition(), offset); 406 | } 407 | 408 | private SinkRecord sinkRecord(TopicPartition tp, long offset, String key, Object value) { 409 | return sinkRecord(tp.topic(), tp.partition(), offset, key, value); 410 | } 411 | 412 | private SinkRecord sinkRecord(String topic, int partition, long offset) { 413 | return sinkRecord(topic, partition, offset, "testKey", ImmutableMap.of("doc_num", offset)); 414 | } 415 | 416 | private SinkRecord sinkRecord(String topic, int partition, long offset, String key, Object value) { 417 | return new SinkRecord(topic, 418 | partition, 419 | null, 420 | key, 421 | null, 422 | value, 423 | offset); 424 | } 425 | 426 | protected Map createProps() { 427 | Map props = new HashMap<>(); 428 | 429 | // generic configs 430 | props.put(CONNECTOR_CLASS_CONFIG, OpenSearchSinkConnector.class.getName()); 431 | props.put(TOPICS_CONFIG, TOPIC); 432 | props.put(TASKS_MAX_CONFIG, Integer.toString(TASKS_MAX)); 433 | props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); 434 | props.put(VALUE_CONVERTER_CLASS_CONFIG, JsonConverter.class.getName()); 435 | props.put("value.converter." + SCHEMAS_ENABLE_CONFIG, "false"); 436 | 437 | // connector specific 438 | props.put(CONNECTION_URL_CONFIG, wireMockServer.url("/")); 439 | props.put(IGNORE_KEY_CONFIG, "true"); 440 | props.put(IGNORE_SCHEMA_CONFIG, "true"); 441 | props.put(WRITE_METHOD_CONFIG, WriteMethod.UPSERT.toString()); 442 | 443 | return props; 444 | } 445 | 446 | private long getOffsetOrZero(Map offsetMap, TopicPartition tp) { 447 | OffsetAndMetadata offsetAndMetadata = offsetMap.get(tp); 448 | return offsetAndMetadata == null ? 0 : offsetAndMetadata.offset(); 449 | } 450 | } 451 | -------------------------------------------------------------------------------- /lib/src/test/resources/default/instances.yml: -------------------------------------------------------------------------------- 1 | instances: 2 | - name: elasticsearch 3 | dns: 4 | - elasticsearch 5 | ip: 6 | - ipAddress 7 | - name: kibana 8 | dns: 9 | - kibana 10 | - localhost 11 | ip: 12 | - ipAddress 13 | - name: logstash 14 | dns: 15 | - logstash 16 | - localhost 17 | ip: 18 | - ipAddress 19 | -------------------------------------------------------------------------------- /lib/src/test/resources/default/opensearch.yml: -------------------------------------------------------------------------------- 1 | ## Used by Docker images in our integration test 2 | http.host: 0.0.0.0 3 | network.host: 0.0.0.0 4 | transport.host: 0.0.0.0 5 | 6 | node.store.allow_mmap: false 7 | cluster.routing.allocation.disk.threshold_enabled: false 8 | -------------------------------------------------------------------------------- /lib/src/test/resources/default/start-opensearch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Generate the certificates used in the HTTPS tests. 4 | # Copy these into the docker image to make available to client (Java class in this repo) and 5 | # elastic server (docker container started in test setup). Volume mounting is unsolved issue. 6 | 7 | OS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}"/ )" >/dev/null 2>&1 && pwd )" 8 | cd $OS_DIR/.. 9 | OS_DIR=$(pwd) 10 | export PATH=/usr/share/elasticsearch/jdk/bin/:$PATH 11 | 12 | if [[ -z "${IP_ADDRESS}" ]]; then 13 | IP_ADDRESS=$(hostname -I) 14 | fi 15 | 16 | echo 17 | echo "Replacing the ip address in the ${OS_DIR}/config/ssl/instances.yml file with ${IP_ADDRESS}" 18 | sed -i "s/ipAddress/${IP_ADDRESS}/g" ${OS_DIR}/config/instances.yml 19 | 20 | echo 21 | echo "Starting OpenSearch ..." 22 | /usr/share/opensearch/opensearch-docker-entrypoint.sh 23 | -------------------------------------------------------------------------------- /lib/src/test/resources/kerberos/instances.yml: -------------------------------------------------------------------------------- 1 | instances: 2 | - name: elasticsearch 3 | dns: 4 | - elasticsearch 5 | ip: 6 | - ipAddress 7 | - name: kibana 8 | dns: 9 | - kibana 10 | - localhost 11 | ip: 12 | - ipAddress 13 | - name: logstash 14 | dns: 15 | - logstash 16 | - localhost 17 | ip: 18 | - ipAddress 19 | -------------------------------------------------------------------------------- /lib/src/test/resources/kerberos/opensearch.yml: -------------------------------------------------------------------------------- 1 | ## Used by Docker images in our integration test 2 | http.host: 0.0.0.0 3 | network.host: 0.0.0.0 4 | transport.host: 0.0.0.0 5 | 6 | node.store.allow_mmap: false 7 | cluster.routing.allocation.disk.threshold_enabled: false 8 | 9 | xpack.license.self_generated.type: trial 10 | 11 | # Kerberos realm 12 | xpack.security.authc.realms.kerberos.kerb1: 13 | order: 3 14 | keytab.path: es.keytab 15 | remove_realm_name: false 16 | 17 | # enable anonymous connections since setting passwords requires running a command 18 | xpack.security.authc: 19 | anonymous: 20 | username: connect_user 21 | roles: superuser 22 | authz_exception: true -------------------------------------------------------------------------------- /lib/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2018 Confluent Inc. 3 | # 4 | # Licensed under the Confluent Community License (the "License"); you may not use 5 | # this file except in compliance with the License. You may obtain a copy of the 6 | # License at 7 | # 8 | # http://www.confluent.io/confluent-community-license 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OF ANY KIND, either express or implied. See the License for the 13 | # specific language governing permissions and limitations under the License. 14 | # 15 | 16 | log4j.rootLogger=INFO, stdout 17 | 18 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 19 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 20 | #log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n 21 | log4j.appender.stdout.layout.ConversionPattern=[%d] %p %X{connector.context}%m (%c:%L)%n 22 | 23 | log4j.logger.org.apache.kafka=ERROR 24 | log4j.logger.org.apache.zookeeper=ERROR 25 | log4j.logger.org.I0Itec.zkclient=ERROR 26 | log4j.logger.org.reflections=ERROR 27 | log4j.logger.org.eclipse.jetty=ERROR 28 | log4j.logger.org.glassfish.jersey.internal=ERROR 29 | 30 | log4j.logger.io.confluent.connect.elasticsearch=DEBUG 31 | -------------------------------------------------------------------------------- /lib/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker: -------------------------------------------------------------------------------- 1 | mock-maker-inline 2 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | * 4 | * The settings file is used to specify which projects to include in your build. 5 | * 6 | * Detailed information about configuring a multi-project build in Gradle can be found 7 | * in the user manual at https://docs.gradle.org/7.2/userguide/multi_project_builds.html 8 | */ 9 | 10 | rootProject.name = 'kafka-connect-opensearch' 11 | include('lib') 12 | --------------------------------------------------------------------------------