├── .gitignore ├── .travis.yml ├── LICENSE.txt ├── README.md ├── assets └── mongodb-leaf-only.png ├── build.gradle ├── config └── MongoDbSinkConnector.properties ├── docs └── logos │ ├── qudosoft.png │ └── runtitle.png ├── pom.xml └── src ├── main ├── assembly │ └── package.xml ├── java │ └── at │ │ └── grahsl │ │ └── kafka │ │ └── connect │ │ └── mongodb │ │ ├── CollectionAwareConfig.java │ │ ├── MongoDbSinkConnector.java │ │ ├── MongoDbSinkConnectorConfig.java │ │ ├── MongoDbSinkRecordBatches.java │ │ ├── MongoDbSinkTask.java │ │ ├── VersionUtil.java │ │ ├── cdc │ │ ├── CdcHandler.java │ │ ├── CdcOperation.java │ │ └── debezium │ │ │ ├── DebeziumCdcHandler.java │ │ │ ├── OperationType.java │ │ │ ├── mongodb │ │ │ ├── MongoDbDelete.java │ │ │ ├── MongoDbHandler.java │ │ │ ├── MongoDbInsert.java │ │ │ ├── MongoDbNoOp.java │ │ │ └── MongoDbUpdate.java │ │ │ └── rdbms │ │ │ ├── RdbmsDelete.java │ │ │ ├── RdbmsHandler.java │ │ │ ├── RdbmsInsert.java │ │ │ ├── RdbmsNoOp.java │ │ │ ├── RdbmsUpdate.java │ │ │ ├── mysql │ │ │ └── MysqlHandler.java │ │ │ └── postgres │ │ │ └── PostgresHandler.java │ │ ├── converter │ │ ├── AvroJsonSchemafulRecordConverter.java │ │ ├── FieldConverter.java │ │ ├── JsonRawStringRecordConverter.java │ │ ├── JsonSchemalessRecordConverter.java │ │ ├── RecordConverter.java │ │ ├── SinkConverter.java │ │ ├── SinkDocument.java │ │ ├── SinkFieldConverter.java │ │ └── types │ │ │ └── sink │ │ │ └── bson │ │ │ ├── BooleanFieldConverter.java │ │ │ ├── BytesFieldConverter.java │ │ │ ├── Float32FieldConverter.java │ │ │ ├── Float64FieldConverter.java │ │ │ ├── Int16FieldConverter.java │ │ │ ├── Int32FieldConverter.java │ │ │ ├── Int64FieldConverter.java │ │ │ ├── Int8FieldConverter.java │ │ │ ├── StringFieldConverter.java │ │ │ └── logical │ │ │ ├── DateFieldConverter.java │ │ │ ├── DecimalFieldConverter.java │ │ │ ├── TimeFieldConverter.java │ │ │ └── TimestampFieldConverter.java │ │ ├── processor │ │ ├── BlacklistKeyProjector.java │ │ ├── BlacklistValueProjector.java │ │ ├── DocumentIdAdder.java │ │ ├── KafkaMetaAdder.java │ │ ├── PostProcessor.java │ │ ├── WhitelistKeyProjector.java │ │ ├── WhitelistValueProjector.java │ │ ├── field │ │ │ ├── projection │ │ │ │ ├── BlacklistProjector.java │ │ │ │ ├── FieldProjector.java │ │ │ │ └── WhitelistProjector.java │ │ │ └── renaming │ │ │ │ ├── FieldnameMapping.java │ │ │ │ ├── RegExpSettings.java │ │ │ │ ├── RenameByMapping.java │ │ │ │ ├── RenameByRegExp.java │ │ │ │ └── Renamer.java │ │ └── id │ │ │ └── strategy │ │ │ ├── BsonOidStrategy.java │ │ │ ├── FullKeyStrategy.java │ │ │ ├── IdStrategy.java │ │ │ ├── KafkaMetaDataStrategy.java │ │ │ ├── PartialKeyStrategy.java │ │ │ ├── PartialValueStrategy.java │ │ │ ├── ProvidedInKeyStrategy.java │ │ │ ├── ProvidedInValueStrategy.java │ │ │ ├── ProvidedStrategy.java │ │ │ └── UuidStrategy.java │ │ └── writemodel │ │ └── strategy │ │ ├── DeleteOneDefaultStrategy.java │ │ ├── MonotonicWritesDefaultStrategy.java │ │ ├── ReplaceOneBusinessKeyStrategy.java │ │ ├── ReplaceOneDefaultStrategy.java │ │ ├── UpdateOneTimestampsStrategy.java │ │ └── WriteModelStrategy.java └── resources │ ├── kafka-connect-mongodb-version.properties │ └── logback.xml └── test ├── java └── at │ └── grahsl │ └── kafka │ └── connect │ └── mongodb │ ├── MongoDbSinkConnectorConfigTest.java │ ├── MongoDbSinkRecordBatchesTest.java │ ├── MongoDbSinkTaskTest.java │ ├── ValidatorWithOperatorsTest.java │ ├── cdc │ └── debezium │ │ ├── OperationTypeTest.java │ │ ├── mongodb │ │ ├── MongoDbDeleteTest.java │ │ ├── MongoDbHandlerTest.java │ │ ├── MongoDbInsertTest.java │ │ ├── MongoDbNoOpTest.java │ │ └── MongoDbUpdateTest.java │ │ └── rdbms │ │ ├── RdbmsDeleteTest.java │ │ ├── RdbmsHandlerTest.java │ │ ├── RdbmsInsertTest.java │ │ ├── RdbmsNoOpTest.java │ │ └── RdbmsUpdateTest.java │ ├── converter │ ├── RecordConverterTest.java │ ├── SinkConverterTest.java │ ├── SinkDocumentTest.java │ └── SinkFieldConverterTest.java │ ├── data │ └── avro │ │ └── TweetMsg.java │ ├── end2end │ └── MinimumViableIT.java │ ├── processor │ ├── DocumentIdAdderTest.java │ ├── field │ │ ├── projection │ │ │ └── FieldProjectorTest.java │ │ └── renaming │ │ │ └── RenamerTest.java │ └── id │ │ └── strategy │ │ └── IdStrategyTest.java │ └── writemodel │ └── strategy │ └── WriteModelStrategyTest.java └── resources ├── avro └── tweetmsg.avsc ├── config └── sink_connector.json └── docker └── compose-env.yml /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *.iml 3 | target/ 4 | 5 | .gradle/ 6 | build/ 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | 3 | services: 4 | - docker 5 | 6 | env: 7 | DOCKER_COMPOSE_VERSION=1.25.0 8 | 9 | addons: 10 | hosts: 11 | - zookeeper 12 | - kafkabroker 13 | - kafkaconnect 14 | - schemaregistry 15 | - mongodb 16 | 17 | language: java 18 | 19 | jdk: 20 | - openjdk8 21 | - openjdk11 22 | 23 | before_install: 24 | - sudo rm /usr/local/bin/docker-compose 25 | - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose 26 | - chmod +x docker-compose 27 | - sudo mv docker-compose /usr/local/bin 28 | 29 | script: 30 | - mvn install -Dmaven.javadoc.skip=true -Dgpg.skip -B -V 31 | 32 | after_success: 33 | - mvn jacoco:prepare-agent test jacoco:report 34 | - mvn com.gavinmogan:codacy-maven-plugin:coverage -DcoverageReportFile=target/site/jacoco/jacoco.xml -DprojectToken=$CODACY_PROJECT_TOKEN -DapiToken=$CODACY_API_TOKEN 35 | -------------------------------------------------------------------------------- /assets/mongodb-leaf-only.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpgrahsl/kafka-connect-mongodb/772f543b363bf708fa4b2af44b148ac2474d492d/assets/mongodb-leaf-only.png -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java' 2 | apply plugin: 'maven' 3 | apply plugin: 'com.github.johnrengelman.shadow' 4 | 5 | group = 'at.grahsl.kafka.connect' 6 | version = '1.4.0' 7 | 8 | description = """kafka-connect-mongodb""" 9 | 10 | sourceCompatibility = 1.8 11 | targetCompatibility = 1.8 12 | 13 | tasks.withType(JavaCompile) { 14 | options.encoding = 'UTF-8' 15 | } 16 | 17 | buildscript { 18 | repositories { 19 | jcenter() 20 | } 21 | dependencies { 22 | classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.3' 23 | } 24 | } 25 | 26 | processResources { 27 | expand(project.properties) 28 | } 29 | 30 | def skipIntegrationTest = 'true' 31 | test { 32 | if (skipIntegrationTest.toBoolean()) { 33 | exclude '**/*IT.class' 34 | } 35 | } 36 | 37 | task copyJarToTarget(type: Copy, dependsOn:[jar, shadowJar]) { 38 | description 'Copies jar files to target directory' 39 | copy { 40 | from 'build/libs' 41 | into 'target' 42 | include '**/*.jar' 43 | } 44 | } 45 | 46 | repositories { 47 | maven { url "http://packages.confluent.io/maven/" } 48 | maven { url "http://repo.maven.apache.org/maven2" } 49 | } 50 | 51 | ext { 52 | kafkaVersion='2.4.0' 53 | mongodbDriverVersion='3.12.0' 54 | logbackVersion='1.2.3' 55 | jacksonVersion='2.10.1' 56 | confluentSerializerVersion='5.3.2' 57 | confluentConnectPluginVersion='0.11.2' 58 | junitJupiterVersion='5.5.2' 59 | junitPlatformVersion='1.5.2' 60 | hamcrestVersion='2.0.0.0' 61 | mockitoVersion='3.2.4' 62 | testcontainersVersion='1.12.3' 63 | avroVersion='1.9.1' 64 | okHttpVersion='3.14.4' 65 | yamlBeansVersion='1.13' 66 | } 67 | 68 | dependencies { 69 | compile "org.apache.kafka:connect-api:${kafkaVersion}" 70 | compile "org.mongodb:mongodb-driver:${mongodbDriverVersion}" 71 | compile "ch.qos.logback:logback-classic:${logbackVersion}" 72 | testCompile "com.github.jcustenborder.kafka.connect:connect-utils:[0.2.31,0.2.1000)" 73 | compile "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}" 74 | compile "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" 75 | compile "io.confluent:kafka-avro-serializer:${confluentSerializerVersion}" 76 | compile "io.confluent:kafka-connect-maven-plugin:${confluentConnectPluginVersion}" 77 | testCompile "org.junit.jupiter:junit-jupiter-engine:${junitJupiterVersion}" 78 | testCompile "org.junit.jupiter:junit-jupiter-params:${junitJupiterVersion}" 79 | testCompile "org.junit.vintage:junit-vintage-engine:${junitJupiterVersion}" 80 | testCompile "org.junit.platform:junit-platform-runner:${junitPlatformVersion}" 81 | testCompile "org.junit.platform:junit-platform-console:${junitPlatformVersion}" 82 | testCompile "org.hamcrest:hamcrest-junit:${hamcrestVersion}" 83 | testCompile "org.mockito:mockito-core:${mockitoVersion}" 84 | testCompile "org.testcontainers:testcontainers:${testcontainersVersion}" 85 | testCompile "org.apache.avro:avro:${avroVersion}" 86 | testCompile "org.apache.avro:avro-maven-plugin:${avroVersion}" 87 | testCompile "com.squareup.okhttp3:okhttp:${okHttpVersion}" 88 | testCompile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jacksonVersion}" 89 | testCompile "com.esotericsoftware.yamlbeans:yamlbeans:${yamlBeansVersion}" 90 | } 91 | -------------------------------------------------------------------------------- /config/MongoDbSinkConnector.properties: -------------------------------------------------------------------------------- 1 | ## 2 | # Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | ## 16 | 17 | name=MyMongoDbSinkConnector 18 | topics=test 19 | tasks.max=1 20 | 21 | key.converter=io.confluent.connect.avro.AvroConverter 22 | key.converter.schema.registry.url=http://localhost:8081 23 | value.converter=io.confluent.connect.avro.AvroConverter 24 | value.converter.schema.registry.url=http://localhost:8081 25 | 26 | connector.class=at.grahsl.kafka.connect.mongodb.MongoDbSinkConnector 27 | 28 | #specific MongoDB sink connector props 29 | #listed below are the defaults 30 | mongodb.connection.uri=mongodb://localhost:27017/kafkaconnect?w=1&journal=true 31 | mongodb.collection= 32 | mongodb.max.num.retries=3 33 | mongodb.retries.defer.timeout=5000 34 | mongodb.value.projection.type=none 35 | mongodb.value.projection.list= 36 | mongodb.document.id.strategy=at.grahsl.kafka.connect.mongodb.processor.id.strategy.BsonOidStrategy 37 | mongodb.document.id.strategies= 38 | mongodb.key.projection.type=none 39 | mongodb.key.projection.list= 40 | mongodb.field.renamer.mapping=[] 41 | mongodb.field.renamer.regexp=[] 42 | mongodb.post.processor.chain=at.grahsl.kafka.connect.mongodb.processor.DocumentIdAdder 43 | mongodb.change.data.capture.handler= 44 | mongodb.change.data.capture.handler.operations=c,r,u,d 45 | mongodb.delete.on.null.values=false 46 | mongodb.writemodel.strategy=at.grahsl.kafka.connect.mongodb.writemodel.strategy.ReplaceOneDefaultStrategy 47 | mongodb.max.batch.size=0 48 | mongodb.rate.limiting.timeout=0 49 | mongodb.rate.limiting.every.n=0 50 | -------------------------------------------------------------------------------- /docs/logos/qudosoft.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpgrahsl/kafka-connect-mongodb/772f543b363bf708fa4b2af44b148ac2474d492d/docs/logos/qudosoft.png -------------------------------------------------------------------------------- /docs/logos/runtitle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hpgrahsl/kafka-connect-mongodb/772f543b363bf708fa4b2af44b148ac2474d492d/docs/logos/runtitle.png -------------------------------------------------------------------------------- /src/main/assembly/package.xml: -------------------------------------------------------------------------------- 1 | 5 | 6 | package 7 | 8 | dir 9 | 10 | false 11 | 12 | 13 | ${project.basedir} 14 | share/doc/${project.name}/ 15 | 16 | README* 17 | LICENSE* 18 | NOTICE* 19 | licenses/ 20 | 21 | 22 | 23 | ${project.basedir}/config 24 | etc/${project.name} 25 | 26 | * 27 | 28 | 29 | 30 | 31 | 32 | share/java/${project.name} 33 | true 34 | true 35 | 36 | org.apache.kafka:connect-api 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/CollectionAwareConfig.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb; 2 | 3 | import org.apache.kafka.common.config.AbstractConfig; 4 | import org.apache.kafka.common.config.ConfigDef; 5 | import org.apache.kafka.common.config.ConfigException; 6 | 7 | import java.util.HashMap; 8 | import java.util.Map; 9 | 10 | public class CollectionAwareConfig extends AbstractConfig { 11 | 12 | //NOTE: the merging of values() and originals() is a workaround 13 | //in order to allow for properties not being given in the 14 | //ConfigDef at compile time to be picked up and available as well... 15 | private final Map collectionAwareSettings; 16 | 17 | public CollectionAwareConfig(ConfigDef definition, Map originals, boolean doLog) { 18 | super(definition, originals, doLog); 19 | collectionAwareSettings = new HashMap<>(256); 20 | collectionAwareSettings.putAll(values()); 21 | collectionAwareSettings.putAll(originals()); 22 | } 23 | 24 | public CollectionAwareConfig(ConfigDef definition, Map originals) { 25 | super(definition, originals); 26 | collectionAwareSettings = new HashMap<>(256); 27 | collectionAwareSettings.putAll(values()); 28 | collectionAwareSettings.putAll(originals()); 29 | } 30 | 31 | protected Object get(String property, String collection) { 32 | String fullProperty = property+"."+collection; 33 | if(collectionAwareSettings.containsKey(fullProperty)) { 34 | return collectionAwareSettings.get(fullProperty); 35 | } 36 | return collectionAwareSettings.get(property); 37 | } 38 | 39 | public String getString(String property, String collection) { 40 | if(collection == null || collection.isEmpty()) { 41 | return (String) get(property); 42 | } 43 | return (String) get(property,collection); 44 | } 45 | 46 | //NOTE: in the this topic aware map, everything is currently stored as 47 | //type String so direct casting won't work which is why the 48 | //*.parse*(String value) methods are to be used for now. 49 | public Boolean getBoolean(String property, String collection) { 50 | Object obj; 51 | 52 | if(collection == null || collection.isEmpty()) { 53 | obj = get(property); 54 | } else { 55 | obj = get(property,collection); 56 | } 57 | 58 | if(obj instanceof Boolean) 59 | return (Boolean) obj; 60 | 61 | if(obj instanceof String) 62 | return Boolean.parseBoolean((String)obj); 63 | 64 | throw new ConfigException("error: unsupported property type for '"+obj+"' where Boolean expected"); 65 | } 66 | 67 | public Integer getInt(String property, String collection) { 68 | 69 | Object obj; 70 | 71 | if(collection == null || collection.isEmpty()) { 72 | obj = get(property); 73 | } else { 74 | obj = get(property,collection); 75 | } 76 | 77 | if(obj instanceof Integer) 78 | return (Integer) obj; 79 | 80 | if(obj instanceof String) 81 | return Integer.parseInt((String)obj); 82 | 83 | throw new ConfigException("error: unsupported property type for '"+obj+"' where Integer expected"); 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/MongoDbSinkConnector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb; 18 | 19 | import org.apache.kafka.common.config.ConfigDef; 20 | import org.apache.kafka.connect.connector.Task; 21 | import org.apache.kafka.connect.sink.SinkConnector; 22 | 23 | import java.util.ArrayList; 24 | import java.util.List; 25 | import java.util.Map; 26 | 27 | public class MongoDbSinkConnector extends SinkConnector { 28 | 29 | private Map settings; 30 | 31 | @Override 32 | public String version() { 33 | return VersionUtil.getVersion(); 34 | } 35 | 36 | @Override 37 | public void start(Map map) { 38 | settings = map; 39 | } 40 | 41 | @Override 42 | public Class taskClass() { 43 | return MongoDbSinkTask.class; 44 | } 45 | 46 | @Override 47 | public List> taskConfigs(int maxTasks) { 48 | 49 | List> taskConfigs = new ArrayList<>(maxTasks); 50 | 51 | for (int i = 0; i < maxTasks; i++) { 52 | taskConfigs.add(settings); 53 | } 54 | 55 | return taskConfigs; 56 | 57 | } 58 | 59 | @Override 60 | public void stop() { 61 | //TODO: what's necessary to stop the connector 62 | } 63 | 64 | @Override 65 | public ConfigDef config() { 66 | return MongoDbSinkConnectorConfig.conf(); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/MongoDbSinkRecordBatches.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb; 18 | 19 | import org.apache.kafka.connect.sink.SinkRecord; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | public class MongoDbSinkRecordBatches { 25 | 26 | private int batchSize; 27 | private int currentBatch = 0; 28 | private List> bufferedBatches = new ArrayList<>(); 29 | 30 | public MongoDbSinkRecordBatches(int batchSize, int records) { 31 | this.batchSize = batchSize; 32 | bufferedBatches.add(batchSize > 0 ? new ArrayList<>(batchSize) : new ArrayList<>(records)); 33 | } 34 | 35 | public void buffer(SinkRecord record) { 36 | if(batchSize > 0) { 37 | if(bufferedBatches.get(currentBatch).size() < batchSize) { 38 | bufferedBatches.get(currentBatch).add(record); 39 | } else { 40 | bufferedBatches.add(new ArrayList<>(batchSize)); 41 | bufferedBatches.get(++currentBatch).add(record); 42 | } 43 | } else { 44 | bufferedBatches.get(0).add(record); 45 | } 46 | } 47 | 48 | public List> getBufferedBatches() { 49 | return bufferedBatches; 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/VersionUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb; 18 | 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | import java.util.Properties; 23 | 24 | public class VersionUtil { 25 | 26 | private static final Logger LOGGER = LoggerFactory.getLogger(VersionUtil.class); 27 | private static String VERSION = "unknown"; 28 | 29 | static { 30 | try { 31 | Properties props = new Properties(); 32 | props.load(VersionUtil.class.getResourceAsStream("/kafka-connect-mongodb-version.properties")); 33 | VERSION = props.getProperty("version", VERSION).trim(); 34 | } catch (Exception e) { 35 | LOGGER.warn("error while loading version:", e); 36 | } 37 | } 38 | 39 | public static String getVersion() { 40 | return VERSION; 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/cdc/CdcHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.cdc; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 21 | import com.mongodb.client.model.WriteModel; 22 | import org.bson.BsonDocument; 23 | 24 | import java.util.Optional; 25 | 26 | public abstract class CdcHandler { 27 | 28 | private final MongoDbSinkConnectorConfig config; 29 | 30 | public CdcHandler(MongoDbSinkConnectorConfig config) { 31 | this.config = config; 32 | } 33 | 34 | public MongoDbSinkConnectorConfig getConfig() { 35 | return this.config; 36 | } 37 | 38 | public abstract Optional> handle(SinkDocument doc); 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/cdc/CdcOperation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.cdc; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 20 | import com.mongodb.client.model.WriteModel; 21 | import org.bson.BsonDocument; 22 | 23 | public interface CdcOperation { 24 | 25 | WriteModel perform(SinkDocument doc); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/DebeziumCdcHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.cdc.debezium; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | import at.grahsl.kafka.connect.mongodb.cdc.CdcHandler; 21 | import at.grahsl.kafka.connect.mongodb.cdc.CdcOperation; 22 | import org.apache.kafka.connect.errors.DataException; 23 | import org.bson.BsonDocument; 24 | 25 | import java.util.HashMap; 26 | import java.util.Map; 27 | 28 | public abstract class DebeziumCdcHandler extends CdcHandler { 29 | 30 | public static final String OPERATION_TYPE_FIELD_PATH = "op"; 31 | 32 | private final Map operations = new HashMap<>(); 33 | 34 | public DebeziumCdcHandler(MongoDbSinkConnectorConfig config) { 35 | super(config); 36 | } 37 | 38 | protected void registerOperations(Map operations) { 39 | this.operations.putAll(operations); 40 | } 41 | 42 | public CdcOperation getCdcOperation(BsonDocument doc) { 43 | try { 44 | if(!doc.containsKey(OPERATION_TYPE_FIELD_PATH) 45 | || !doc.get(OPERATION_TYPE_FIELD_PATH).isString()) { 46 | throw new DataException("error: value doc is missing CDC operation type of type string"); 47 | } 48 | CdcOperation op = operations.get(OperationType.fromText( 49 | doc.get(OPERATION_TYPE_FIELD_PATH).asString().getValue()) 50 | ); 51 | if(op == null) { 52 | throw new DataException("error: no CDC operation found in mapping for op=" 53 | + doc.get(OPERATION_TYPE_FIELD_PATH).asString().getValue()); 54 | } 55 | return op; 56 | } catch (IllegalArgumentException exc){ 57 | throw new DataException("error: parsing CDC operation failed",exc); 58 | } 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/OperationType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.cdc.debezium; 18 | 19 | public enum OperationType { 20 | 21 | CREATE("c"), 22 | READ("r"), 23 | UPDATE("u"), 24 | DELETE("d"); 25 | 26 | private final String text; 27 | 28 | OperationType(String text) { 29 | this.text = text; 30 | } 31 | 32 | String type() { 33 | return this.text; 34 | } 35 | 36 | public static OperationType fromText(String text) { 37 | switch(text) { 38 | case "c": return CREATE; 39 | case "r": return READ; 40 | case "u": return UPDATE; 41 | case "d": return DELETE; 42 | default: 43 | throw new IllegalArgumentException( 44 | "error: unknown operation type " + text 45 | ); 46 | } 47 | } 48 | 49 | } -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/mongodb/MongoDbDelete.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.mongodb; 18 | 19 | import at.grahsl.kafka.connect.mongodb.cdc.CdcOperation; 20 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 21 | import com.mongodb.DBCollection; 22 | import com.mongodb.client.model.DeleteOneModel; 23 | import com.mongodb.client.model.WriteModel; 24 | import org.apache.kafka.connect.errors.DataException; 25 | import org.bson.BsonDocument; 26 | 27 | public class MongoDbDelete implements CdcOperation { 28 | 29 | @Override 30 | public WriteModel perform(SinkDocument doc) { 31 | 32 | BsonDocument keyDoc = doc.getKeyDoc().orElseThrow( 33 | () -> new DataException("error: key doc must not be missing for delete operation") 34 | ); 35 | 36 | try { 37 | BsonDocument filterDoc = BsonDocument.parse( 38 | "{"+DBCollection.ID_FIELD_NAME+ 39 | ":"+keyDoc.getString(MongoDbHandler.JSON_ID_FIELD_PATH) 40 | .getValue()+"}" 41 | ); 42 | return new DeleteOneModel<>(filterDoc); 43 | } catch(Exception exc) { 44 | throw new DataException(exc); 45 | } 46 | 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/mongodb/MongoDbHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.mongodb; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | import at.grahsl.kafka.connect.mongodb.cdc.CdcOperation; 21 | import at.grahsl.kafka.connect.mongodb.cdc.debezium.DebeziumCdcHandler; 22 | import at.grahsl.kafka.connect.mongodb.cdc.debezium.OperationType; 23 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 24 | import com.mongodb.client.model.WriteModel; 25 | import org.apache.kafka.connect.errors.DataException; 26 | import org.bson.BsonDocument; 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | import java.util.HashMap; 31 | import java.util.List; 32 | import java.util.Map; 33 | import java.util.Optional; 34 | import java.util.stream.Stream; 35 | 36 | public class MongoDbHandler extends DebeziumCdcHandler { 37 | 38 | public static final String JSON_ID_FIELD_PATH = "id"; 39 | 40 | private static Logger logger = LoggerFactory.getLogger(MongoDbHandler.class); 41 | 42 | public MongoDbHandler(MongoDbSinkConnectorConfig config) { 43 | super(config); 44 | final Map operations = new HashMap<>(); 45 | operations.put(OperationType.CREATE,new MongoDbInsert()); 46 | operations.put(OperationType.READ,new MongoDbInsert()); 47 | operations.put(OperationType.UPDATE,new MongoDbUpdate()); 48 | operations.put(OperationType.DELETE,new MongoDbDelete()); 49 | registerOperations(operations); 50 | } 51 | 52 | public MongoDbHandler(MongoDbSinkConnectorConfig config, 53 | Map operations) { 54 | super(config); 55 | registerOperations(operations); 56 | } 57 | 58 | public MongoDbHandler(MongoDbSinkConnectorConfig config, List supportedTypes) { 59 | super(config); 60 | final Map operations = new HashMap<>(); 61 | Stream.of(OperationType.values()).forEach(ot -> operations.put(ot, new MongoDbNoOp())); 62 | supportedTypes.forEach(ot -> { 63 | switch (ot) { 64 | case CREATE: 65 | case READ: 66 | operations.put(ot,new MongoDbInsert()); 67 | break; 68 | case UPDATE: 69 | operations.put(ot,new MongoDbUpdate()); 70 | break; 71 | case DELETE: 72 | operations.put(ot,new MongoDbDelete()); 73 | break; 74 | } 75 | }); 76 | registerOperations(operations); 77 | } 78 | 79 | @Override 80 | public Optional> handle(SinkDocument doc) { 81 | 82 | BsonDocument keyDoc = doc.getKeyDoc().orElseThrow( 83 | () -> new DataException("error: key document must not be missing for CDC mode") 84 | ); 85 | 86 | BsonDocument valueDoc = doc.getValueDoc() 87 | .orElseGet(BsonDocument::new); 88 | 89 | if(keyDoc.containsKey(JSON_ID_FIELD_PATH) 90 | && valueDoc.isEmpty()) { 91 | logger.debug("skipping debezium tombstone event for kafka topic compaction"); 92 | return Optional.empty(); 93 | } 94 | 95 | logger.debug("key: "+keyDoc.toString()); 96 | logger.debug("value: "+valueDoc.toString()); 97 | 98 | return Optional.ofNullable(getCdcOperation(valueDoc).perform(doc)); 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/mongodb/MongoDbInsert.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.mongodb; 18 | 19 | import at.grahsl.kafka.connect.mongodb.cdc.CdcOperation; 20 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 21 | import com.mongodb.DBCollection; 22 | import com.mongodb.client.model.ReplaceOneModel; 23 | import com.mongodb.client.model.UpdateOptions; 24 | import com.mongodb.client.model.WriteModel; 25 | import org.apache.kafka.connect.errors.DataException; 26 | import org.bson.BsonDocument; 27 | 28 | public class MongoDbInsert implements CdcOperation { 29 | 30 | public static final String JSON_DOC_FIELD_PATH = "after"; 31 | 32 | private static final UpdateOptions UPDATE_OPTIONS = 33 | new UpdateOptions().upsert(true); 34 | 35 | @Override 36 | public WriteModel perform(SinkDocument doc) { 37 | 38 | BsonDocument valueDoc = doc.getValueDoc().orElseThrow( 39 | () -> new DataException("error: value doc must not be missing for insert operation") 40 | ); 41 | 42 | try { 43 | BsonDocument insertDoc = BsonDocument.parse( 44 | valueDoc.get(JSON_DOC_FIELD_PATH).asString().getValue() 45 | ); 46 | return new ReplaceOneModel<>( 47 | new BsonDocument(DBCollection.ID_FIELD_NAME, 48 | insertDoc.get(DBCollection.ID_FIELD_NAME)), 49 | insertDoc, 50 | UPDATE_OPTIONS 51 | ); 52 | } catch(Exception exc) { 53 | throw new DataException(exc); 54 | } 55 | 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/mongodb/MongoDbNoOp.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.mongodb; 2 | 3 | import at.grahsl.kafka.connect.mongodb.cdc.CdcOperation; 4 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 5 | import com.mongodb.client.model.WriteModel; 6 | import org.bson.BsonDocument; 7 | 8 | public class MongoDbNoOp implements CdcOperation { 9 | 10 | @Override 11 | public WriteModel perform(SinkDocument doc) { 12 | return null; 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/mongodb/MongoDbUpdate.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.mongodb; 18 | 19 | import at.grahsl.kafka.connect.mongodb.cdc.CdcOperation; 20 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 21 | import com.mongodb.DBCollection; 22 | import com.mongodb.client.model.ReplaceOneModel; 23 | import com.mongodb.client.model.UpdateOneModel; 24 | import com.mongodb.client.model.UpdateOptions; 25 | import com.mongodb.client.model.WriteModel; 26 | import org.apache.kafka.connect.errors.DataException; 27 | import org.bson.BsonDocument; 28 | 29 | public class MongoDbUpdate implements CdcOperation { 30 | 31 | public static final String JSON_DOC_FIELD_PATH = "patch"; 32 | public static final String INTERNAL_OPLOG_FIELD_V = "$v"; 33 | 34 | private static final UpdateOptions UPDATE_OPTIONS = 35 | new UpdateOptions().upsert(true); 36 | 37 | @Override 38 | public WriteModel perform(SinkDocument doc) { 39 | 40 | BsonDocument valueDoc = doc.getValueDoc().orElseThrow( 41 | () -> new DataException("error: value doc must not be missing for update operation") 42 | ); 43 | 44 | try { 45 | 46 | BsonDocument updateDoc = BsonDocument.parse( 47 | valueDoc.getString(JSON_DOC_FIELD_PATH).getValue() 48 | ); 49 | 50 | //Check if the internal "$v" field is contained which was added to the 51 | //oplog format in 3.6+ If so, then we simply remove it for now since 52 | //it's not used by the sink connector at the moment and would break 53 | //CDC-mode based "replication". 54 | updateDoc.remove(INTERNAL_OPLOG_FIELD_V); 55 | 56 | //patch contains full new document for replacement 57 | if(updateDoc.containsKey(DBCollection.ID_FIELD_NAME)) { 58 | BsonDocument filterDoc = 59 | new BsonDocument(DBCollection.ID_FIELD_NAME, 60 | updateDoc.get(DBCollection.ID_FIELD_NAME)); 61 | return new ReplaceOneModel<>(filterDoc, updateDoc, UPDATE_OPTIONS); 62 | } 63 | 64 | //patch contains idempotent change only to update original document with 65 | BsonDocument keyDoc = doc.getKeyDoc().orElseThrow( 66 | () -> new DataException("error: key doc must not be missing for update operation") 67 | ); 68 | 69 | BsonDocument filterDoc = BsonDocument.parse( 70 | "{"+DBCollection.ID_FIELD_NAME+ 71 | ":"+keyDoc.getString(MongoDbHandler.JSON_ID_FIELD_PATH) 72 | .getValue()+"}" 73 | ); 74 | 75 | return new UpdateOneModel<>(filterDoc, updateDoc); 76 | 77 | } catch (DataException exc) { 78 | exc.printStackTrace(); 79 | throw exc; 80 | } 81 | catch (Exception exc) { 82 | exc.printStackTrace(); 83 | throw new DataException(exc.getMessage(),exc); 84 | } 85 | 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/rdbms/RdbmsDelete.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.rdbms; 18 | 19 | import at.grahsl.kafka.connect.mongodb.cdc.CdcOperation; 20 | import at.grahsl.kafka.connect.mongodb.cdc.debezium.OperationType; 21 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 22 | import com.mongodb.client.model.DeleteOneModel; 23 | import com.mongodb.client.model.WriteModel; 24 | import org.apache.kafka.connect.errors.DataException; 25 | import org.bson.BsonDocument; 26 | 27 | public class RdbmsDelete implements CdcOperation { 28 | 29 | @Override 30 | public WriteModel perform(SinkDocument doc) { 31 | 32 | BsonDocument keyDoc = doc.getKeyDoc().orElseThrow( 33 | () -> new DataException("error: key doc must not be missing for delete operation") 34 | ); 35 | 36 | BsonDocument valueDoc = doc.getValueDoc().orElseThrow( 37 | () -> new DataException("error: value doc must not be missing for delete operation") 38 | ); 39 | 40 | try { 41 | BsonDocument filterDoc = RdbmsHandler.generateFilterDoc(keyDoc, valueDoc, OperationType.DELETE); 42 | return new DeleteOneModel<>(filterDoc); 43 | } catch(Exception exc) { 44 | throw new DataException(exc); 45 | } 46 | 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/rdbms/RdbmsHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.rdbms; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | import at.grahsl.kafka.connect.mongodb.cdc.CdcOperation; 21 | import at.grahsl.kafka.connect.mongodb.cdc.debezium.DebeziumCdcHandler; 22 | import at.grahsl.kafka.connect.mongodb.cdc.debezium.OperationType; 23 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 24 | import com.mongodb.DBCollection; 25 | import com.mongodb.client.model.WriteModel; 26 | import org.apache.kafka.connect.errors.DataException; 27 | import org.bson.BsonDocument; 28 | import org.bson.BsonInvalidOperationException; 29 | import org.bson.BsonObjectId; 30 | import org.slf4j.Logger; 31 | import org.slf4j.LoggerFactory; 32 | 33 | import java.util.HashMap; 34 | import java.util.List; 35 | import java.util.Map; 36 | import java.util.Optional; 37 | import java.util.stream.Stream; 38 | 39 | public class RdbmsHandler extends DebeziumCdcHandler { 40 | 41 | public static final String JSON_DOC_BEFORE_FIELD = "before"; 42 | public static final String JSON_DOC_AFTER_FIELD = "after"; 43 | 44 | private static Logger logger = LoggerFactory.getLogger(RdbmsHandler.class); 45 | 46 | public RdbmsHandler(MongoDbSinkConnectorConfig config) { 47 | super(config); 48 | final Map operations = new HashMap<>(); 49 | operations.put(OperationType.CREATE,new RdbmsInsert()); 50 | operations.put(OperationType.READ,new RdbmsInsert()); 51 | operations.put(OperationType.UPDATE,new RdbmsUpdate()); 52 | operations.put(OperationType.DELETE,new RdbmsDelete()); 53 | registerOperations(operations); 54 | } 55 | 56 | public RdbmsHandler(MongoDbSinkConnectorConfig config, 57 | Map operations) { 58 | super(config); 59 | registerOperations(operations); 60 | } 61 | 62 | public RdbmsHandler(MongoDbSinkConnectorConfig config, List supportedTypes) { 63 | super(config); 64 | final Map operations = new HashMap<>(); 65 | Stream.of(OperationType.values()).forEach(ot -> operations.put(ot, new RdbmsNoOp())); 66 | supportedTypes.forEach(ot -> { 67 | switch (ot) { 68 | case CREATE: 69 | case READ: 70 | operations.put(ot,new RdbmsInsert()); 71 | break; 72 | case UPDATE: 73 | operations.put(ot,new RdbmsUpdate()); 74 | break; 75 | case DELETE: 76 | operations.put(ot,new RdbmsDelete()); 77 | break; 78 | } 79 | }); 80 | registerOperations(operations); 81 | } 82 | 83 | 84 | @Override 85 | public Optional> handle(SinkDocument doc) { 86 | 87 | BsonDocument keyDoc = doc.getKeyDoc().orElseGet(BsonDocument::new); 88 | 89 | BsonDocument valueDoc = doc.getValueDoc().orElseGet(BsonDocument::new); 90 | 91 | if (valueDoc.isEmpty()) { 92 | logger.debug("skipping debezium tombstone event for kafka topic compaction"); 93 | return Optional.empty(); 94 | } 95 | 96 | return Optional.ofNullable(getCdcOperation(valueDoc) 97 | .perform(new SinkDocument(keyDoc,valueDoc))); 98 | } 99 | 100 | protected static BsonDocument generateFilterDoc(BsonDocument keyDoc, BsonDocument valueDoc, OperationType opType) { 101 | if (keyDoc.keySet().isEmpty()) { 102 | if (opType.equals(OperationType.CREATE) || opType.equals(OperationType.READ)) { 103 | //create: no PK info in keyDoc -> generate ObjectId 104 | return new BsonDocument(DBCollection.ID_FIELD_NAME,new BsonObjectId()); 105 | } 106 | //update or delete: no PK info in keyDoc -> take everything in 'before' field 107 | try { 108 | BsonDocument filter = valueDoc.getDocument(JSON_DOC_BEFORE_FIELD); 109 | if (filter.isEmpty()) 110 | throw new BsonInvalidOperationException("value doc before field is empty"); 111 | return filter; 112 | } catch(BsonInvalidOperationException exc) { 113 | throw new DataException("error: value doc 'before' field is empty or has invalid type" + 114 | " for update/delete operation which seems severely wrong -> defensive actions taken!",exc); 115 | } 116 | } 117 | //build filter document composed of all PK columns 118 | BsonDocument pk = new BsonDocument(); 119 | for (String f : keyDoc.keySet()) { 120 | pk.put(f,keyDoc.get(f)); 121 | } 122 | return new BsonDocument(DBCollection.ID_FIELD_NAME,pk); 123 | } 124 | 125 | protected static BsonDocument generateUpsertOrReplaceDoc(BsonDocument keyDoc, BsonDocument valueDoc, BsonDocument filterDoc) { 126 | 127 | if (!valueDoc.containsKey(JSON_DOC_AFTER_FIELD) 128 | || valueDoc.get(JSON_DOC_AFTER_FIELD).isNull() 129 | || !valueDoc.get(JSON_DOC_AFTER_FIELD).isDocument() 130 | || valueDoc.getDocument(JSON_DOC_AFTER_FIELD).isEmpty()) { 131 | throw new DataException("error: valueDoc must contain non-empty 'after' field" + 132 | " of type document for insert/update operation"); 133 | } 134 | 135 | BsonDocument upsertDoc = new BsonDocument(); 136 | if(filterDoc.containsKey(DBCollection.ID_FIELD_NAME)) { 137 | upsertDoc.put(DBCollection.ID_FIELD_NAME,filterDoc.get(DBCollection.ID_FIELD_NAME)); 138 | } 139 | 140 | BsonDocument afterDoc = valueDoc.getDocument(JSON_DOC_AFTER_FIELD); 141 | for (String f : afterDoc.keySet()) { 142 | if (!keyDoc.containsKey(f)) { 143 | upsertDoc.put(f,afterDoc.get(f)); 144 | } 145 | } 146 | return upsertDoc; 147 | } 148 | 149 | } 150 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/rdbms/RdbmsInsert.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.rdbms; 18 | 19 | import at.grahsl.kafka.connect.mongodb.cdc.CdcOperation; 20 | import at.grahsl.kafka.connect.mongodb.cdc.debezium.OperationType; 21 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 22 | import com.mongodb.client.model.ReplaceOneModel; 23 | import com.mongodb.client.model.UpdateOptions; 24 | import com.mongodb.client.model.WriteModel; 25 | import org.apache.kafka.connect.errors.DataException; 26 | import org.bson.BsonDocument; 27 | 28 | public class RdbmsInsert implements CdcOperation { 29 | 30 | private static final UpdateOptions UPDATE_OPTIONS = 31 | new UpdateOptions().upsert(true); 32 | 33 | @Override 34 | public WriteModel perform(SinkDocument doc) { 35 | 36 | BsonDocument keyDoc = doc.getKeyDoc().orElseThrow( 37 | () -> new DataException("error: key doc must not be missing for insert operation") 38 | ); 39 | 40 | BsonDocument valueDoc = doc.getValueDoc().orElseThrow( 41 | () -> new DataException("error: value doc must not be missing for insert operation") 42 | ); 43 | 44 | try { 45 | BsonDocument filterDoc = RdbmsHandler.generateFilterDoc(keyDoc, valueDoc, OperationType.CREATE); 46 | BsonDocument upsertDoc = RdbmsHandler.generateUpsertOrReplaceDoc(keyDoc, valueDoc, filterDoc); 47 | return new ReplaceOneModel<>(filterDoc, upsertDoc, UPDATE_OPTIONS); 48 | } catch (Exception exc) { 49 | throw new DataException(exc); 50 | } 51 | 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/rdbms/RdbmsNoOp.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.rdbms; 2 | 3 | import at.grahsl.kafka.connect.mongodb.cdc.CdcOperation; 4 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 5 | import com.mongodb.client.model.WriteModel; 6 | import org.bson.BsonDocument; 7 | 8 | public class RdbmsNoOp implements CdcOperation { 9 | 10 | @Override 11 | public WriteModel perform(SinkDocument doc) { 12 | return null; 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/rdbms/RdbmsUpdate.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.rdbms; 18 | 19 | import at.grahsl.kafka.connect.mongodb.cdc.CdcOperation; 20 | import at.grahsl.kafka.connect.mongodb.cdc.debezium.OperationType; 21 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 22 | import com.mongodb.client.model.ReplaceOneModel; 23 | import com.mongodb.client.model.UpdateOptions; 24 | import com.mongodb.client.model.WriteModel; 25 | import org.apache.kafka.connect.errors.DataException; 26 | import org.bson.BsonDocument; 27 | 28 | public class RdbmsUpdate implements CdcOperation { 29 | 30 | private static final UpdateOptions UPDATE_OPTIONS = 31 | new UpdateOptions().upsert(true); 32 | 33 | @Override 34 | public WriteModel perform(SinkDocument doc) { 35 | 36 | BsonDocument keyDoc = doc.getKeyDoc().orElseThrow( 37 | () -> new DataException("error: key doc must not be missing for update operation") 38 | ); 39 | 40 | BsonDocument valueDoc = doc.getValueDoc().orElseThrow( 41 | () -> new DataException("error: value doc must not be missing for update operation") 42 | ); 43 | 44 | try { 45 | BsonDocument filterDoc = RdbmsHandler.generateFilterDoc(keyDoc, valueDoc, OperationType.UPDATE); 46 | BsonDocument replaceDoc = RdbmsHandler.generateUpsertOrReplaceDoc(keyDoc, valueDoc, filterDoc); 47 | return new ReplaceOneModel<>(filterDoc, replaceDoc, UPDATE_OPTIONS); 48 | } catch (Exception exc) { 49 | throw new DataException(exc); 50 | } 51 | 52 | } 53 | 54 | 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/rdbms/mysql/MysqlHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.rdbms.mysql; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | import at.grahsl.kafka.connect.mongodb.cdc.CdcOperation; 21 | import at.grahsl.kafka.connect.mongodb.cdc.debezium.OperationType; 22 | import at.grahsl.kafka.connect.mongodb.cdc.debezium.rdbms.RdbmsHandler; 23 | 24 | import java.util.List; 25 | import java.util.Map; 26 | 27 | public class MysqlHandler extends RdbmsHandler { 28 | 29 | //NOTE: this class is prepared in case there are 30 | //mysql specific differences to be considered 31 | //and the CDC handling deviates from the standard 32 | //behaviour as implemented in RdbmsHandler.class 33 | 34 | public MysqlHandler(MongoDbSinkConnectorConfig config) { 35 | super(config); 36 | } 37 | 38 | public MysqlHandler(MongoDbSinkConnectorConfig config, Map operations) { 39 | super(config, operations); 40 | } 41 | 42 | public MysqlHandler(MongoDbSinkConnectorConfig config, List supportedTypes) { 43 | super(config, supportedTypes); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/rdbms/postgres/PostgresHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.rdbms.postgres; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | import at.grahsl.kafka.connect.mongodb.cdc.CdcOperation; 21 | import at.grahsl.kafka.connect.mongodb.cdc.debezium.OperationType; 22 | import at.grahsl.kafka.connect.mongodb.cdc.debezium.rdbms.RdbmsHandler; 23 | 24 | import java.util.List; 25 | import java.util.Map; 26 | 27 | public class PostgresHandler extends RdbmsHandler { 28 | 29 | //NOTE: this class is prepared in case there are 30 | //postgres specific differences to be considered 31 | //and the CDC handling deviates from the standard 32 | //behaviour as implemented in RdbmsHandler.class 33 | 34 | public PostgresHandler(MongoDbSinkConnectorConfig config) { 35 | super(config); 36 | } 37 | 38 | public PostgresHandler(MongoDbSinkConnectorConfig config, Map operations) { 39 | super(config, operations); 40 | } 41 | 42 | public PostgresHandler(MongoDbSinkConnectorConfig config, List supportedTypes) { 43 | super(config, supportedTypes); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/FieldConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter; 18 | 19 | import org.apache.kafka.connect.data.Schema; 20 | 21 | public abstract class FieldConverter { 22 | 23 | private final Schema schema; 24 | 25 | public FieldConverter(Schema schema) { 26 | this.schema = schema; 27 | } 28 | 29 | public Schema getSchema() { return schema; } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/JsonRawStringRecordConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter; 18 | 19 | import org.apache.kafka.connect.data.Schema; 20 | import org.apache.kafka.connect.errors.DataException; 21 | import org.bson.BsonDocument; 22 | 23 | public class JsonRawStringRecordConverter implements RecordConverter { 24 | 25 | @Override 26 | public BsonDocument convert(Schema schema, Object value) { 27 | 28 | if(value == null) { 29 | throw new DataException("error: value was null for JSON conversion"); 30 | } 31 | 32 | return BsonDocument.parse((String)value); 33 | 34 | } 35 | } 36 | 37 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/JsonSchemalessRecordConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter; 18 | 19 | import org.apache.kafka.connect.data.Schema; 20 | import org.apache.kafka.connect.errors.DataException; 21 | import org.bson.BsonDocument; 22 | import org.bson.Document; 23 | import org.bson.codecs.BsonValueCodecProvider; 24 | import org.bson.codecs.DocumentCodecProvider; 25 | import org.bson.codecs.ValueCodecProvider; 26 | import org.bson.codecs.configuration.CodecRegistries; 27 | import org.bson.codecs.configuration.CodecRegistry; 28 | 29 | import java.util.Map; 30 | 31 | public class JsonSchemalessRecordConverter implements RecordConverter { 32 | 33 | private CodecRegistry codecRegistry = 34 | CodecRegistries.fromProviders( 35 | new DocumentCodecProvider(), 36 | new BsonValueCodecProvider(), 37 | new ValueCodecProvider() 38 | ); 39 | 40 | @Override 41 | public BsonDocument convert(Schema schema, Object value) { 42 | 43 | if(value == null) { 44 | throw new DataException("error: value was null for JSON conversion"); 45 | } 46 | 47 | return new Document((Map)value).toBsonDocument(null, codecRegistry); 48 | 49 | } 50 | } 51 | 52 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/RecordConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter; 18 | 19 | import org.apache.kafka.connect.data.Schema; 20 | import org.bson.BsonDocument; 21 | 22 | public interface RecordConverter { 23 | 24 | BsonDocument convert(Schema schema, Object value); 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/SinkConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter; 18 | 19 | import org.apache.kafka.connect.data.Schema; 20 | import org.apache.kafka.connect.data.Struct; 21 | import org.apache.kafka.connect.errors.DataException; 22 | import org.apache.kafka.connect.sink.SinkRecord; 23 | import org.bson.BsonDocument; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | import java.util.Map; 28 | 29 | public class SinkConverter { 30 | 31 | private static Logger logger = LoggerFactory.getLogger(SinkConverter.class); 32 | 33 | private RecordConverter schemafulConverter = new AvroJsonSchemafulRecordConverter(); 34 | private RecordConverter schemalessConverter = new JsonSchemalessRecordConverter(); 35 | private RecordConverter rawConverter = new JsonRawStringRecordConverter(); 36 | 37 | public SinkDocument convert(SinkRecord record) { 38 | 39 | logger.debug(record.toString()); 40 | 41 | BsonDocument keyDoc = null; 42 | if(record.key() != null) { 43 | keyDoc = getRecordConverter(record.key(),record.keySchema()) 44 | .convert(record.keySchema(), record.key()); 45 | } 46 | 47 | BsonDocument valueDoc = null; 48 | if(record.value() != null) { 49 | valueDoc = getRecordConverter(record.value(),record.valueSchema()) 50 | .convert(record.valueSchema(), record.value()); 51 | } 52 | 53 | return new SinkDocument(keyDoc, valueDoc); 54 | 55 | } 56 | 57 | private RecordConverter getRecordConverter(Object data, Schema schema) { 58 | 59 | //AVRO or JSON with schema 60 | if(schema != null && data instanceof Struct) { 61 | logger.debug("using schemaful converter"); 62 | return schemafulConverter; 63 | } 64 | 65 | //structured JSON without schema 66 | if(data instanceof Map) { 67 | logger.debug("using schemaless converter"); 68 | return schemalessConverter; 69 | } 70 | 71 | //raw JSON string 72 | if(data instanceof String) { 73 | logger.debug("using raw converter"); 74 | return rawConverter; 75 | } 76 | 77 | throw new DataException("error: no converter present due to unexpected object type " 78 | + data.getClass().getName()); 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/SinkDocument.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter; 18 | 19 | import org.bson.BsonDocument; 20 | 21 | import java.util.Optional; 22 | 23 | public class SinkDocument { 24 | 25 | private final Optional keyDoc; 26 | private final Optional valueDoc; 27 | 28 | public SinkDocument(BsonDocument keyDoc, BsonDocument valueDoc) { 29 | this.keyDoc = Optional.ofNullable(keyDoc); 30 | this.valueDoc = Optional.ofNullable(valueDoc); 31 | } 32 | 33 | public Optional getKeyDoc() { 34 | return keyDoc; 35 | } 36 | 37 | public Optional getValueDoc() { 38 | return valueDoc; 39 | } 40 | 41 | public SinkDocument clone() { 42 | BsonDocument kd = keyDoc.isPresent() ? keyDoc.get().clone() : null; 43 | BsonDocument vd = valueDoc.isPresent() ? valueDoc.get().clone() : null; 44 | return new SinkDocument(kd,vd); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/SinkFieldConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter; 18 | 19 | import org.apache.kafka.connect.data.Schema; 20 | import org.apache.kafka.connect.errors.DataException; 21 | import org.bson.BsonNull; 22 | import org.bson.BsonValue; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | public abstract class SinkFieldConverter extends FieldConverter { 27 | 28 | private static Logger logger = LoggerFactory.getLogger(SinkFieldConverter.class); 29 | 30 | public SinkFieldConverter(Schema schema) { 31 | super(schema); 32 | } 33 | 34 | public abstract BsonValue toBson(Object data); 35 | 36 | public BsonValue toBson(Object data, Schema fieldSchema) { 37 | if(!fieldSchema.isOptional()) { 38 | 39 | if(data == null) 40 | throw new DataException("error: schema not optional but data was null"); 41 | 42 | logger.trace("field not optional and data is '{}'",data.toString()); 43 | return toBson(data); 44 | } 45 | 46 | if(data != null) { 47 | logger.trace("field optional and data is '{}'",data.toString()); 48 | return toBson(data); 49 | } 50 | 51 | if(fieldSchema.defaultValue() != null) { 52 | logger.trace("field optional and no data but default value is '{}'",fieldSchema.defaultValue().toString()); 53 | return toBson(fieldSchema.defaultValue()); 54 | } 55 | 56 | logger.trace("field optional, no data and no default value thus '{}'", BsonNull.VALUE); 57 | return BsonNull.VALUE; 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/types/sink/bson/BooleanFieldConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter.types.sink.bson; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkFieldConverter; 20 | import org.apache.kafka.connect.data.Schema; 21 | import org.bson.BsonBoolean; 22 | import org.bson.BsonValue; 23 | 24 | public class BooleanFieldConverter extends SinkFieldConverter { 25 | 26 | public BooleanFieldConverter() { 27 | super(Schema.BOOLEAN_SCHEMA); 28 | } 29 | 30 | public BsonValue toBson(Object data) { 31 | return new BsonBoolean((Boolean) data); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/types/sink/bson/BytesFieldConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter.types.sink.bson; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkFieldConverter; 20 | import org.apache.kafka.connect.data.Schema; 21 | import org.apache.kafka.connect.errors.DataException; 22 | import org.bson.BsonBinary; 23 | import org.bson.BsonValue; 24 | 25 | import java.nio.ByteBuffer; 26 | 27 | public class BytesFieldConverter extends SinkFieldConverter { 28 | 29 | public BytesFieldConverter() { 30 | super(Schema.BYTES_SCHEMA); 31 | } 32 | 33 | @Override 34 | public BsonValue toBson(Object data) { 35 | 36 | //obviously SinkRecords may contain different types 37 | //to represent byte arrays 38 | if(data instanceof ByteBuffer) 39 | return new BsonBinary(((ByteBuffer) data).array()); 40 | 41 | if(data instanceof byte[]) 42 | return new BsonBinary((byte[])data); 43 | 44 | throw new DataException("error: bytes field conversion failed to due" 45 | + " unexpected object type "+ data.getClass().getName()); 46 | 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/types/sink/bson/Float32FieldConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter.types.sink.bson; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkFieldConverter; 20 | import org.apache.kafka.connect.data.Schema; 21 | import org.bson.BsonDouble; 22 | import org.bson.BsonValue; 23 | 24 | public class Float32FieldConverter extends SinkFieldConverter { 25 | 26 | public Float32FieldConverter() { 27 | super(Schema.FLOAT32_SCHEMA); 28 | } 29 | 30 | @Override 31 | public BsonValue toBson(Object data) { 32 | return new BsonDouble((Float) data); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/types/sink/bson/Float64FieldConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter.types.sink.bson; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkFieldConverter; 20 | import org.apache.kafka.connect.data.Schema; 21 | import org.bson.BsonDouble; 22 | import org.bson.BsonValue; 23 | 24 | public class Float64FieldConverter extends SinkFieldConverter { 25 | 26 | public Float64FieldConverter() { 27 | super(Schema.FLOAT64_SCHEMA); 28 | } 29 | 30 | @Override 31 | public BsonValue toBson(Object data) { 32 | return new BsonDouble((Double) data); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/types/sink/bson/Int16FieldConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter.types.sink.bson; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkFieldConverter; 20 | import org.apache.kafka.connect.data.Schema; 21 | import org.bson.BsonInt32; 22 | import org.bson.BsonValue; 23 | 24 | public class Int16FieldConverter extends SinkFieldConverter { 25 | 26 | public Int16FieldConverter() { 27 | super(Schema.INT16_SCHEMA); 28 | } 29 | 30 | @Override 31 | public BsonValue toBson(Object data) { 32 | return new BsonInt32(((Short) data).intValue()); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/types/sink/bson/Int32FieldConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter.types.sink.bson; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkFieldConverter; 20 | import org.apache.kafka.connect.data.Schema; 21 | import org.bson.BsonInt32; 22 | import org.bson.BsonValue; 23 | 24 | public class Int32FieldConverter extends SinkFieldConverter { 25 | 26 | public Int32FieldConverter() { 27 | super(Schema.INT32_SCHEMA); 28 | } 29 | 30 | @Override 31 | public BsonValue toBson(Object data) { 32 | return new BsonInt32((Integer) data); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/types/sink/bson/Int64FieldConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter.types.sink.bson; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkFieldConverter; 20 | import org.apache.kafka.connect.data.Schema; 21 | import org.bson.BsonInt64; 22 | import org.bson.BsonValue; 23 | 24 | public class Int64FieldConverter extends SinkFieldConverter { 25 | 26 | public Int64FieldConverter() { 27 | super(Schema.INT64_SCHEMA); 28 | } 29 | 30 | @Override 31 | public BsonValue toBson(Object data) { 32 | return new BsonInt64((Long) data); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/types/sink/bson/Int8FieldConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter.types.sink.bson; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkFieldConverter; 20 | import org.apache.kafka.connect.data.Schema; 21 | import org.bson.BsonInt32; 22 | import org.bson.BsonValue; 23 | 24 | public class Int8FieldConverter extends SinkFieldConverter { 25 | 26 | public Int8FieldConverter() { 27 | super(Schema.INT8_SCHEMA); 28 | } 29 | 30 | @Override 31 | public BsonValue toBson(Object data) { 32 | return new BsonInt32(((Byte) data).intValue()); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/types/sink/bson/StringFieldConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter.types.sink.bson; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkFieldConverter; 20 | import org.apache.kafka.connect.data.Schema; 21 | import org.bson.BsonString; 22 | import org.bson.BsonValue; 23 | 24 | public class StringFieldConverter extends SinkFieldConverter { 25 | 26 | public StringFieldConverter() { 27 | super(Schema.STRING_SCHEMA); 28 | } 29 | 30 | @Override 31 | public BsonValue toBson(Object data) { 32 | return new BsonString((String) data); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/types/sink/bson/logical/DateFieldConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter.types.sink.bson.logical; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkFieldConverter; 20 | import org.apache.kafka.connect.data.Date; 21 | import org.bson.BsonDateTime; 22 | import org.bson.BsonValue; 23 | 24 | public class DateFieldConverter extends SinkFieldConverter { 25 | 26 | public DateFieldConverter() { 27 | super(Date.SCHEMA); 28 | } 29 | 30 | @Override 31 | public BsonValue toBson(Object data) { 32 | return new BsonDateTime(((java.util.Date)data).getTime()); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/types/sink/bson/logical/DecimalFieldConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter.types.sink.bson.logical; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkFieldConverter; 20 | import org.apache.kafka.connect.data.Decimal; 21 | import org.apache.kafka.connect.errors.DataException; 22 | import org.bson.BsonDecimal128; 23 | import org.bson.BsonDouble; 24 | import org.bson.BsonValue; 25 | import org.bson.types.Decimal128; 26 | 27 | import java.math.BigDecimal; 28 | 29 | public class DecimalFieldConverter extends SinkFieldConverter { 30 | 31 | public enum Format { 32 | DECIMAL128, //needs MongoDB v3.4+ 33 | LEGACYDOUBLE //results in double approximation 34 | } 35 | 36 | private Format format; 37 | 38 | public DecimalFieldConverter() { 39 | super(Decimal.schema(0)); 40 | this.format = Format.DECIMAL128; 41 | } 42 | 43 | public DecimalFieldConverter(Format format) { 44 | super(Decimal.schema(0)); 45 | this.format = format; 46 | } 47 | 48 | @Override 49 | public BsonValue toBson(Object data) { 50 | 51 | if(data instanceof BigDecimal) { 52 | if(format.equals(Format.DECIMAL128)) 53 | return new BsonDecimal128(new Decimal128((BigDecimal)data)); 54 | 55 | if(format.equals(Format.LEGACYDOUBLE)) 56 | return new BsonDouble(((BigDecimal)data).doubleValue()); 57 | } 58 | 59 | throw new DataException("error: decimal conversion not possible when data is" 60 | + " of type "+data.getClass().getName() + " and format is "+format); 61 | 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/types/sink/bson/logical/TimeFieldConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter.types.sink.bson.logical; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkFieldConverter; 20 | import org.apache.kafka.connect.data.Time; 21 | import org.bson.BsonDateTime; 22 | import org.bson.BsonValue; 23 | 24 | public class TimeFieldConverter extends SinkFieldConverter { 25 | 26 | public TimeFieldConverter() { 27 | super(Time.SCHEMA); 28 | } 29 | 30 | @Override 31 | public BsonValue toBson(Object data) { 32 | return new BsonDateTime(((java.util.Date)data).getTime()); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/converter/types/sink/bson/logical/TimestampFieldConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter.types.sink.bson.logical; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkFieldConverter; 20 | import org.apache.kafka.connect.data.Timestamp; 21 | import org.bson.BsonDateTime; 22 | import org.bson.BsonValue; 23 | 24 | public class TimestampFieldConverter extends SinkFieldConverter { 25 | 26 | public TimestampFieldConverter() { 27 | super(Timestamp.SCHEMA); 28 | } 29 | 30 | @Override 31 | public BsonValue toBson(Object data) { 32 | return new BsonDateTime(((java.util.Date)data).getTime()); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/BlacklistKeyProjector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 21 | import at.grahsl.kafka.connect.mongodb.processor.field.projection.BlacklistProjector; 22 | import org.apache.kafka.connect.sink.SinkRecord; 23 | 24 | import java.util.Set; 25 | import java.util.function.Predicate; 26 | 27 | public class BlacklistKeyProjector extends BlacklistProjector { 28 | 29 | private Predicate predicate; 30 | 31 | public BlacklistKeyProjector(MongoDbSinkConnectorConfig config,String collection) { 32 | this(config,config.getKeyProjectionList(collection), 33 | cfg -> cfg.isUsingBlacklistKeyProjection(collection), collection); 34 | } 35 | 36 | public BlacklistKeyProjector(MongoDbSinkConnectorConfig config, Set fields, 37 | Predicate predicate, String collection) { 38 | super(config,collection); 39 | this.fields = fields; 40 | this.predicate = predicate; 41 | } 42 | 43 | @Override 44 | public void process(SinkDocument doc, SinkRecord orig) { 45 | 46 | if(predicate.test(getConfig())) { 47 | doc.getKeyDoc().ifPresent(kd -> 48 | fields.forEach(f -> doProjection(f,kd)) 49 | ); 50 | } 51 | 52 | getNext().ifPresent(pp -> pp.process(doc,orig)); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/BlacklistValueProjector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 21 | import at.grahsl.kafka.connect.mongodb.processor.field.projection.BlacklistProjector; 22 | import org.apache.kafka.connect.sink.SinkRecord; 23 | 24 | import java.util.Set; 25 | import java.util.function.Predicate; 26 | 27 | public class BlacklistValueProjector extends BlacklistProjector { 28 | 29 | private Predicate predicate; 30 | 31 | public BlacklistValueProjector(MongoDbSinkConnectorConfig config,String collection) { 32 | this(config,config.getValueProjectionList(collection), 33 | cfg -> cfg.isUsingBlacklistValueProjection(collection),collection); 34 | } 35 | 36 | public BlacklistValueProjector(MongoDbSinkConnectorConfig config, Set fields, 37 | Predicate predicate, String collection) { 38 | super(config,collection); 39 | this.fields = fields; 40 | this.predicate = predicate; 41 | } 42 | 43 | @Override 44 | public void process(SinkDocument doc, SinkRecord orig) { 45 | 46 | if(predicate.test(getConfig())) { 47 | doc.getValueDoc().ifPresent(vd -> 48 | fields.forEach(f -> doProjection(f,vd)) 49 | ); 50 | } 51 | 52 | getNext().ifPresent(pp -> pp.process(doc,orig)); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/DocumentIdAdder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 21 | import at.grahsl.kafka.connect.mongodb.processor.id.strategy.IdStrategy; 22 | import com.mongodb.DBCollection; 23 | import org.apache.kafka.connect.sink.SinkRecord; 24 | 25 | public class DocumentIdAdder extends PostProcessor { 26 | 27 | protected final IdStrategy idStrategy; 28 | 29 | public DocumentIdAdder(MongoDbSinkConnectorConfig config, String collection) { 30 | this(config,config.getIdStrategy(collection),collection); 31 | } 32 | 33 | public DocumentIdAdder(MongoDbSinkConnectorConfig config, IdStrategy idStrategy, String collection) { 34 | super(config,collection); 35 | this.idStrategy = idStrategy; 36 | } 37 | 38 | @Override 39 | public void process(SinkDocument doc, SinkRecord orig) { 40 | doc.getValueDoc().ifPresent(vd -> 41 | vd.append(DBCollection.ID_FIELD_NAME, idStrategy.generateId(doc,orig)) 42 | ); 43 | getNext().ifPresent(pp -> pp.process(doc, orig)); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/KafkaMetaAdder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 21 | import org.apache.kafka.connect.sink.SinkRecord; 22 | import org.bson.BsonInt64; 23 | import org.bson.BsonString; 24 | 25 | public class KafkaMetaAdder extends PostProcessor { 26 | 27 | public static final String KAFKA_META_DATA = "topic-partition-offset"; 28 | 29 | public KafkaMetaAdder(MongoDbSinkConnectorConfig config,String collection) { 30 | super(config,collection); 31 | } 32 | 33 | @Override 34 | public void process(SinkDocument doc, SinkRecord orig) { 35 | 36 | doc.getValueDoc().ifPresent(vd -> { 37 | vd.put(KAFKA_META_DATA, new BsonString(orig.topic() 38 | + "-" + orig.kafkaPartition() + "-" + orig.kafkaOffset())); 39 | vd.put(orig.timestampType().name(), new BsonInt64(orig.timestamp())); 40 | }); 41 | 42 | getNext().ifPresent(pp -> pp.process(doc, orig)); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/PostProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 21 | import org.apache.kafka.connect.sink.SinkRecord; 22 | 23 | import java.util.Optional; 24 | 25 | public abstract class PostProcessor { 26 | 27 | private final MongoDbSinkConnectorConfig config; 28 | private Optional next = Optional.empty(); 29 | private final String collection; 30 | 31 | public PostProcessor(MongoDbSinkConnectorConfig config, String collection) { 32 | this.config = config; 33 | this.collection = collection; 34 | } 35 | 36 | public PostProcessor chain(PostProcessor next) { 37 | // intentionally throws NPE here if someone 38 | // tries to be 'smart' by chaining with null 39 | this.next = Optional.of(next); 40 | return this.next.get(); 41 | } 42 | 43 | public abstract void process(SinkDocument doc, SinkRecord orig); 44 | 45 | public MongoDbSinkConnectorConfig getConfig() { 46 | return this.config; 47 | } 48 | 49 | public Optional getNext() { 50 | return this.next; 51 | } 52 | 53 | public String getCollection() { 54 | return this.collection; 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/WhitelistKeyProjector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 21 | import at.grahsl.kafka.connect.mongodb.processor.field.projection.WhitelistProjector; 22 | import org.apache.kafka.connect.sink.SinkRecord; 23 | 24 | import java.util.Set; 25 | import java.util.function.Predicate; 26 | 27 | public class WhitelistKeyProjector extends WhitelistProjector { 28 | 29 | private Predicate predicate; 30 | 31 | public WhitelistKeyProjector(MongoDbSinkConnectorConfig config,String collection) { 32 | this(config, config.getKeyProjectionList(collection), 33 | cfg -> cfg.isUsingWhitelistKeyProjection(collection),collection); 34 | } 35 | 36 | public WhitelistKeyProjector(MongoDbSinkConnectorConfig config, Set fields, 37 | Predicate predicate, String collection) { 38 | super(config,collection); 39 | this.fields = fields; 40 | this.predicate = predicate; 41 | } 42 | 43 | @Override 44 | public void process(SinkDocument doc, SinkRecord orig) { 45 | 46 | if(predicate.test(getConfig())) { 47 | doc.getKeyDoc().ifPresent(kd -> 48 | doProjection("", kd) 49 | ); 50 | } 51 | 52 | getNext().ifPresent(pp -> pp.process(doc,orig)); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/WhitelistValueProjector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 21 | import at.grahsl.kafka.connect.mongodb.processor.field.projection.WhitelistProjector; 22 | import org.apache.kafka.connect.sink.SinkRecord; 23 | 24 | import java.util.Set; 25 | import java.util.function.Predicate; 26 | 27 | public class WhitelistValueProjector extends WhitelistProjector { 28 | 29 | private Predicate predicate; 30 | 31 | public WhitelistValueProjector(MongoDbSinkConnectorConfig config,String collection) { 32 | this(config, config.getValueProjectionList(collection), 33 | cfg -> cfg.isUsingWhitelistValueProjection(collection),collection); 34 | } 35 | 36 | public WhitelistValueProjector(MongoDbSinkConnectorConfig config, Set fields, 37 | Predicate predicate, String collection) { 38 | super(config,collection); 39 | this.fields = fields; 40 | this.predicate = predicate; 41 | } 42 | 43 | @Override 44 | public void process(SinkDocument doc, SinkRecord orig) { 45 | 46 | if(predicate.test(getConfig())) { 47 | doc.getValueDoc().ifPresent(vd -> 48 | doProjection("", vd) 49 | ); 50 | } 51 | 52 | getNext().ifPresent(pp -> pp.process(doc,orig)); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/field/projection/BlacklistProjector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.field.projection; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | import com.mongodb.DBCollection; 21 | import org.bson.BsonArray; 22 | import org.bson.BsonDocument; 23 | import org.bson.BsonValue; 24 | 25 | import java.util.Iterator; 26 | import java.util.Map; 27 | import java.util.Set; 28 | 29 | public abstract class BlacklistProjector extends FieldProjector { 30 | 31 | public BlacklistProjector(MongoDbSinkConnectorConfig config, String collection) { 32 | this(config,config.getValueProjectionList(collection),collection); 33 | } 34 | 35 | public BlacklistProjector(MongoDbSinkConnectorConfig config, 36 | Set fields, String collection) { 37 | super(config,collection); 38 | this.fields = fields; 39 | } 40 | 41 | @Override 42 | protected void doProjection(String field, BsonDocument doc) { 43 | 44 | if(!field.contains(FieldProjector.SUB_FIELD_DOT_SEPARATOR)) { 45 | 46 | if(field.equals(FieldProjector.SINGLE_WILDCARD) 47 | || field.equals(FieldProjector.DOUBLE_WILDCARD)) { 48 | handleWildcard(field,"",doc); 49 | return; 50 | } 51 | 52 | //NOTE: never try to remove the _id field 53 | if(!field.equals(DBCollection.ID_FIELD_NAME)) 54 | doc.remove(field); 55 | 56 | return; 57 | } 58 | 59 | int dotIdx = field.indexOf(FieldProjector.SUB_FIELD_DOT_SEPARATOR); 60 | String firstPart = field.substring(0,dotIdx); 61 | String otherParts = field.length() >= dotIdx 62 | ? field.substring(dotIdx+1) : ""; 63 | 64 | if(firstPart.equals(FieldProjector.SINGLE_WILDCARD) 65 | || firstPart.equals(FieldProjector.DOUBLE_WILDCARD)) { 66 | handleWildcard(firstPart,otherParts,doc); 67 | return; 68 | } 69 | 70 | BsonValue value = doc.get(firstPart); 71 | if(value != null) { 72 | if(value.isDocument()) { 73 | doProjection(otherParts, (BsonDocument)value); 74 | } 75 | if(value.isArray()) { 76 | BsonArray values = (BsonArray)value; 77 | for(BsonValue v : values.getValues()) { 78 | if(v != null && v.isDocument()) { 79 | doProjection(otherParts,(BsonDocument)v); 80 | } 81 | } 82 | } 83 | } 84 | 85 | } 86 | 87 | private void handleWildcard(String firstPart, String otherParts, BsonDocument doc) { 88 | Iterator> iter = doc.entrySet().iterator(); 89 | while(iter.hasNext()) { 90 | Map.Entry entry = iter.next(); 91 | BsonValue value = entry.getValue(); 92 | 93 | //NOTE: never try to remove the _id field 94 | if(entry.getKey().equals(DBCollection.ID_FIELD_NAME)) 95 | continue; 96 | 97 | if(firstPart.equals(FieldProjector.DOUBLE_WILDCARD)) { 98 | iter.remove(); 99 | } 100 | 101 | if(firstPart.equals(FieldProjector.SINGLE_WILDCARD)) { 102 | if(!value.isDocument()) { 103 | iter.remove(); 104 | } else { 105 | if(!otherParts.isEmpty()) { 106 | doProjection(otherParts, (BsonDocument)value); 107 | } 108 | } 109 | } 110 | } 111 | } 112 | 113 | } 114 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/field/projection/FieldProjector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.field.projection; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | import at.grahsl.kafka.connect.mongodb.processor.PostProcessor; 21 | import org.bson.BsonDocument; 22 | 23 | import java.util.Set; 24 | 25 | public abstract class FieldProjector extends PostProcessor { 26 | 27 | public static final String SINGLE_WILDCARD = "*"; 28 | public static final String DOUBLE_WILDCARD = "**"; 29 | public static final String SUB_FIELD_DOT_SEPARATOR = "."; 30 | 31 | protected Set fields; 32 | 33 | public FieldProjector(MongoDbSinkConnectorConfig config,String collection) { 34 | super(config,collection); 35 | } 36 | 37 | protected abstract void doProjection(String field, BsonDocument doc); 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/field/projection/WhitelistProjector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.field.projection; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | import com.mongodb.DBCollection; 21 | import org.bson.BsonArray; 22 | import org.bson.BsonDocument; 23 | import org.bson.BsonValue; 24 | 25 | import java.util.Arrays; 26 | import java.util.Iterator; 27 | import java.util.Map; 28 | import java.util.Set; 29 | 30 | public abstract class WhitelistProjector extends FieldProjector { 31 | 32 | public WhitelistProjector(MongoDbSinkConnectorConfig config,String collection) { 33 | this(config, config.getValueProjectionList(collection), collection); 34 | } 35 | 36 | public WhitelistProjector(MongoDbSinkConnectorConfig config, 37 | Set fields, String collection) { 38 | super(config,collection); 39 | this.fields = fields; 40 | } 41 | 42 | @Override 43 | protected void doProjection(String field, BsonDocument doc) { 44 | 45 | //special case short circuit check for '**' pattern 46 | //this is essentially the same as not using 47 | //whitelisting at all but instead take the full record 48 | if(fields.contains(FieldProjector.DOUBLE_WILDCARD)) { 49 | return; 50 | } 51 | 52 | Iterator> iter = doc.entrySet().iterator(); 53 | while(iter.hasNext()) { 54 | Map.Entry entry = iter.next(); 55 | 56 | String key = field.isEmpty() ? entry.getKey() 57 | : field + FieldProjector.SUB_FIELD_DOT_SEPARATOR + entry.getKey(); 58 | BsonValue value = entry.getValue(); 59 | 60 | if(!fields.contains(key) 61 | //NOTE: always keep the _id field 62 | && !key.equals(DBCollection.ID_FIELD_NAME)) { 63 | 64 | if(!checkForWildcardMatch(key)) 65 | iter.remove(); 66 | 67 | } 68 | 69 | if(value != null) { 70 | if(value.isDocument()) { 71 | //short circuit check to avoid recursion 72 | //if 'key.**' pattern exists 73 | String matchDoubleWildCard = key 74 | + FieldProjector.SUB_FIELD_DOT_SEPARATOR 75 | + FieldProjector.DOUBLE_WILDCARD; 76 | if(!fields.contains(matchDoubleWildCard)) { 77 | doProjection(key, (BsonDocument)value); 78 | } 79 | } 80 | if(value.isArray()) { 81 | BsonArray values = (BsonArray)value; 82 | for(BsonValue v : values.getValues()) { 83 | if(v != null && v.isDocument()) { 84 | doProjection(key,(BsonDocument)v); 85 | } 86 | } 87 | } 88 | } 89 | 90 | } 91 | } 92 | 93 | private boolean checkForWildcardMatch(String key) { 94 | 95 | String[] keyParts = key.split("\\"+FieldProjector.SUB_FIELD_DOT_SEPARATOR); 96 | String[] pattern = new String[keyParts.length]; 97 | Arrays.fill(pattern,FieldProjector.SINGLE_WILDCARD); 98 | 99 | for(int c=(int)Math.pow(2, keyParts.length)-1;c >= 0;c--) { 100 | 101 | int mask = 0x1; 102 | for(int d = keyParts.length-1;d >= 0;d--) { 103 | if((c & mask) != 0x0) { 104 | pattern[d] = keyParts[d]; 105 | } 106 | mask <<= 1; 107 | } 108 | 109 | if(fields.contains(String.join(FieldProjector.SUB_FIELD_DOT_SEPARATOR,pattern))) 110 | return true; 111 | 112 | Arrays.fill(pattern,FieldProjector.SINGLE_WILDCARD); 113 | } 114 | 115 | return false; 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/field/renaming/FieldnameMapping.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.field.renaming; 18 | 19 | public class FieldnameMapping { 20 | 21 | public String oldName; 22 | public String newName; 23 | 24 | public FieldnameMapping() {} 25 | 26 | public FieldnameMapping(String oldName, String newName) { 27 | this.oldName = oldName; 28 | this.newName = newName; 29 | } 30 | 31 | @Override 32 | public String toString() { 33 | return "FieldnameMapping{" + 34 | "oldName='" + oldName + '\'' + 35 | ", newName='" + newName + '\'' + 36 | '}'; 37 | } 38 | 39 | @Override 40 | public boolean equals(Object o) { 41 | if (this == o) return true; 42 | if (o == null || getClass() != o.getClass()) return false; 43 | 44 | FieldnameMapping that = (FieldnameMapping) o; 45 | 46 | if (oldName != null ? !oldName.equals(that.oldName) : that.oldName != null) return false; 47 | return newName != null ? newName.equals(that.newName) : that.newName == null; 48 | } 49 | 50 | @Override 51 | public int hashCode() { 52 | int result = oldName != null ? oldName.hashCode() : 0; 53 | result = 31 * result + (newName != null ? newName.hashCode() : 0); 54 | return result; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/field/renaming/RegExpSettings.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.field.renaming; 18 | 19 | public class RegExpSettings { 20 | 21 | public String regexp; 22 | public String pattern; 23 | public String replace; 24 | 25 | public RegExpSettings() {} 26 | 27 | public RegExpSettings(String regexp, String pattern, String replace) { 28 | this.regexp = regexp; 29 | this.pattern = pattern; 30 | this.replace = replace; 31 | } 32 | 33 | @Override 34 | public String toString() { 35 | return "RegExpSettings{" + 36 | "regexp='" + regexp + '\'' + 37 | ", pattern='" + pattern + '\'' + 38 | ", replace='" + replace + '\'' + 39 | '}'; 40 | } 41 | 42 | @Override 43 | public boolean equals(Object o) { 44 | if (this == o) return true; 45 | if (o == null || getClass() != o.getClass()) return false; 46 | 47 | RegExpSettings that = (RegExpSettings) o; 48 | 49 | if (regexp != null ? !regexp.equals(that.regexp) : that.regexp != null) return false; 50 | if (pattern != null ? !pattern.equals(that.pattern) : that.pattern != null) return false; 51 | return replace != null ? replace.equals(that.replace) : that.replace == null; 52 | } 53 | 54 | @Override 55 | public int hashCode() { 56 | int result = regexp != null ? regexp.hashCode() : 0; 57 | result = 31 * result + (pattern != null ? pattern.hashCode() : 0); 58 | result = 31 * result + (replace != null ? replace.hashCode() : 0); 59 | return result; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/field/renaming/RenameByMapping.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.field.renaming; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | 21 | import java.util.Map; 22 | 23 | public class RenameByMapping extends Renamer { 24 | 25 | private Map fieldMappings; 26 | 27 | public RenameByMapping(MongoDbSinkConnectorConfig config, String collection) { 28 | super(config,collection); 29 | this.fieldMappings = config.parseRenameFieldnameMappings(collection); 30 | } 31 | 32 | public RenameByMapping(MongoDbSinkConnectorConfig config, 33 | Map fieldMappings, String collection) { 34 | super(config,collection); 35 | this.fieldMappings = fieldMappings; 36 | } 37 | 38 | @Override 39 | protected boolean isActive() { 40 | return !fieldMappings.isEmpty(); 41 | } 42 | 43 | protected String renamed(String path, String name) { 44 | String newName = fieldMappings.get(path+SUB_FIELD_DOT_SEPARATOR+name); 45 | return newName != null ? newName : name; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/field/renaming/RenameByRegExp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.field.renaming; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | 21 | import java.util.Map; 22 | 23 | public class RenameByRegExp extends Renamer { 24 | 25 | private Map fieldRegExps; 26 | 27 | public static class PatternReplace { 28 | 29 | public final String pattern; 30 | public final String replace; 31 | 32 | public PatternReplace(String pattern, String replace) { 33 | this.pattern = pattern; 34 | this.replace = replace; 35 | } 36 | 37 | @Override 38 | public String toString() { 39 | return "PatternReplace{" + 40 | "pattern='" + pattern + '\'' + 41 | ", replace='" + replace + '\'' + 42 | '}'; 43 | } 44 | 45 | @Override 46 | public boolean equals(Object o) { 47 | if (this == o) return true; 48 | if (o == null || getClass() != o.getClass()) return false; 49 | 50 | PatternReplace that = (PatternReplace) o; 51 | 52 | if (pattern != null ? !pattern.equals(that.pattern) : that.pattern != null) return false; 53 | return replace != null ? replace.equals(that.replace) : that.replace == null; 54 | } 55 | 56 | @Override 57 | public int hashCode() { 58 | int result = pattern != null ? pattern.hashCode() : 0; 59 | result = 31 * result + (replace != null ? replace.hashCode() : 0); 60 | return result; 61 | } 62 | } 63 | 64 | public RenameByRegExp(MongoDbSinkConnectorConfig config, String collection) { 65 | super(config,collection); 66 | this.fieldRegExps = config.parseRenameRegExpSettings(collection); 67 | } 68 | 69 | public RenameByRegExp(MongoDbSinkConnectorConfig config, 70 | Map fieldRegExps, String collection) { 71 | super(config,collection); 72 | this.fieldRegExps = fieldRegExps; 73 | } 74 | 75 | @Override 76 | protected boolean isActive() { 77 | return !fieldRegExps.isEmpty(); 78 | } 79 | 80 | protected String renamed(String path, String name) { 81 | String newName = name; 82 | for(Map.Entry e : fieldRegExps.entrySet()) { 83 | if((path+SUB_FIELD_DOT_SEPARATOR+name).matches(e.getKey())) { 84 | newName = newName.replaceAll(e.getValue().pattern,e.getValue().replace); 85 | } 86 | } 87 | return newName; 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/field/renaming/Renamer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.field.renaming; 18 | 19 | import at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig; 20 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 21 | import at.grahsl.kafka.connect.mongodb.processor.PostProcessor; 22 | import org.apache.kafka.connect.sink.SinkRecord; 23 | import org.bson.BsonDocument; 24 | import org.bson.BsonValue; 25 | 26 | import java.util.Iterator; 27 | import java.util.LinkedHashMap; 28 | import java.util.Map; 29 | 30 | public abstract class Renamer extends PostProcessor { 31 | 32 | //PATH PREFIXES used as a simple means to 33 | //distinguish whether we operate on key or value 34 | //structure of a record and match name mappings 35 | //or regexp patterns accordingly 36 | public static final String PATH_PREFIX_KEY = "key"; 37 | public static final String PATH_PREFIX_VALUE = "value"; 38 | 39 | public static final String SUB_FIELD_DOT_SEPARATOR = "."; 40 | 41 | public Renamer(MongoDbSinkConnectorConfig config,String collection) { 42 | super(config,collection); 43 | } 44 | 45 | protected abstract String renamed(String path, String name); 46 | 47 | protected abstract boolean isActive(); 48 | 49 | protected void doRenaming(String field, BsonDocument doc) { 50 | Map temp = new LinkedHashMap<>(); 51 | 52 | Iterator> iter = doc.entrySet().iterator(); 53 | while(iter.hasNext()) { 54 | Map.Entry entry = iter.next(); 55 | String oldKey = entry.getKey(); 56 | BsonValue value = entry.getValue(); 57 | String newKey = renamed(field, oldKey); 58 | 59 | if(!oldKey.equals(newKey)) { 60 | //IF NEW KEY ALREADY EXISTS WE THEN DON'T RENAME 61 | //AS IT WOULD CAUSE OTHER DATA TO BE SILENTLY OVERWRITTEN 62 | //WHICH IS ALMOST NEVER WHAT YOU WANT 63 | //MAYBE LOG WARNING HERE? 64 | doc.computeIfAbsent(newKey, k -> temp.putIfAbsent(k,value)); 65 | iter.remove(); 66 | } 67 | 68 | if(value instanceof BsonDocument) { 69 | String pathToField = field+SUB_FIELD_DOT_SEPARATOR+newKey; 70 | doRenaming(pathToField, (BsonDocument)value); 71 | } 72 | } 73 | 74 | doc.putAll(temp); 75 | } 76 | 77 | @Override 78 | public void process(SinkDocument doc, SinkRecord orig) { 79 | 80 | if(isActive()) { 81 | doc.getKeyDoc().ifPresent(kd -> doRenaming(PATH_PREFIX_KEY, kd)); 82 | doc.getValueDoc().ifPresent(vd -> doRenaming(PATH_PREFIX_VALUE, vd)); 83 | } 84 | 85 | getNext().ifPresent(pp -> pp.process(doc, orig)); 86 | 87 | } 88 | 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/id/strategy/BsonOidStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.id.strategy; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 20 | import org.apache.kafka.connect.sink.SinkRecord; 21 | import org.bson.BsonObjectId; 22 | import org.bson.BsonValue; 23 | import org.bson.types.ObjectId; 24 | 25 | public class BsonOidStrategy implements IdStrategy { 26 | 27 | @Override 28 | public BsonValue generateId(SinkDocument doc, SinkRecord orig) { 29 | return new BsonObjectId(ObjectId.get()); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/id/strategy/FullKeyStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.id.strategy; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 20 | import org.apache.kafka.connect.sink.SinkRecord; 21 | import org.bson.BsonDocument; 22 | import org.bson.BsonValue; 23 | 24 | public class FullKeyStrategy implements IdStrategy { 25 | 26 | @Override 27 | public BsonValue generateId(SinkDocument doc, SinkRecord orig) { 28 | //NOTE: If there is no key doc present the strategy 29 | //simply returns an empty BSON document per default. 30 | return doc.getKeyDoc().orElseGet(() -> new BsonDocument()); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/id/strategy/IdStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.id.strategy; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 20 | import org.apache.kafka.connect.sink.SinkRecord; 21 | import org.bson.BsonValue; 22 | 23 | public interface IdStrategy { 24 | 25 | BsonValue generateId(SinkDocument doc, SinkRecord orig); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/id/strategy/KafkaMetaDataStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.id.strategy; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 20 | import org.apache.kafka.connect.sink.SinkRecord; 21 | import org.bson.BsonString; 22 | import org.bson.BsonValue; 23 | 24 | public class KafkaMetaDataStrategy implements IdStrategy { 25 | 26 | public static final String DELIMITER = "#"; 27 | 28 | @Override 29 | public BsonValue generateId(SinkDocument doc, SinkRecord orig) { 30 | 31 | return new BsonString(orig.topic() 32 | + DELIMITER + orig.kafkaPartition() 33 | + DELIMITER + orig.kafkaOffset()); 34 | 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/id/strategy/PartialKeyStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.id.strategy; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 20 | import at.grahsl.kafka.connect.mongodb.processor.field.projection.FieldProjector; 21 | import org.apache.kafka.connect.sink.SinkRecord; 22 | import org.bson.BsonDocument; 23 | import org.bson.BsonValue; 24 | 25 | public class PartialKeyStrategy implements IdStrategy { 26 | 27 | private FieldProjector fieldProjector; 28 | 29 | public PartialKeyStrategy(FieldProjector fieldProjector) { 30 | this.fieldProjector = fieldProjector; 31 | } 32 | 33 | @Override 34 | public BsonValue generateId(SinkDocument doc, SinkRecord orig) { 35 | 36 | fieldProjector.process(doc,orig); 37 | //NOTE: If there is no key doc present the strategy 38 | //simply returns an empty BSON document per default. 39 | return doc.getKeyDoc().orElseGet(() -> new BsonDocument()); 40 | 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/id/strategy/PartialValueStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.id.strategy; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 20 | import at.grahsl.kafka.connect.mongodb.processor.field.projection.FieldProjector; 21 | import org.apache.kafka.connect.sink.SinkRecord; 22 | import org.bson.BsonDocument; 23 | import org.bson.BsonValue; 24 | 25 | public class PartialValueStrategy implements IdStrategy { 26 | 27 | private FieldProjector fieldProjector; 28 | 29 | public PartialValueStrategy(FieldProjector fieldProjector) { 30 | this.fieldProjector = fieldProjector; 31 | } 32 | 33 | @Override 34 | public BsonValue generateId(SinkDocument doc, SinkRecord orig) { 35 | 36 | //NOTE: this has to operate on a clone because 37 | //otherwise it would interfere with further projections 38 | //happening later in the chain e.g. for value fields 39 | SinkDocument clone = doc.clone(); 40 | fieldProjector.process(clone,orig); 41 | //NOTE: If there is no key doc present the strategy 42 | //simply returns an empty BSON document per default. 43 | return clone.getValueDoc().orElseGet(() -> new BsonDocument()); 44 | 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/id/strategy/ProvidedInKeyStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.id.strategy; 18 | 19 | public class ProvidedInKeyStrategy extends ProvidedStrategy { 20 | 21 | public ProvidedInKeyStrategy() { 22 | super(ProvidedIn.KEY); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/id/strategy/ProvidedInValueStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.id.strategy; 18 | 19 | public class ProvidedInValueStrategy extends ProvidedStrategy { 20 | 21 | public ProvidedInValueStrategy() { 22 | super(ProvidedIn.VALUE); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/id/strategy/ProvidedStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.id.strategy; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 20 | import com.mongodb.DBCollection; 21 | import org.apache.kafka.connect.errors.DataException; 22 | import org.apache.kafka.connect.sink.SinkRecord; 23 | import org.bson.BsonDocument; 24 | import org.bson.BsonNull; 25 | import org.bson.BsonValue; 26 | 27 | import java.util.Optional; 28 | 29 | public class ProvidedStrategy implements IdStrategy { 30 | 31 | protected enum ProvidedIn { 32 | KEY, 33 | VALUE 34 | } 35 | 36 | protected ProvidedIn where; 37 | 38 | public ProvidedStrategy(ProvidedIn where) { 39 | this.where = where; 40 | } 41 | 42 | @Override 43 | public BsonValue generateId(SinkDocument doc, SinkRecord orig) { 44 | 45 | Optional bd = Optional.empty(); 46 | 47 | if(where.equals(ProvidedIn.KEY)) { 48 | bd = doc.getKeyDoc(); 49 | } 50 | 51 | if(where.equals(ProvidedIn.VALUE)) { 52 | bd = doc.getValueDoc(); 53 | } 54 | 55 | BsonValue _id = bd.map(d -> d.get(DBCollection.ID_FIELD_NAME)) 56 | .orElseThrow(() -> new DataException("error: provided id strategy is used " 57 | + "but the document structure either contained no _id field or it was null")); 58 | 59 | if(_id instanceof BsonNull) { 60 | throw new DataException("error: provided id strategy used " 61 | + "but the document structure contained an _id of type BsonNull"); 62 | } 63 | 64 | return _id; 65 | 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/processor/id/strategy/UuidStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.id.strategy; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 20 | import org.apache.kafka.connect.sink.SinkRecord; 21 | import org.bson.BsonString; 22 | import org.bson.BsonValue; 23 | 24 | import java.util.UUID; 25 | 26 | public class UuidStrategy implements IdStrategy { 27 | 28 | @Override 29 | public BsonValue generateId(SinkDocument doc, SinkRecord orig) { 30 | return new BsonString(UUID.randomUUID().toString()); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/writemodel/strategy/DeleteOneDefaultStrategy.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb.writemodel.strategy; 2 | 3 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 4 | import at.grahsl.kafka.connect.mongodb.processor.id.strategy.IdStrategy; 5 | import com.mongodb.DBCollection; 6 | import com.mongodb.client.model.DeleteOneModel; 7 | import com.mongodb.client.model.WriteModel; 8 | import org.apache.kafka.connect.errors.DataException; 9 | import org.bson.BsonDocument; 10 | import org.bson.BsonValue; 11 | 12 | public class DeleteOneDefaultStrategy implements WriteModelStrategy { 13 | 14 | private IdStrategy idStrategy; 15 | 16 | @Deprecated 17 | public DeleteOneDefaultStrategy() {} 18 | 19 | public DeleteOneDefaultStrategy(IdStrategy idStrategy) { 20 | this.idStrategy = idStrategy; 21 | } 22 | 23 | @Override 24 | public WriteModel createWriteModel(SinkDocument document) { 25 | 26 | BsonDocument kd = document.getKeyDoc().orElseThrow( 27 | () -> new DataException("error: cannot build the WriteModel since" 28 | + " the key document was missing unexpectedly") 29 | ); 30 | 31 | //NOTE: fallback for backwards / deprecation compatibility 32 | if(idStrategy == null) { 33 | return kd.containsKey(DBCollection.ID_FIELD_NAME) 34 | ? new DeleteOneModel<>(kd) 35 | : new DeleteOneModel<>(new BsonDocument(DBCollection.ID_FIELD_NAME,kd)); 36 | } 37 | 38 | //NOTE: current design doesn't allow to access original SinkRecord (= null) 39 | BsonValue _id = idStrategy.generateId(document,null); 40 | return new DeleteOneModel<>( 41 | new BsonDocument(DBCollection.ID_FIELD_NAME,_id) 42 | ); 43 | 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/writemodel/strategy/MonotonicWritesDefaultStrategy.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb.writemodel.strategy; 2 | 3 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 4 | import com.mongodb.DBCollection; 5 | import com.mongodb.client.model.UpdateOneModel; 6 | import com.mongodb.client.model.UpdateOptions; 7 | import com.mongodb.client.model.WriteModel; 8 | import org.apache.kafka.connect.errors.DataException; 9 | import org.apache.kafka.connect.sink.SinkRecord; 10 | import org.bson.*; 11 | import org.bson.conversions.Bson; 12 | 13 | import java.util.ArrayList; 14 | import java.util.Arrays; 15 | import java.util.List; 16 | 17 | /** 18 | * This WriteModelStrategy implementation adds the kafka coordinates of processed 19 | * records to the actual SinkDocument as meta-data before it gets written to the 20 | * MongoDB collection. The pre-defined and currently not(!) configurable data format 21 | * for this is using a sub-document with the following structure, field names 22 | * and value <PLACEHOLDERS>: 23 | * 24 | * { 25 | * ..., 26 | * 27 | * "_kafkaCoords":{ 28 | * "_topic": "<TOPIC_NAME>", 29 | * "_partition": <PARTITION_NUMBER>, 30 | * "_offset": <OFFSET_NUMBER> 31 | * }, 32 | * 33 | * ... 34 | * } 35 | * 36 | * This "meta-data" is used to perform the actual staleness check, namely, that upsert operations 37 | * based on the corresponding document's _id field will get suppressed in case newer data has 38 | * already been written to the collection in question. Newer data means a document exhibiting 39 | * a greater than or equal offset for the same kafka topic and partition is already present in the sink. 40 | * 41 | * ! IMPORTANT NOTE ! 42 | * This WriteModelStrategy needs MongoDB version 4.2+ and Java Driver 3.11+ since 43 | * lower versions of either lack the support for leveraging update pipeline syntax. 44 | * 45 | */ 46 | public class MonotonicWritesDefaultStrategy implements WriteModelStrategy { 47 | 48 | public static final String FIELD_KAFKA_COORDS = "_kafkaCoords"; 49 | public static final String FIELD_TOPIC = "_topic"; 50 | public static final String FIELD_PARTITION = "_partition"; 51 | public static final String FIELD_OFFSET = "_offset"; 52 | 53 | private static final UpdateOptions UPDATE_OPTIONS = 54 | new UpdateOptions().upsert(true); 55 | 56 | @Override 57 | public WriteModel createWriteModel(SinkDocument document) { 58 | throw new DataException("error: the write model strategy " + MonotonicWritesDefaultStrategy.class.getName() 59 | + " needs the SinkRecord's data and thus cannot work on the SinkDocument param alone." 60 | + " please use the provided method overloading for this." 61 | ); 62 | } 63 | 64 | @Override 65 | public WriteModel createWriteModel(SinkDocument document, SinkRecord record) { 66 | 67 | BsonDocument vd = document.getValueDoc().orElseThrow( 68 | () -> new DataException("error: cannot build the WriteModel since" 69 | + " the value document was missing unexpectedly") 70 | ); 71 | 72 | //1) add kafka coordinates to the value document 73 | //NOTE: future versions might allow to configure the fieldnames 74 | //via external configuration properties, for now this is pre-defined. 75 | vd.append(FIELD_KAFKA_COORDS, new BsonDocument( 76 | FIELD_TOPIC, new BsonString(record.topic())) 77 | .append(FIELD_PARTITION, new BsonInt32(record.kafkaPartition())) 78 | .append(FIELD_OFFSET, new BsonInt64(record.kafkaOffset())) 79 | ); 80 | 81 | //2) build the conditional update pipeline based on Kafka coordinates 82 | //which makes sure that in case records get replayed - e.g. either due to 83 | //uncommitted offsets or newly started connectors with different names - 84 | //that stale data never overwrites newer data which was previously written 85 | //to the sink already. 86 | List conditionalUpdatePipeline = new ArrayList<>(); 87 | conditionalUpdatePipeline.add(new BsonDocument("$replaceRoot", 88 | new BsonDocument("newRoot", new BsonDocument("$cond", 89 | new BsonDocument("if", new BsonDocument("$and", 90 | new BsonArray(Arrays.asList( 91 | new BsonDocument("$eq", new BsonArray(Arrays.asList( 92 | new BsonString("$$ROOT." + FIELD_KAFKA_COORDS + "." + FIELD_TOPIC), 93 | new BsonString(record.topic())))), 94 | new BsonDocument("$eq", new BsonArray(Arrays.asList( 95 | new BsonString("$$ROOT." + FIELD_KAFKA_COORDS + "." + FIELD_PARTITION), 96 | new BsonInt32(record.kafkaPartition())))), 97 | new BsonDocument("$gte", new BsonArray(Arrays.asList( 98 | new BsonString("$$ROOT." + FIELD_KAFKA_COORDS + "." + FIELD_OFFSET), 99 | new BsonInt64(record.kafkaOffset())))) 100 | )))) 101 | .append("then", new BsonString("$$ROOT")) 102 | .append("else", vd) 103 | )) 104 | )); 105 | 106 | return new UpdateOneModel<>( 107 | new BsonDocument(DBCollection.ID_FIELD_NAME, vd.get(DBCollection.ID_FIELD_NAME)), 108 | conditionalUpdatePipeline, 109 | UPDATE_OPTIONS 110 | ); 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/writemodel/strategy/ReplaceOneBusinessKeyStrategy.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb.writemodel.strategy; 2 | 3 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 4 | import com.mongodb.DBCollection; 5 | import com.mongodb.client.model.ReplaceOneModel; 6 | import com.mongodb.client.model.UpdateOptions; 7 | import com.mongodb.client.model.WriteModel; 8 | import org.apache.kafka.connect.errors.DataException; 9 | import org.bson.BsonDocument; 10 | import org.bson.BsonValue; 11 | 12 | public class ReplaceOneBusinessKeyStrategy implements WriteModelStrategy { 13 | 14 | private static final UpdateOptions UPDATE_OPTIONS = 15 | new UpdateOptions().upsert(true); 16 | 17 | @Override 18 | public WriteModel createWriteModel(SinkDocument document) { 19 | 20 | BsonDocument vd = document.getValueDoc().orElseThrow( 21 | () -> new DataException("error: cannot build the WriteModel since" 22 | + " the value document was missing unexpectedly") 23 | ); 24 | 25 | BsonValue businessKey = vd.get(DBCollection.ID_FIELD_NAME); 26 | 27 | if(businessKey == null || !(businessKey instanceof BsonDocument)) { 28 | throw new DataException("error: cannot build the WriteModel since" 29 | + " the value document does not contain an _id field of type BsonDocument" 30 | + " which holds the business key fields"); 31 | } 32 | 33 | vd.remove(DBCollection.ID_FIELD_NAME); 34 | 35 | return new ReplaceOneModel<>((BsonDocument)businessKey, vd, UPDATE_OPTIONS); 36 | 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/writemodel/strategy/ReplaceOneDefaultStrategy.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb.writemodel.strategy; 2 | 3 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 4 | import com.mongodb.DBCollection; 5 | import com.mongodb.client.model.ReplaceOneModel; 6 | import com.mongodb.client.model.UpdateOptions; 7 | import com.mongodb.client.model.WriteModel; 8 | import org.apache.kafka.connect.errors.DataException; 9 | import org.bson.BsonDocument; 10 | 11 | public class ReplaceOneDefaultStrategy implements WriteModelStrategy { 12 | 13 | private static final UpdateOptions UPDATE_OPTIONS = 14 | new UpdateOptions().upsert(true); 15 | 16 | @Override 17 | public WriteModel createWriteModel(SinkDocument document) { 18 | 19 | BsonDocument vd = document.getValueDoc().orElseThrow( 20 | () -> new DataException("error: cannot build the WriteModel since" 21 | + " the value document was missing unexpectedly") 22 | ); 23 | 24 | return new ReplaceOneModel<>( 25 | new BsonDocument(DBCollection.ID_FIELD_NAME, 26 | vd.get(DBCollection.ID_FIELD_NAME)), 27 | vd, 28 | UPDATE_OPTIONS); 29 | 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/writemodel/strategy/UpdateOneTimestampsStrategy.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb.writemodel.strategy; 2 | 3 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 4 | import com.mongodb.DBCollection; 5 | import com.mongodb.client.model.UpdateOneModel; 6 | import com.mongodb.client.model.UpdateOptions; 7 | import com.mongodb.client.model.WriteModel; 8 | import org.apache.kafka.connect.errors.DataException; 9 | import org.bson.BsonDateTime; 10 | import org.bson.BsonDocument; 11 | 12 | import java.time.Instant; 13 | 14 | public class UpdateOneTimestampsStrategy implements WriteModelStrategy { 15 | 16 | public static final String FIELDNAME_MODIFIED_TS = "_modifiedTS"; 17 | public static final String FIELDNAME_INSERTED_TS = "_insertedTS"; 18 | 19 | private static final UpdateOptions UPDATE_OPTIONS = 20 | new UpdateOptions().upsert(true); 21 | 22 | @Override 23 | public WriteModel createWriteModel(SinkDocument document) { 24 | 25 | BsonDocument vd = document.getValueDoc().orElseThrow( 26 | () -> new DataException("error: cannot build the WriteModel since" 27 | + " the value document was missing unexpectedly") 28 | ); 29 | 30 | BsonDateTime dateTime = new BsonDateTime(Instant.now().toEpochMilli()); 31 | 32 | return new UpdateOneModel<>( 33 | new BsonDocument(DBCollection.ID_FIELD_NAME, 34 | vd.get(DBCollection.ID_FIELD_NAME)), 35 | new BsonDocument("$set", vd.append(FIELDNAME_MODIFIED_TS, dateTime)) 36 | .append("$setOnInsert", new BsonDocument(FIELDNAME_INSERTED_TS, dateTime)), 37 | UPDATE_OPTIONS 38 | ); 39 | 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/at/grahsl/kafka/connect/mongodb/writemodel/strategy/WriteModelStrategy.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb.writemodel.strategy; 2 | 3 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 4 | import com.mongodb.client.model.WriteModel; 5 | import org.apache.kafka.connect.sink.SinkRecord; 6 | import org.bson.BsonDocument; 7 | 8 | public interface WriteModelStrategy { 9 | 10 | WriteModel createWriteModel(SinkDocument document); 11 | 12 | default WriteModel createWriteModel(SinkDocument document, SinkRecord record) { 13 | return createWriteModel(document); 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/main/resources/kafka-connect-mongodb-version.properties: -------------------------------------------------------------------------------- 1 | ## 2 | # Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | ## 16 | 17 | version=${project.version} 18 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/test/java/at/grahsl/kafka/connect/mongodb/MongoDbSinkRecordBatchesTest.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb; 2 | 3 | import com.google.common.collect.Lists; 4 | import org.apache.kafka.connect.sink.SinkRecord; 5 | import org.junit.jupiter.api.BeforeAll; 6 | import org.junit.jupiter.api.DisplayName; 7 | import org.junit.jupiter.api.DynamicTest; 8 | import org.junit.jupiter.api.TestFactory; 9 | import org.junit.platform.runner.JUnitPlatform; 10 | import org.junit.runner.RunWith; 11 | 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | import java.util.stream.Stream; 15 | 16 | import static org.junit.Assert.assertEquals; 17 | import static org.junit.jupiter.api.DynamicTest.dynamicTest; 18 | 19 | @RunWith(JUnitPlatform.class) 20 | public class MongoDbSinkRecordBatchesTest { 21 | 22 | private static List> LIST_INITIAL_EMPTY = new ArrayList<>(); 23 | private static final int NUM_FAKE_RECORDS = 50; 24 | 25 | @BeforeAll 26 | static void setupVerificationList() { 27 | LIST_INITIAL_EMPTY.add(new ArrayList<>()); 28 | } 29 | 30 | @TestFactory 31 | @DisplayName("test batching with different config params for max.batch.size") 32 | Stream testBatchingWithDifferentConfigsForBatchSize() { 33 | 34 | return Stream.iterate(0, r -> r + 1).limit(NUM_FAKE_RECORDS+1) 35 | .map(batchSize -> dynamicTest("test batching for " 36 | +NUM_FAKE_RECORDS+" records with batchsize="+batchSize, () -> { 37 | MongoDbSinkRecordBatches batches = new MongoDbSinkRecordBatches(batchSize, NUM_FAKE_RECORDS); 38 | assertEquals(LIST_INITIAL_EMPTY, batches.getBufferedBatches()); 39 | List recordList = createSinkRecordList("foo",0,0,NUM_FAKE_RECORDS); 40 | recordList.forEach(batches::buffer); 41 | List> batchedList = createBatchedSinkRecordList(recordList,batchSize); 42 | assertEquals(batchedList,batches.getBufferedBatches()); 43 | })); 44 | 45 | } 46 | 47 | private static List createSinkRecordList(String topic, int partition, int beginOffset, int size) { 48 | List list = new ArrayList<>(); 49 | for(int i = 0; i < size; i++) { 50 | list.add(new SinkRecord(topic,partition,null,null,null,null, beginOffset+i)); 51 | } 52 | return list; 53 | } 54 | 55 | private static List> createBatchedSinkRecordList(List sinkRecordList, int batchSize) { 56 | if(batchSize > 0) { 57 | return Lists.partition(sinkRecordList,batchSize); 58 | } 59 | List> batchedList = new ArrayList<>(); 60 | batchedList.add(sinkRecordList); 61 | return batchedList; 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /src/test/java/at/grahsl/kafka/connect/mongodb/ValidatorWithOperatorsTest.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb; 2 | 3 | import org.apache.kafka.common.config.ConfigException; 4 | import org.junit.jupiter.api.DisplayName; 5 | import org.junit.jupiter.api.Test; 6 | import org.junit.platform.runner.JUnitPlatform; 7 | import org.junit.runner.RunWith; 8 | 9 | import java.util.regex.Pattern; 10 | 11 | import static at.grahsl.kafka.connect.mongodb.MongoDbSinkConnectorConfig.ValidatorWithOperators; 12 | import static org.junit.jupiter.api.Assertions.assertThrows; 13 | 14 | @RunWith(JUnitPlatform.class) 15 | public class ValidatorWithOperatorsTest { 16 | 17 | 18 | public static final String NAME = "name"; 19 | public static final Object ANY_VALUE = null; 20 | 21 | final ValidatorWithOperators PASS = (name, value) -> { 22 | // ignore, always passes 23 | }; 24 | 25 | final ValidatorWithOperators FAIL = (name, value) -> { 26 | throw new ConfigException(name, value, "always fails"); 27 | }; 28 | 29 | @Test 30 | @DisplayName("validate empty string") 31 | public void emptyString() { 32 | ValidatorWithOperators validator = MongoDbSinkConnectorConfig.emptyString(); 33 | validator.ensureValid(NAME, ""); 34 | } 35 | 36 | @Test 37 | @DisplayName("invalidate non-empty string") 38 | public void invalidateNonEmptyString() { 39 | ValidatorWithOperators validator = MongoDbSinkConnectorConfig.emptyString(); 40 | assertThrows(ConfigException.class, () -> validator.ensureValid(NAME, "value")); 41 | } 42 | 43 | @Test 44 | @DisplayName("validate regex") 45 | public void simpleRegex() { 46 | ValidatorWithOperators validator = MongoDbSinkConnectorConfig.matching(Pattern.compile("fo+ba[rz]")); 47 | validator.ensureValid(NAME, "foobar"); 48 | validator.ensureValid(NAME, "foobaz"); 49 | } 50 | 51 | @Test 52 | @DisplayName("invalidate regex") 53 | public void invalidateSimpleRegex() { 54 | ValidatorWithOperators validator = MongoDbSinkConnectorConfig.matching(Pattern.compile("fo+ba[rz]")); 55 | assertThrows(ConfigException.class, () -> validator.ensureValid(NAME, "foobax")); 56 | assertThrows(ConfigException.class, () -> validator.ensureValid(NAME, "fbar")); 57 | } 58 | 59 | @Test 60 | @DisplayName("validate arithmetic or") 61 | public void arithmeticOr() { 62 | PASS.or(PASS).ensureValid(NAME, ANY_VALUE); 63 | PASS.or(FAIL).ensureValid(NAME, ANY_VALUE); 64 | FAIL.or(PASS).ensureValid(NAME, ANY_VALUE); 65 | PASS.or(PASS).or(PASS).ensureValid(NAME, ANY_VALUE); 66 | PASS.or(PASS).or(FAIL).ensureValid(NAME, ANY_VALUE); 67 | PASS.or(FAIL).or(PASS).ensureValid(NAME, ANY_VALUE); 68 | PASS.or(FAIL).or(FAIL).ensureValid(NAME, ANY_VALUE); 69 | FAIL.or(PASS).or(PASS).ensureValid(NAME, ANY_VALUE); 70 | FAIL.or(PASS).or(FAIL).ensureValid(NAME, ANY_VALUE); 71 | FAIL.or(FAIL).or(PASS).ensureValid(NAME, ANY_VALUE); 72 | } 73 | 74 | @Test 75 | @DisplayName("invalidate arithmetic or") 76 | public void invalidateArithmeticOr() { 77 | assertThrows(ConfigException.class, () -> FAIL.or(FAIL).ensureValid(NAME, ANY_VALUE)); 78 | assertThrows(ConfigException.class, () -> FAIL.or(FAIL).or(FAIL).ensureValid(NAME, ANY_VALUE)); 79 | } 80 | 81 | @Test 82 | @DisplayName("arithmetic and") 83 | public void arithmeticAnd() { 84 | PASS.and(PASS).ensureValid(NAME, ANY_VALUE); 85 | PASS.and(PASS).and(PASS).ensureValid(NAME, ANY_VALUE); 86 | } 87 | 88 | @Test 89 | @DisplayName("invalidate arithmetic and") 90 | public void invalidateArithmeticAnd() { 91 | assertThrows(ConfigException.class, () -> PASS.and(FAIL).ensureValid(NAME, ANY_VALUE)); 92 | assertThrows(ConfigException.class, () -> FAIL.and(PASS).ensureValid(NAME, ANY_VALUE)); 93 | assertThrows(ConfigException.class, () -> FAIL.and(FAIL).ensureValid(NAME, ANY_VALUE)); 94 | } 95 | 96 | } 97 | -------------------------------------------------------------------------------- /src/test/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/OperationTypeTest.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb.cdc.debezium; 2 | 3 | import org.junit.jupiter.api.DisplayName; 4 | import org.junit.jupiter.api.Test; 5 | import org.junit.platform.runner.JUnitPlatform; 6 | import org.junit.runner.RunWith; 7 | 8 | import static org.junit.jupiter.api.Assertions.*; 9 | 10 | @RunWith(JUnitPlatform.class) 11 | public class OperationTypeTest { 12 | 13 | @Test 14 | @DisplayName("when op type 'c' then type CREATE") 15 | public void testOperationTypeCreate() { 16 | String textType = "c"; 17 | OperationType otCreate = OperationType.fromText(textType); 18 | assertAll( 19 | () -> assertEquals(OperationType.CREATE,otCreate), 20 | () -> assertEquals(textType,otCreate.type()) 21 | ); 22 | } 23 | 24 | @Test 25 | @DisplayName("when op type 'r' then type READ") 26 | public void testOperationTypeRead() { 27 | String textType = "r"; 28 | OperationType otRead = OperationType.fromText(textType); 29 | assertAll( 30 | () -> assertEquals(OperationType.READ,otRead), 31 | () -> assertEquals(textType,otRead.type()) 32 | ); 33 | } 34 | 35 | @Test 36 | @DisplayName("when op type 'u' then type UPDATE") 37 | public void testOperationTypeUpdate() { 38 | String textType = "u"; 39 | OperationType otUpdate = OperationType.fromText(textType); 40 | assertAll( 41 | () -> assertEquals(OperationType.UPDATE,otUpdate), 42 | () -> assertEquals(textType,otUpdate.type()) 43 | ); 44 | } 45 | 46 | @Test 47 | @DisplayName("when op type 'd' then type DELETE") 48 | public void testOperationTypeDelete() { 49 | String textType = "d"; 50 | OperationType otDelete = OperationType.fromText(textType); 51 | assertAll( 52 | () -> assertEquals(OperationType.DELETE,otDelete), 53 | () -> assertEquals(textType,otDelete.type()) 54 | ); 55 | } 56 | 57 | @Test 58 | @DisplayName("when invalid op type IllegalArgumentException") 59 | public void testOperationTypeInvalid() { 60 | assertThrows(IllegalArgumentException.class, 61 | () -> OperationType.fromText("x")); 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /src/test/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/mongodb/MongoDbDeleteTest.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.mongodb; 2 | 3 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 4 | import com.mongodb.DBCollection; 5 | import com.mongodb.client.model.DeleteOneModel; 6 | import com.mongodb.client.model.WriteModel; 7 | import org.apache.kafka.connect.errors.DataException; 8 | import org.bson.BsonDocument; 9 | import org.bson.BsonInt32; 10 | import org.bson.BsonString; 11 | import org.junit.jupiter.api.DisplayName; 12 | import org.junit.jupiter.api.Test; 13 | import org.junit.platform.runner.JUnitPlatform; 14 | import org.junit.runner.RunWith; 15 | 16 | import static org.junit.jupiter.api.Assertions.*; 17 | 18 | @RunWith(JUnitPlatform.class) 19 | public class MongoDbDeleteTest { 20 | 21 | public static final MongoDbDelete MONGODB_DELETE = new MongoDbDelete(); 22 | 23 | public static final BsonDocument FILTER_DOC = 24 | new BsonDocument(DBCollection.ID_FIELD_NAME,new BsonInt32(1004)); 25 | 26 | @Test 27 | @DisplayName("when valid cdc event then correct DeleteOneModel") 28 | public void testValidSinkDocument() { 29 | BsonDocument keyDoc = new BsonDocument("id",new BsonString("1004")); 30 | 31 | WriteModel result = 32 | MONGODB_DELETE.perform(new SinkDocument(keyDoc,null)); 33 | 34 | assertTrue(result instanceof DeleteOneModel, 35 | () -> "result expected to be of type DeleteOneModel"); 36 | 37 | DeleteOneModel writeModel = 38 | (DeleteOneModel) result; 39 | 40 | assertTrue(writeModel.getFilter() instanceof BsonDocument, 41 | () -> "filter expected to be of type BsonDocument"); 42 | 43 | assertEquals(FILTER_DOC,writeModel.getFilter()); 44 | 45 | } 46 | 47 | @Test 48 | @DisplayName("when missing key doc then DataException") 49 | public void testMissingKeyDocument() { 50 | assertThrows(DataException.class,() -> 51 | MONGODB_DELETE.perform(new SinkDocument(null,new BsonDocument())) 52 | ); 53 | } 54 | 55 | @Test 56 | @DisplayName("when key doc 'id' field not of type String then DataException") 57 | public void testInvalidTypeIdFieldInKeyDocument() { 58 | BsonDocument keyDoc = new BsonDocument("id",new BsonInt32(1004)); 59 | assertThrows(DataException.class,() -> 60 | MONGODB_DELETE.perform(new SinkDocument(keyDoc,new BsonDocument())) 61 | ); 62 | } 63 | 64 | @Test 65 | @DisplayName("when key doc 'id' field contains invalid JSON then DataException") 66 | public void testInvalidJsonIdFieldInKeyDocument() { 67 | BsonDocument keyDoc = new BsonDocument("id",new BsonString("{,NOT:JSON,}")); 68 | assertThrows(DataException.class,() -> 69 | MONGODB_DELETE.perform(new SinkDocument(keyDoc,new BsonDocument())) 70 | ); 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /src/test/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/mongodb/MongoDbInsertTest.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.mongodb; 2 | 3 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 4 | import com.mongodb.DBCollection; 5 | import com.mongodb.client.model.ReplaceOneModel; 6 | import com.mongodb.client.model.WriteModel; 7 | import org.apache.kafka.connect.errors.DataException; 8 | import org.bson.BsonDocument; 9 | import org.bson.BsonInt32; 10 | import org.bson.BsonString; 11 | import org.junit.jupiter.api.DisplayName; 12 | import org.junit.jupiter.api.Test; 13 | import org.junit.platform.runner.JUnitPlatform; 14 | import org.junit.runner.RunWith; 15 | 16 | import static org.junit.jupiter.api.Assertions.*; 17 | 18 | @RunWith(JUnitPlatform.class) 19 | public class MongoDbInsertTest { 20 | 21 | public static final MongoDbInsert MONGODB_INSERT = new MongoDbInsert(); 22 | 23 | public static final BsonDocument FILTER_DOC = 24 | new BsonDocument(DBCollection.ID_FIELD_NAME,new BsonInt32(1004)); 25 | 26 | public static final BsonDocument REPLACEMENT_DOC = 27 | new BsonDocument(DBCollection.ID_FIELD_NAME,new BsonInt32(1004)) 28 | .append("first_name",new BsonString("Anne")) 29 | .append("last_name",new BsonString("Kretchmar")) 30 | .append("email",new BsonString("annek@noanswer.org")); 31 | 32 | @Test 33 | @DisplayName("when valid cdc event then correct ReplaceOneModel") 34 | public void testValidSinkDocument() { 35 | 36 | BsonDocument keyDoc = new BsonDocument("id",new BsonString("1004")); 37 | 38 | BsonDocument valueDoc = new BsonDocument("op",new BsonString("c")) 39 | .append("after",new BsonString(REPLACEMENT_DOC.toJson())); 40 | 41 | WriteModel result = 42 | MONGODB_INSERT.perform(new SinkDocument(keyDoc,valueDoc)); 43 | 44 | assertTrue(result instanceof ReplaceOneModel, 45 | () -> "result expected to be of type ReplaceOneModel"); 46 | 47 | ReplaceOneModel writeModel = 48 | (ReplaceOneModel) result; 49 | 50 | assertEquals(REPLACEMENT_DOC,writeModel.getReplacement(), 51 | ()-> "replacement doc not matching what is expected"); 52 | 53 | assertTrue(writeModel.getFilter() instanceof BsonDocument, 54 | () -> "filter expected to be of type BsonDocument"); 55 | 56 | assertEquals(FILTER_DOC,writeModel.getFilter()); 57 | 58 | assertTrue(writeModel.getOptions().isUpsert(), 59 | () -> "replacement expected to be done in upsert mode"); 60 | 61 | } 62 | 63 | @Test 64 | @DisplayName("when missing value doc then DataException") 65 | public void testMissingValueDocument() { 66 | assertThrows(DataException.class,() -> 67 | MONGODB_INSERT.perform(new SinkDocument(new BsonDocument(),null)) 68 | ); 69 | } 70 | 71 | @Test 72 | @DisplayName("when invalid json in value doc 'after' field then DataException") 73 | public void testInvalidAfterField() { 74 | assertThrows(DataException.class,() -> 75 | MONGODB_INSERT.perform( 76 | new SinkDocument(new BsonDocument(), 77 | new BsonDocument("op",new BsonString("c")) 78 | .append("after",new BsonString("{NO : JSON [HERE] GO : AWAY}"))) 79 | ) 80 | ); 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /src/test/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/mongodb/MongoDbNoOpTest.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.mongodb; 2 | 3 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 4 | import org.junit.jupiter.api.DisplayName; 5 | import org.junit.jupiter.api.Test; 6 | import org.junit.platform.runner.JUnitPlatform; 7 | import org.junit.runner.RunWith; 8 | 9 | import static org.junit.jupiter.api.Assertions.assertAll; 10 | import static org.junit.jupiter.api.Assertions.assertNull; 11 | 12 | @RunWith(JUnitPlatform.class) 13 | public class MongoDbNoOpTest { 14 | 15 | @Test 16 | @DisplayName("when any cdc event then WriteModel is null resulting in Optional.empty() in the corresponding handler") 17 | public void testValidSinkDocument() { 18 | assertAll("test behaviour of MongoDbNoOp", 19 | () -> assertNull(new MongoDbNoOp().perform(new SinkDocument(null,null)),"MongoDbNoOp must result in null WriteModel"), 20 | () -> assertNull(new MongoDbNoOp().perform(null),"MongoDbNoOp must result in null WriteModel") 21 | ); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/test/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/mongodb/MongoDbUpdateTest.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.mongodb; 2 | 3 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 4 | import com.mongodb.DBCollection; 5 | import com.mongodb.client.model.ReplaceOneModel; 6 | import com.mongodb.client.model.UpdateOneModel; 7 | import com.mongodb.client.model.WriteModel; 8 | import org.apache.kafka.connect.errors.DataException; 9 | import org.bson.BsonDocument; 10 | import org.bson.BsonInt32; 11 | import org.bson.BsonString; 12 | import org.junit.jupiter.api.DisplayName; 13 | import org.junit.jupiter.api.Test; 14 | import org.junit.platform.runner.JUnitPlatform; 15 | import org.junit.runner.RunWith; 16 | 17 | import static org.junit.jupiter.api.Assertions.*; 18 | 19 | @RunWith(JUnitPlatform.class) 20 | public class MongoDbUpdateTest { 21 | 22 | public static final MongoDbUpdate MONGODB_UPDATE = new MongoDbUpdate(); 23 | 24 | public static final BsonDocument FILTER_DOC = 25 | new BsonDocument(DBCollection.ID_FIELD_NAME,new BsonInt32(1004)); 26 | 27 | public static final BsonDocument REPLACEMENT_DOC = 28 | new BsonDocument(DBCollection.ID_FIELD_NAME,new BsonInt32(1004)) 29 | .append("first_name",new BsonString("Anne")) 30 | .append("last_name",new BsonString("Kretchmar")) 31 | .append("email",new BsonString("annek@noanswer.org")); 32 | 33 | public static final BsonDocument UPDATE_DOC = 34 | new BsonDocument("$set",new BsonDocument("first_name",new BsonString("Anna")) 35 | .append("last_name",new BsonString("Kretchmer")) 36 | ); 37 | 38 | //USED to verify if oplog internals ($v field) are removed correctly 39 | public static final BsonDocument UPDATE_DOC_WITH_OPLOG_INTERNALS = 40 | UPDATE_DOC.clone().append("$v",new BsonInt32(1)); 41 | 42 | @Test 43 | @DisplayName("when valid doc replace cdc event then correct ReplaceOneModel") 44 | public void testValidSinkDocumentForReplacement() { 45 | 46 | BsonDocument keyDoc = new BsonDocument("id",new BsonString("1004")); 47 | 48 | BsonDocument valueDoc = new BsonDocument("op",new BsonString("u")) 49 | .append("patch",new BsonString(REPLACEMENT_DOC.toJson())); 50 | 51 | WriteModel result = 52 | MONGODB_UPDATE.perform(new SinkDocument(keyDoc,valueDoc)); 53 | 54 | assertTrue(result instanceof ReplaceOneModel, 55 | () -> "result expected to be of type ReplaceOneModel"); 56 | 57 | ReplaceOneModel writeModel = 58 | (ReplaceOneModel) result; 59 | 60 | assertEquals(REPLACEMENT_DOC,writeModel.getReplacement(), 61 | ()-> "replacement doc not matching what is expected"); 62 | 63 | assertTrue(writeModel.getFilter() instanceof BsonDocument, 64 | () -> "filter expected to be of type BsonDocument"); 65 | 66 | assertEquals(FILTER_DOC,writeModel.getFilter()); 67 | 68 | assertTrue(writeModel.getOptions().isUpsert(), 69 | () -> "replacement expected to be done in upsert mode"); 70 | 71 | } 72 | 73 | @Test 74 | @DisplayName("when valid doc change cdc event then correct UpdateOneModel") 75 | public void testValidSinkDocumentForUpdate() { 76 | BsonDocument keyDoc = new BsonDocument("id",new BsonString("1004")); 77 | 78 | BsonDocument valueDoc = new BsonDocument("op",new BsonString("u")) 79 | .append("patch",new BsonString(UPDATE_DOC.toJson())); 80 | 81 | WriteModel result = 82 | MONGODB_UPDATE.perform(new SinkDocument(keyDoc,valueDoc)); 83 | 84 | assertTrue(result instanceof UpdateOneModel, 85 | () -> "result expected to be of type UpdateOneModel"); 86 | 87 | UpdateOneModel writeModel = 88 | (UpdateOneModel) result; 89 | 90 | assertEquals(UPDATE_DOC,writeModel.getUpdate(), 91 | ()-> "update doc not matching what is expected"); 92 | 93 | assertTrue(writeModel.getFilter() instanceof BsonDocument, 94 | () -> "filter expected to be of type BsonDocument"); 95 | 96 | assertEquals(FILTER_DOC,writeModel.getFilter()); 97 | 98 | } 99 | 100 | @Test 101 | @DisplayName("when valid doc change cdc event containing internal oplog fields then correct UpdateOneModel") 102 | public void testValidSinkDocumentWithInternalOploagFieldForUpdate() { 103 | BsonDocument keyDoc = new BsonDocument("id",new BsonString("1004")); 104 | 105 | BsonDocument valueDoc = new BsonDocument("op",new BsonString("u")) 106 | .append("patch",new BsonString(UPDATE_DOC_WITH_OPLOG_INTERNALS.toJson())); 107 | 108 | WriteModel result = 109 | MONGODB_UPDATE.perform(new SinkDocument(keyDoc,valueDoc)); 110 | 111 | assertTrue(result instanceof UpdateOneModel, 112 | () -> "result expected to be of type UpdateOneModel"); 113 | 114 | UpdateOneModel writeModel = 115 | (UpdateOneModel) result; 116 | 117 | assertEquals(UPDATE_DOC,writeModel.getUpdate(), 118 | ()-> "update doc not matching what is expected"); 119 | 120 | assertTrue(writeModel.getFilter() instanceof BsonDocument, 121 | () -> "filter expected to be of type BsonDocument"); 122 | 123 | assertEquals(FILTER_DOC,writeModel.getFilter()); 124 | 125 | } 126 | 127 | @Test 128 | @DisplayName("when missing value doc then DataException") 129 | public void testMissingValueDocument() { 130 | assertThrows(DataException.class,() -> 131 | MONGODB_UPDATE.perform(new SinkDocument(new BsonDocument(),null)) 132 | ); 133 | } 134 | 135 | @Test 136 | @DisplayName("when missing key doc then DataException") 137 | public void testMissingKeyDocument() { 138 | assertThrows(DataException.class,() -> 139 | MONGODB_UPDATE.perform(new SinkDocument(null, 140 | new BsonDocument("patch",new BsonString("{}")))) 141 | ); 142 | } 143 | 144 | @Test 145 | @DisplayName("when 'update' field missing in value doc then DataException") 146 | public void testMissingPatchFieldInValueDocument() { 147 | assertThrows(DataException.class,() -> 148 | MONGODB_UPDATE.perform(new SinkDocument(new BsonDocument("id",new BsonString("1004")), 149 | new BsonDocument("nopatch",new BsonString("{}")))) 150 | ); 151 | } 152 | 153 | @Test 154 | @DisplayName("when 'id' field not of type String in key doc then DataException") 155 | public void testIdFieldNoStringInKeyDocument() { 156 | assertThrows(DataException.class,() -> 157 | MONGODB_UPDATE.perform(new SinkDocument(new BsonDocument("id",new BsonInt32(1004)), 158 | new BsonDocument("patch",new BsonString("{}")))) 159 | ); 160 | } 161 | 162 | @Test 163 | @DisplayName("when 'id' field invalid JSON in key doc then DataException") 164 | public void testIdFieldInvalidJsonInKeyDocument() { 165 | assertThrows(DataException.class,() -> 166 | MONGODB_UPDATE.perform(new SinkDocument(new BsonDocument("id",new BsonString("{no-JSON}")), 167 | new BsonDocument("patch",new BsonString("{}")))) 168 | ); 169 | } 170 | 171 | } 172 | -------------------------------------------------------------------------------- /src/test/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/rdbms/RdbmsDeleteTest.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.rdbms; 2 | 3 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 4 | import com.mongodb.DBCollection; 5 | import com.mongodb.client.model.DeleteOneModel; 6 | import com.mongodb.client.model.WriteModel; 7 | import org.apache.kafka.connect.errors.DataException; 8 | import org.bson.BsonBoolean; 9 | import org.bson.BsonDocument; 10 | import org.bson.BsonInt32; 11 | import org.bson.BsonString; 12 | import org.junit.jupiter.api.DisplayName; 13 | import org.junit.jupiter.api.Test; 14 | import org.junit.platform.runner.JUnitPlatform; 15 | import org.junit.runner.RunWith; 16 | 17 | import static org.junit.jupiter.api.Assertions.*; 18 | 19 | @RunWith(JUnitPlatform.class) 20 | public class RdbmsDeleteTest { 21 | 22 | public static final RdbmsDelete RDBMS_DELETE = new RdbmsDelete(); 23 | 24 | @Test 25 | @DisplayName("when valid cdc event with single field PK then correct DeleteOneModel") 26 | public void testValidSinkDocumentSingleFieldPK() { 27 | 28 | BsonDocument filterDoc = 29 | new BsonDocument(DBCollection.ID_FIELD_NAME, 30 | new BsonDocument("id",new BsonInt32(1004))); 31 | 32 | BsonDocument keyDoc = new BsonDocument("id",new BsonInt32(1004)); 33 | BsonDocument valueDoc = new BsonDocument("op",new BsonString("d")); 34 | 35 | WriteModel result = 36 | RDBMS_DELETE.perform(new SinkDocument(keyDoc,valueDoc)); 37 | 38 | assertTrue(result instanceof DeleteOneModel, 39 | () -> "result expected to be of type DeleteOneModel"); 40 | 41 | DeleteOneModel writeModel = 42 | (DeleteOneModel) result; 43 | 44 | assertTrue(writeModel.getFilter() instanceof BsonDocument, 45 | () -> "filter expected to be of type BsonDocument"); 46 | 47 | assertEquals(filterDoc,writeModel.getFilter()); 48 | 49 | } 50 | 51 | @Test 52 | @DisplayName("when valid cdc event with compound PK then correct DeleteOneModel") 53 | public void testValidSinkDocumentCompoundPK() { 54 | 55 | BsonDocument filterDoc = 56 | new BsonDocument(DBCollection.ID_FIELD_NAME, 57 | new BsonDocument("idA",new BsonInt32(123)) 58 | .append("idB",new BsonString("ABC"))); 59 | 60 | BsonDocument keyDoc = new BsonDocument("idA",new BsonInt32(123)) 61 | .append("idB",new BsonString("ABC")); 62 | BsonDocument valueDoc = new BsonDocument("op",new BsonString("d")); 63 | 64 | WriteModel result = 65 | RDBMS_DELETE.perform(new SinkDocument(keyDoc,valueDoc)); 66 | 67 | assertTrue(result instanceof DeleteOneModel, 68 | () -> "result expected to be of type DeleteOneModel"); 69 | 70 | DeleteOneModel writeModel = 71 | (DeleteOneModel) result; 72 | 73 | assertTrue(writeModel.getFilter() instanceof BsonDocument, 74 | () -> "filter expected to be of type BsonDocument"); 75 | 76 | assertEquals(filterDoc,writeModel.getFilter()); 77 | 78 | } 79 | 80 | @Test 81 | @DisplayName("when valid cdc event without PK then correct DeleteOneModel") 82 | public void testValidSinkDocumentNoPK() { 83 | 84 | BsonDocument filterDoc = new BsonDocument("text", new BsonString("hohoho")) 85 | .append("number", new BsonInt32(9876)) 86 | .append("active", new BsonBoolean(true)); 87 | 88 | BsonDocument keyDoc = new BsonDocument(); 89 | 90 | BsonDocument valueDoc = new BsonDocument("op",new BsonString("c")) 91 | .append("before",new BsonDocument("text", new BsonString("hohoho")) 92 | .append("number", new BsonInt32(9876)) 93 | .append("active", new BsonBoolean(true))); 94 | 95 | WriteModel result = 96 | RDBMS_DELETE.perform(new SinkDocument(keyDoc,valueDoc)); 97 | 98 | assertTrue(result instanceof DeleteOneModel, 99 | () -> "result expected to be of type DeleteOneModel"); 100 | 101 | DeleteOneModel writeModel = 102 | (DeleteOneModel) result; 103 | 104 | assertTrue(writeModel.getFilter() instanceof BsonDocument, 105 | () -> "filter expected to be of type BsonDocument"); 106 | 107 | assertEquals(filterDoc,writeModel.getFilter()); 108 | 109 | } 110 | 111 | @Test 112 | @DisplayName("when missing key doc then DataException") 113 | public void testMissingKeyDocument() { 114 | assertThrows(DataException.class,() -> 115 | RDBMS_DELETE.perform(new SinkDocument(null,new BsonDocument())) 116 | ); 117 | } 118 | 119 | @Test 120 | @DisplayName("when missing value doc then DataException") 121 | public void testMissingValueDocument() { 122 | assertThrows(DataException.class,() -> 123 | RDBMS_DELETE.perform(new SinkDocument(new BsonDocument(),null)) 124 | ); 125 | } 126 | 127 | @Test 128 | @DisplayName("when key doc and value 'before' field both empty then DataException") 129 | public void testEmptyKeyDocAndEmptyValueBeforeField() { 130 | assertThrows(DataException.class,() -> 131 | RDBMS_DELETE.perform(new SinkDocument(new BsonDocument(), 132 | new BsonDocument("op",new BsonString("d")).append("before",new BsonDocument()))) 133 | ); 134 | } 135 | 136 | } 137 | -------------------------------------------------------------------------------- /src/test/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/rdbms/RdbmsInsertTest.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.rdbms; 2 | 3 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 4 | import com.mongodb.DBCollection; 5 | import com.mongodb.client.model.ReplaceOneModel; 6 | import com.mongodb.client.model.WriteModel; 7 | import org.apache.kafka.connect.errors.DataException; 8 | import org.bson.*; 9 | import org.junit.jupiter.api.DisplayName; 10 | import org.junit.jupiter.api.Test; 11 | import org.junit.platform.runner.JUnitPlatform; 12 | import org.junit.runner.RunWith; 13 | 14 | import static org.junit.jupiter.api.Assertions.*; 15 | 16 | @RunWith(JUnitPlatform.class) 17 | public class RdbmsInsertTest { 18 | 19 | public static final RdbmsInsert RDBMS_INSERT = new RdbmsInsert(); 20 | 21 | @Test 22 | @DisplayName("when valid cdc event with single field PK then correct ReplaceOneModel") 23 | public void testValidSinkDocumentSingleFieldPK() { 24 | 25 | BsonDocument filterDoc = 26 | new BsonDocument(DBCollection.ID_FIELD_NAME, 27 | new BsonDocument("id",new BsonInt32(1004))); 28 | 29 | BsonDocument replacementDoc = 30 | new BsonDocument(DBCollection.ID_FIELD_NAME, 31 | new BsonDocument("id",new BsonInt32(1004))) 32 | .append("first_name",new BsonString("Anne")) 33 | .append("last_name",new BsonString("Kretchmar")) 34 | .append("email",new BsonString("annek@noanswer.org")); 35 | 36 | BsonDocument keyDoc = new BsonDocument("id",new BsonInt32(1004)); 37 | 38 | BsonDocument valueDoc = new BsonDocument("op",new BsonString("c")) 39 | .append("after",new BsonDocument("id",new BsonInt32(1004)) 40 | .append("first_name",new BsonString("Anne")) 41 | .append("last_name",new BsonString("Kretchmar")) 42 | .append("email",new BsonString("annek@noanswer.org"))); 43 | 44 | WriteModel result = 45 | RDBMS_INSERT.perform(new SinkDocument(keyDoc,valueDoc)); 46 | 47 | assertTrue(result instanceof ReplaceOneModel, 48 | () -> "result expected to be of type ReplaceOneModel"); 49 | 50 | ReplaceOneModel writeModel = 51 | (ReplaceOneModel) result; 52 | 53 | assertEquals(replacementDoc,writeModel.getReplacement(), 54 | ()-> "replacement doc not matching what is expected"); 55 | 56 | assertTrue(writeModel.getFilter() instanceof BsonDocument, 57 | () -> "filter expected to be of type BsonDocument"); 58 | 59 | assertEquals(filterDoc,writeModel.getFilter()); 60 | 61 | assertTrue(writeModel.getOptions().isUpsert(), 62 | () -> "replacement expected to be done in upsert mode"); 63 | 64 | } 65 | 66 | @Test 67 | @DisplayName("when valid cdc event with compound PK then correct ReplaceOneModel") 68 | public void testValidSinkDocumentCompoundPK() { 69 | 70 | BsonDocument filterDoc = 71 | new BsonDocument(DBCollection.ID_FIELD_NAME, 72 | new BsonDocument("idA",new BsonInt32(123)) 73 | .append("idB",new BsonString("ABC"))); 74 | 75 | BsonDocument replacementDoc = 76 | new BsonDocument(DBCollection.ID_FIELD_NAME, 77 | new BsonDocument("idA",new BsonInt32(123)) 78 | .append("idB",new BsonString("ABC"))) 79 | .append("number", new BsonDouble(567.89)) 80 | .append("active", new BsonBoolean(true)); 81 | 82 | BsonDocument keyDoc = new BsonDocument("idA",new BsonInt32(123)) 83 | .append("idB",new BsonString("ABC")); 84 | 85 | BsonDocument valueDoc = new BsonDocument("op",new BsonString("c")) 86 | .append("after",new BsonDocument("idA",new BsonInt32(123)) 87 | .append("idB",new BsonString("ABC")) 88 | .append("number", new BsonDouble(567.89)) 89 | .append("active", new BsonBoolean(true))); 90 | 91 | WriteModel result = 92 | RDBMS_INSERT.perform(new SinkDocument(keyDoc,valueDoc)); 93 | 94 | assertTrue(result instanceof ReplaceOneModel, 95 | () -> "result expected to be of type ReplaceOneModel"); 96 | 97 | ReplaceOneModel writeModel = 98 | (ReplaceOneModel) result; 99 | 100 | assertEquals(replacementDoc,writeModel.getReplacement(), 101 | ()-> "replacement doc not matching what is expected"); 102 | 103 | assertTrue(writeModel.getFilter() instanceof BsonDocument, 104 | () -> "filter expected to be of type BsonDocument"); 105 | 106 | assertEquals(filterDoc,writeModel.getFilter()); 107 | 108 | assertTrue(writeModel.getOptions().isUpsert(), 109 | () -> "replacement expected to be done in upsert mode"); 110 | 111 | } 112 | 113 | @Test 114 | @DisplayName("when valid cdc event without PK then correct ReplaceOneModel") 115 | public void testValidSinkDocumentNoPK() { 116 | 117 | BsonDocument valueDocCreate = new BsonDocument("op",new BsonString("c")) 118 | .append("after",new BsonDocument("text", new BsonString("lalala")) 119 | .append("number", new BsonInt32(1234)) 120 | .append("active", new BsonBoolean(false))); 121 | 122 | verifyResultsNoPK(valueDocCreate); 123 | 124 | BsonDocument valueDocRead = new BsonDocument("op",new BsonString("r")) 125 | .append("after",new BsonDocument("text", new BsonString("lalala")) 126 | .append("number", new BsonInt32(1234)) 127 | .append("active", new BsonBoolean(false))); 128 | 129 | verifyResultsNoPK(valueDocRead); 130 | 131 | } 132 | 133 | private void verifyResultsNoPK(BsonDocument valueDoc) { 134 | 135 | //NOTE: for both filterDoc and replacementDoc _id is a generated ObjectId 136 | //which cannot be set from outside for testing thus it is set 137 | //by taking it from the resulting writeModel in order to do an equals comparison 138 | //for all contained fields 139 | 140 | BsonDocument filterDoc = new BsonDocument(); 141 | 142 | BsonDocument replacementDoc = 143 | new BsonDocument("text", new BsonString("lalala")) 144 | .append("number", new BsonInt32(1234)) 145 | .append("active", new BsonBoolean(false)); 146 | 147 | BsonDocument keyDoc = new BsonDocument(); 148 | 149 | WriteModel result = 150 | RDBMS_INSERT.perform(new SinkDocument(keyDoc,valueDoc)); 151 | 152 | assertTrue(result instanceof ReplaceOneModel, 153 | () -> "result expected to be of type ReplaceOneModel"); 154 | 155 | ReplaceOneModel writeModel = 156 | (ReplaceOneModel) result; 157 | 158 | assertTrue(writeModel.getReplacement().isObjectId(DBCollection.ID_FIELD_NAME), 159 | () -> "replacement doc must contain _id field of type ObjectID"); 160 | 161 | replacementDoc.put(DBCollection.ID_FIELD_NAME, 162 | writeModel.getReplacement().get(DBCollection.ID_FIELD_NAME,new BsonObjectId())); 163 | 164 | assertEquals(replacementDoc,writeModel.getReplacement(), 165 | ()-> "replacement doc not matching what is expected"); 166 | 167 | assertTrue(writeModel.getFilter() instanceof BsonDocument, 168 | () -> "filter expected to be of type BsonDocument"); 169 | 170 | assertTrue(((BsonDocument)writeModel.getFilter()).isObjectId(DBCollection.ID_FIELD_NAME), 171 | () -> "filter doc must contain _id field of type ObjectID"); 172 | 173 | filterDoc.put(DBCollection.ID_FIELD_NAME, 174 | ((BsonDocument)writeModel.getFilter()).get(DBCollection.ID_FIELD_NAME,new BsonObjectId())); 175 | 176 | assertEquals(filterDoc,writeModel.getFilter()); 177 | 178 | assertTrue(writeModel.getOptions().isUpsert(), 179 | () -> "replacement expected to be done in upsert mode"); 180 | } 181 | 182 | @Test 183 | @DisplayName("when missing key doc then DataException") 184 | public void testMissingKeyDocument() { 185 | assertThrows(DataException.class,() -> 186 | RDBMS_INSERT.perform(new SinkDocument(null, new BsonDocument())) 187 | ); 188 | } 189 | 190 | @Test 191 | @DisplayName("when missing value doc then DataException") 192 | public void testMissingValueDocument() { 193 | assertThrows(DataException.class,() -> 194 | RDBMS_INSERT.perform(new SinkDocument(new BsonDocument(),null)) 195 | ); 196 | } 197 | 198 | @Test 199 | @DisplayName("when invalid json in value doc 'after' field then DataException") 200 | public void testInvalidAfterField() { 201 | assertThrows(DataException.class,() -> 202 | RDBMS_INSERT.perform( 203 | new SinkDocument(new BsonDocument(), 204 | new BsonDocument("op",new BsonString("c")) 205 | .append("after",new BsonString("{NO : JSON [HERE] GO : AWAY}"))) 206 | ) 207 | ); 208 | } 209 | 210 | } 211 | -------------------------------------------------------------------------------- /src/test/java/at/grahsl/kafka/connect/mongodb/cdc/debezium/rdbms/RdbmsNoOpTest.java: -------------------------------------------------------------------------------- 1 | package at.grahsl.kafka.connect.mongodb.cdc.debezium.rdbms; 2 | 3 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 4 | import org.junit.jupiter.api.DisplayName; 5 | import org.junit.jupiter.api.Test; 6 | import org.junit.platform.runner.JUnitPlatform; 7 | import org.junit.runner.RunWith; 8 | 9 | import static org.junit.jupiter.api.Assertions.assertAll; 10 | import static org.junit.jupiter.api.Assertions.assertNull; 11 | 12 | @RunWith(JUnitPlatform.class) 13 | public class RdbmsNoOpTest { 14 | 15 | @Test 16 | @DisplayName("when any cdc event then WriteModel is null resulting in Optional.empty() in the corresponding handler") 17 | public void testValidSinkDocument() { 18 | assertAll("test behaviour of MongoDbNoOp", 19 | () -> assertNull(new RdbmsNoOp().perform(new SinkDocument(null,null)),"RdbmsNoOp must result in null WriteModel"), 20 | () -> assertNull(new RdbmsNoOp().perform(null),"RdbmsNoOp must result in null WriteModel") 21 | ); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/test/java/at/grahsl/kafka/connect/mongodb/converter/SinkConverterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter; 18 | 19 | import org.apache.kafka.connect.data.Schema; 20 | import org.apache.kafka.connect.data.SchemaBuilder; 21 | import org.apache.kafka.connect.data.Struct; 22 | import org.apache.kafka.connect.errors.DataException; 23 | import org.apache.kafka.connect.sink.SinkRecord; 24 | import org.bson.BsonDocument; 25 | import org.bson.BsonString; 26 | import org.junit.jupiter.api.*; 27 | import org.junit.platform.runner.JUnitPlatform; 28 | import org.junit.runner.RunWith; 29 | 30 | import java.util.*; 31 | 32 | import static org.junit.jupiter.api.Assertions.*; 33 | import static org.junit.jupiter.api.DynamicTest.dynamicTest; 34 | 35 | @RunWith(JUnitPlatform.class) 36 | public class SinkConverterTest { 37 | 38 | public static String JSON_STRING_1; 39 | public static Schema OBJ_SCHEMA_1; 40 | public static Struct OBJ_STRUCT_1; 41 | public static Map OBJ_MAP_1; 42 | public static BsonDocument EXPECTED_BSON_DOC; 43 | 44 | private static Map combinations; 45 | private SinkConverter sinkConverter = new SinkConverter(); 46 | 47 | 48 | @BeforeAll 49 | public static void initializeTestData() { 50 | 51 | JSON_STRING_1 = "{\"myField\":\"some text\"}"; 52 | 53 | OBJ_SCHEMA_1 = SchemaBuilder.struct() 54 | .field("myField", Schema.STRING_SCHEMA); 55 | 56 | OBJ_STRUCT_1 = new Struct(OBJ_SCHEMA_1) 57 | .put("myField", "some text"); 58 | 59 | OBJ_MAP_1 = new LinkedHashMap<>(); 60 | OBJ_MAP_1.put("myField", "some text"); 61 | 62 | EXPECTED_BSON_DOC = new BsonDocument("myField", new BsonString("some text")); 63 | 64 | combinations = new HashMap<>(); 65 | combinations.put(JSON_STRING_1, null); 66 | combinations.put(OBJ_STRUCT_1, OBJ_SCHEMA_1); 67 | combinations.put(OBJ_MAP_1, null); 68 | } 69 | 70 | @TestFactory 71 | @DisplayName("test different combinations for sink record conversions") 72 | public List testDifferentOptionsForSinkRecordConversion() { 73 | 74 | List tests = new ArrayList<>(); 75 | 76 | for (Map.Entry entry : combinations.entrySet()) { 77 | 78 | tests.add(dynamicTest("key only SinkRecord conversion for type " + entry.getKey().getClass().getName() 79 | + " with data -> " + entry.getKey(), () -> { 80 | SinkDocument converted = sinkConverter.convert( 81 | new SinkRecord( 82 | "topic", 1, entry.getValue(), entry.getKey(), null, null, 0L 83 | ) 84 | ); 85 | assertAll("checks on conversion results", 86 | () -> assertNotNull(converted), 87 | () -> assertEquals(EXPECTED_BSON_DOC, converted.getKeyDoc().get()), 88 | () -> assertEquals(Optional.empty(), converted.getValueDoc()) 89 | ); 90 | })); 91 | 92 | tests.add(dynamicTest("value only SinkRecord conversion for type " + entry.getKey().getClass().getName() 93 | + " with data -> " + entry.getKey(), () -> { 94 | SinkDocument converted = sinkConverter.convert( 95 | new SinkRecord( 96 | "topic", 1, null, null, entry.getValue(), entry.getKey(), 0L 97 | ) 98 | ); 99 | assertAll("checks on conversion results", 100 | () -> assertNotNull(converted), 101 | () -> assertEquals(Optional.empty(), converted.getKeyDoc()), 102 | () -> assertEquals(EXPECTED_BSON_DOC, converted.getValueDoc().get()) 103 | ); 104 | })); 105 | 106 | tests.add(dynamicTest("key + value SinkRecord conversion for type " + entry.getKey().getClass().getName() 107 | + " with data -> " + entry.getKey(), () -> { 108 | SinkDocument converted = sinkConverter.convert( 109 | new SinkRecord( 110 | "topic", 1, entry.getValue(), entry.getKey(), entry.getValue(), entry.getKey(), 0L 111 | ) 112 | ); 113 | assertAll("checks on conversion results", 114 | () -> assertNotNull(converted), 115 | () -> assertEquals(EXPECTED_BSON_DOC, converted.getKeyDoc().get()), 116 | () -> assertEquals(EXPECTED_BSON_DOC, converted.getValueDoc().get()) 117 | ); 118 | })); 119 | 120 | } 121 | 122 | return tests; 123 | 124 | } 125 | 126 | @Test 127 | @DisplayName("test empty sink record conversion") 128 | public void testEmptySinkRecordConversion() { 129 | 130 | SinkDocument converted = sinkConverter.convert( 131 | new SinkRecord( 132 | "topic", 1, null, null, null, null, 0L 133 | ) 134 | ); 135 | 136 | assertAll("checks on conversion result", 137 | () -> assertNotNull(converted), 138 | () -> assertEquals(Optional.empty(), converted.getKeyDoc()), 139 | () -> assertEquals(Optional.empty(), converted.getValueDoc()) 140 | ); 141 | 142 | } 143 | 144 | @Test 145 | @DisplayName("test invalid sink record conversion") 146 | public void testInvalidSinkRecordConversion() { 147 | 148 | assertAll("checks on conversion result", 149 | () -> assertThrows(DataException.class, () -> sinkConverter.convert( 150 | new SinkRecord( 151 | "topic", 1, null, new Object(), null, null, 0L 152 | ) 153 | )), 154 | () -> assertThrows(DataException.class, () -> sinkConverter.convert( 155 | new SinkRecord( 156 | "topic", 1, null, null, null, new Object(), 0L 157 | ) 158 | )), 159 | () -> assertThrows(DataException.class, () -> sinkConverter.convert( 160 | new SinkRecord( 161 | "topic", 1, null, new Object(), null, new Object(), 0L 162 | ) 163 | )) 164 | ); 165 | 166 | } 167 | 168 | } 169 | -------------------------------------------------------------------------------- /src/test/java/at/grahsl/kafka/connect/mongodb/converter/SinkDocumentTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.converter; 18 | 19 | import org.bson.*; 20 | import org.bson.types.ObjectId; 21 | import org.junit.jupiter.api.BeforeAll; 22 | import org.junit.jupiter.api.DisplayName; 23 | import org.junit.jupiter.api.Test; 24 | import org.junit.platform.runner.JUnitPlatform; 25 | import org.junit.runner.RunWith; 26 | 27 | import java.util.Arrays; 28 | 29 | import static org.junit.jupiter.api.Assertions.*; 30 | 31 | @RunWith(JUnitPlatform.class) 32 | public class SinkDocumentTest { 33 | 34 | private static BsonDocument flatStructKey; 35 | private static BsonDocument flatStructValue; 36 | 37 | private static BsonDocument nestedStructKey; 38 | private static BsonDocument nestedStructValue; 39 | 40 | @BeforeAll 41 | public static void initBsonDocs() { 42 | 43 | flatStructKey = new BsonDocument(); 44 | flatStructKey.put("_id", new BsonObjectId(ObjectId.get())); 45 | flatStructKey.put("myBoolean",new BsonBoolean(true)); 46 | flatStructKey.put("myInt",new BsonInt32(42)); 47 | flatStructKey.put("myBytes",new BsonBinary(new byte[] {65,66,67})); 48 | BsonArray ba1 = new BsonArray(); 49 | ba1.addAll(Arrays.asList(new BsonInt32(1),new BsonInt32(2),new BsonInt32(3))); 50 | flatStructKey.put("myArray", ba1); 51 | 52 | flatStructValue = new BsonDocument(); 53 | flatStructValue.put("myLong",new BsonInt64(42L)); 54 | flatStructValue.put("myDouble",new BsonDouble(23.23d)); 55 | flatStructValue.put("myString",new BsonString("BSON")); 56 | flatStructValue.put("myBytes",new BsonBinary(new byte[] {120,121,122})); 57 | BsonArray ba2 = new BsonArray(); 58 | ba2.addAll(Arrays.asList(new BsonInt32(9),new BsonInt32(8),new BsonInt32(7))); 59 | flatStructValue.put("myArray", ba2); 60 | 61 | nestedStructKey = new BsonDocument(); 62 | nestedStructKey.put("_id", new BsonDocument("myString", new BsonString("doc"))); 63 | nestedStructKey.put("mySubDoc", new BsonDocument("mySubSubDoc", 64 | new BsonDocument("myInt",new BsonInt32(23)))); 65 | 66 | nestedStructValue = new BsonDocument(); 67 | nestedStructValue.put("mySubDocA", new BsonDocument("myBoolean", new BsonBoolean(false))); 68 | nestedStructValue.put("mySubDocB", new BsonDocument("mySubSubDocC", 69 | new BsonDocument("myString",new BsonString("some text...")))); 70 | 71 | } 72 | 73 | @Test 74 | @DisplayName("test SinkDocument clone with missing key / value") 75 | public void testCloneNoKeyValue() { 76 | 77 | SinkDocument orig = new SinkDocument(null,null); 78 | 79 | assertAll("orig key/value docs NOT present", 80 | () -> assertFalse(orig.getKeyDoc().isPresent()), 81 | () -> assertFalse(orig.getValueDoc().isPresent()) 82 | ); 83 | 84 | SinkDocument clone = orig.clone(); 85 | 86 | assertAll("clone key/value docs NOT present", 87 | () -> assertFalse(clone.getKeyDoc().isPresent()), 88 | () -> assertFalse(clone.getValueDoc().isPresent()) 89 | ); 90 | 91 | } 92 | 93 | @Test 94 | @DisplayName("test SinkDocument clone of flat key / value") 95 | public void testCloneFlatKeyValue() { 96 | 97 | SinkDocument orig = new SinkDocument(flatStructKey, flatStructValue); 98 | 99 | checkClonedAsserations(orig); 100 | 101 | } 102 | 103 | @Test 104 | @DisplayName("test SinkDocument clone of nested key / value") 105 | public void testCloneNestedKeyValue() { 106 | 107 | SinkDocument orig = new SinkDocument(nestedStructKey, nestedStructValue); 108 | 109 | checkClonedAsserations(orig); 110 | 111 | } 112 | 113 | private void checkClonedAsserations(SinkDocument orig) { 114 | 115 | assertAll("orig key/value docs present", 116 | () -> assertTrue(orig.getKeyDoc().isPresent()), 117 | () -> assertTrue(orig.getValueDoc().isPresent()) 118 | ); 119 | 120 | SinkDocument clone = orig.clone(); 121 | 122 | assertAll("clone key/value docs present", 123 | () -> assertTrue(clone.getKeyDoc().isPresent()), 124 | () -> assertTrue(clone.getValueDoc().isPresent()) 125 | ); 126 | 127 | assertAll("check equality of key/value BSON document structure of clone vs. orig", 128 | () -> assertTrue(clone.getKeyDoc().get().equals(orig.getKeyDoc().get())), 129 | () -> assertTrue(clone.getValueDoc().get().equals(orig.getValueDoc().get())) 130 | ); 131 | } 132 | 133 | } 134 | -------------------------------------------------------------------------------- /src/test/java/at/grahsl/kafka/connect/mongodb/processor/DocumentIdAdderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 20 | import at.grahsl.kafka.connect.mongodb.processor.id.strategy.IdStrategy; 21 | import com.mongodb.DBCollection; 22 | import org.bson.BsonDocument; 23 | import org.bson.BsonValue; 24 | import org.junit.jupiter.api.DisplayName; 25 | import org.junit.jupiter.api.Test; 26 | import org.junit.platform.runner.JUnitPlatform; 27 | import org.junit.runner.RunWith; 28 | import org.mockito.ArgumentMatchers; 29 | 30 | import static org.junit.jupiter.api.Assertions.assertAll; 31 | import static org.junit.jupiter.api.Assertions.assertTrue; 32 | import static org.mockito.Mockito.*; 33 | 34 | @RunWith(JUnitPlatform.class) 35 | public class DocumentIdAdderTest { 36 | 37 | @Test 38 | @DisplayName("test _id field added by IdStrategy") 39 | public void testAddingIdFieldByStrategy() { 40 | 41 | BsonValue fakeId = mock(BsonValue.class); 42 | 43 | IdStrategy ids = mock(IdStrategy.class); 44 | when(ids.generateId(any(SinkDocument.class), ArgumentMatchers.isNull())) 45 | .thenReturn(fakeId); 46 | 47 | DocumentIdAdder idAdder = new DocumentIdAdder(null,ids,""); 48 | SinkDocument sinkDocWithValueDoc = new SinkDocument(null,new BsonDocument()); 49 | SinkDocument sinkDocWithoutValueDoc = new SinkDocument(null,null); 50 | 51 | assertAll("check for _id field when processing DocumentIdAdder", 52 | () -> { 53 | idAdder.process(sinkDocWithValueDoc,null); 54 | assertAll("_id checks", 55 | () -> assertTrue(sinkDocWithValueDoc.getValueDoc().orElseGet(() -> new BsonDocument()) 56 | .keySet().contains(DBCollection.ID_FIELD_NAME), 57 | "must contain _id field in valueDoc" 58 | ), 59 | () -> assertTrue(sinkDocWithValueDoc.getValueDoc().orElseGet(() -> new BsonDocument()) 60 | .get(DBCollection.ID_FIELD_NAME) instanceof BsonValue, 61 | "_id field must be of type BsonValue") 62 | ); 63 | }, 64 | () -> { 65 | idAdder.process(sinkDocWithoutValueDoc,null); 66 | assertTrue(!sinkDocWithoutValueDoc.getValueDoc().isPresent(), 67 | "no _id added since valueDoc cannot not be present" 68 | ); 69 | } 70 | ); 71 | 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /src/test/java/at/grahsl/kafka/connect/mongodb/processor/field/renaming/RenamerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package at.grahsl.kafka.connect.mongodb.processor.field.renaming; 18 | 19 | import at.grahsl.kafka.connect.mongodb.converter.SinkDocument; 20 | import org.bson.*; 21 | import org.junit.jupiter.api.BeforeAll; 22 | import org.junit.jupiter.api.BeforeEach; 23 | import org.junit.jupiter.api.DisplayName; 24 | import org.junit.jupiter.api.Test; 25 | import org.junit.platform.runner.JUnitPlatform; 26 | import org.junit.runner.RunWith; 27 | 28 | import java.util.HashMap; 29 | import java.util.Map; 30 | 31 | import static org.junit.jupiter.api.Assertions.assertAll; 32 | import static org.junit.jupiter.api.Assertions.assertEquals; 33 | 34 | @RunWith(JUnitPlatform.class) 35 | public class RenamerTest { 36 | 37 | public static BsonDocument keyDoc; 38 | public static BsonDocument valueDoc; 39 | 40 | public static Map fieldnameMappings; 41 | public static BsonDocument expectedKeyDocFieldnameMapping; 42 | public static BsonDocument expectedValueDocFieldnameMapping; 43 | 44 | public static Map regExpSettings; 45 | public static BsonDocument expectedKeyDocRegExpSettings; 46 | public static BsonDocument expectedValueDocRegExpSettings; 47 | 48 | @BeforeEach 49 | public void setupDocumentsToRename() { 50 | keyDoc = new BsonDocument("fieldA",new BsonString("my field value")); 51 | keyDoc.put("f2",new BsonBoolean(true)); 52 | keyDoc.put("subDoc",new BsonDocument("fieldX",new BsonInt32(42))); 53 | keyDoc.put("my_field1",new BsonDocument("my_field2",new BsonString("testing rocks!"))); 54 | 55 | valueDoc = new BsonDocument("abc",new BsonString("my field value")); 56 | valueDoc.put("f2",new BsonBoolean(false)); 57 | valueDoc.put("subDoc",new BsonDocument("123",new BsonDouble(0.0))); 58 | valueDoc.put("foo.foo.foo",new BsonDocument(".blah..blah.",new BsonInt32(23))); 59 | } 60 | 61 | @BeforeAll 62 | public static void setupDocumentsToCompare() { 63 | expectedKeyDocFieldnameMapping = new BsonDocument("f1",new BsonString("my field value")); 64 | expectedKeyDocFieldnameMapping.put("fieldB",new BsonBoolean(true)); 65 | expectedKeyDocFieldnameMapping.put("subDoc",new BsonDocument("name_x",new BsonInt32(42))); 66 | expectedKeyDocFieldnameMapping.put("my_field1",new BsonDocument("my_field2",new BsonString("testing rocks!"))); 67 | 68 | expectedValueDocFieldnameMapping = new BsonDocument("xyz",new BsonString("my field value")); 69 | expectedValueDocFieldnameMapping.put("f_two",new BsonBoolean(false)); 70 | expectedValueDocFieldnameMapping.put("subDoc",new BsonDocument("789",new BsonDouble(0.0))); 71 | expectedValueDocFieldnameMapping.put("foo.foo.foo",new BsonDocument(".blah..blah.",new BsonInt32(23))); 72 | 73 | expectedKeyDocRegExpSettings = new BsonDocument("FA",new BsonString("my field value")); 74 | expectedKeyDocRegExpSettings.put("f2",new BsonBoolean(true)); 75 | expectedKeyDocRegExpSettings.put("subDoc",new BsonDocument("FX",new BsonInt32(42))); 76 | expectedKeyDocRegExpSettings.put("_F1",new BsonDocument("_F2",new BsonString("testing rocks!"))); 77 | 78 | expectedValueDocRegExpSettings = new BsonDocument("abc",new BsonString("my field value")); 79 | expectedValueDocRegExpSettings.put("f2",new BsonBoolean(false)); 80 | expectedValueDocRegExpSettings.put("subDoc",new BsonDocument("123",new BsonDouble(0.0))); 81 | expectedValueDocRegExpSettings.put("foo_foo_foo",new BsonDocument("_blah__blah_",new BsonInt32(23))); 82 | } 83 | 84 | @BeforeAll 85 | public static void setupRenamerSettings() { 86 | fieldnameMappings = new HashMap<>(); 87 | fieldnameMappings.put(Renamer.PATH_PREFIX_KEY+".fieldA","f1"); 88 | fieldnameMappings.put(Renamer.PATH_PREFIX_KEY+".f2","fieldB"); 89 | fieldnameMappings.put(Renamer.PATH_PREFIX_KEY+".subDoc.fieldX","name_x"); 90 | fieldnameMappings.put(Renamer.PATH_PREFIX_VALUE+".abc","xyz"); 91 | fieldnameMappings.put(Renamer.PATH_PREFIX_VALUE+".f2","f_two"); 92 | fieldnameMappings.put(Renamer.PATH_PREFIX_VALUE+".subDoc.123","789"); 93 | 94 | regExpSettings = new HashMap<>(); 95 | regExpSettings.put("^"+Renamer.PATH_PREFIX_KEY+"\\..*my.*$",new RenameByRegExp.PatternReplace("my","")); 96 | regExpSettings.put("^"+Renamer.PATH_PREFIX_KEY+"\\..*field.*$",new RenameByRegExp.PatternReplace("field","F")); 97 | regExpSettings.put("^"+Renamer.PATH_PREFIX_VALUE+"\\..*$",new RenameByRegExp.PatternReplace("\\.","_")); 98 | } 99 | 100 | 101 | 102 | @Test 103 | @DisplayName("simple field renamer test with custom field name mappings") 104 | public void testRenamerUsingFieldnameMapping() { 105 | 106 | SinkDocument sd = new SinkDocument(keyDoc, valueDoc); 107 | Renamer renamer = new RenameByMapping(null, fieldnameMappings, ""); 108 | renamer.process(sd, null); 109 | 110 | assertAll("key and value doc checks", 111 | () -> assertEquals(expectedKeyDocFieldnameMapping,sd.getKeyDoc().orElse(new BsonDocument())), 112 | () -> assertEquals(expectedValueDocFieldnameMapping,sd.getValueDoc().orElse(new BsonDocument())) 113 | ); 114 | 115 | } 116 | 117 | @Test 118 | @DisplayName("simple field renamer test with custom regexp settings") 119 | public void testRenamerUsingRegExpSettings() { 120 | 121 | SinkDocument sd = new SinkDocument(keyDoc, valueDoc); 122 | Renamer renamer = new RenameByRegExp(null, regExpSettings, ""); 123 | renamer.process(sd, null); 124 | 125 | assertAll("key and value doc checks", 126 | () -> assertEquals(expectedKeyDocRegExpSettings,sd.getKeyDoc().orElse(new BsonDocument())), 127 | () -> assertEquals(expectedValueDocRegExpSettings,sd.getValueDoc().orElse(new BsonDocument())) 128 | ); 129 | 130 | } 131 | 132 | } 133 | -------------------------------------------------------------------------------- /src/test/resources/avro/tweetmsg.avsc: -------------------------------------------------------------------------------- 1 | {"namespace": "at.grahsl.kafka.connect.mongodb.data.avro", 2 | "type": "record", 3 | "name": "TweetMsg", 4 | "fields": [ 5 | {"name": "_id", "type": "long"}, 6 | {"name": "text", "type": "string"}, 7 | {"name": "hashtags", "type": {"type": "array", "items": "string"}} 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /src/test/resources/config/sink_connector.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "e2e-test-mongo-sink", 3 | "config": { 4 | "connector.class": "at.grahsl.kafka.connect.mongodb.MongoDbSinkConnector", 5 | "topics": "e2e-test-topic", 6 | "mongodb.connection.uri": "mongodb://mongodb:27017/kafkaconnect?w=1&journal=true", 7 | "mongodb.document.id.strategy": "at.grahsl.kafka.connect.mongodb.processor.id.strategy.ProvidedInValueStrategy", 8 | "mongodb.collection": "e2e-test-collection", 9 | "key.converter":"io.confluent.connect.avro.AvroConverter", 10 | "key.converter.schema.registry.url":"http://schemaregistry:8081", 11 | "value.converter":"io.confluent.connect.avro.AvroConverter", 12 | "value.converter.schema.registry.url":"http://schemaregistry:8081" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/test/resources/docker/compose-env.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: '2' 3 | services: 4 | mongodb: 5 | image: mongo:4.2.2 6 | hostname: mongodb 7 | ports: 8 | - "27017:27017" 9 | 10 | zookeeper: 11 | image: confluentinc/cp-zookeeper:5.3.2 12 | hostname: zookeeper 13 | ports: 14 | - "2181:2181" 15 | environment: 16 | ZOOKEEPER_CLIENT_PORT: 2181 17 | ZOOKEEPER_TICK_TIME: 2000 18 | 19 | kafkabroker: 20 | image: confluentinc/cp-kafka:5.3.2 21 | hostname: kafkabroker 22 | depends_on: 23 | - zookeeper 24 | ports: 25 | - "9092:9092" 26 | environment: 27 | KAFKA_BROKER_ID: 1 28 | KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' 29 | KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafkabroker:9092' 30 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 31 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 32 | 33 | schemaregistry: 34 | image: confluentinc/cp-schema-registry:5.3.2 35 | hostname: schemaregistry 36 | depends_on: 37 | - zookeeper 38 | - kafkabroker 39 | ports: 40 | - "8081:8081" 41 | environment: 42 | SCHEMA_REGISTRY_HOST_NAME: schemaregistry 43 | SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' 44 | 45 | kafkaconnect: 46 | image: confluentinc/cp-kafka-connect:5.3.2 47 | hostname: kafkaconnect 48 | depends_on: 49 | - zookeeper 50 | - kafkabroker 51 | - schemaregistry 52 | ports: 53 | - "8083:8083" 54 | volumes: 55 | - ../../../../target/kafka-connect-mongodb/:/etc/kafka-connect/jars/kafka-connect-mongodb/ 56 | environment: 57 | CONNECT_BOOTSTRAP_SERVERS: 'kafkabroker:9092' 58 | CONNECT_REST_ADVERTISED_HOST_NAME: kafkaconnect 59 | CONNECT_REST_PORT: 8083 60 | CONNECT_GROUP_ID: compose-connect-group 61 | CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs 62 | CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 63 | CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 64 | CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets 65 | CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 66 | CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status 67 | CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 68 | CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter 69 | CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schemaregistry:8081' 70 | CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter 71 | CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schemaregistry:8081' 72 | CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter 73 | CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter 74 | CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181' 75 | CONNECT_PLUGIN_PATH: '/usr/share/java,/etc/kafka-connect/jars' --------------------------------------------------------------------------------