├── .gitignore
├── .travis.yml
├── LICENSE.txt
├── README.md
├── assets
└── mongodb-leaf-only.png
├── build.gradle
├── config
└── MongoDbSinkConnector.properties
├── docs
└── logos
│ ├── qudosoft.png
│ └── runtitle.png
├── pom.xml
└── src
├── main
├── assembly
│ └── package.xml
├── java
│ └── at
│ │ └── grahsl
│ │ └── kafka
│ │ └── connect
│ │ └── mongodb
│ │ ├── CollectionAwareConfig.java
│ │ ├── MongoDbSinkConnector.java
│ │ ├── MongoDbSinkConnectorConfig.java
│ │ ├── MongoDbSinkRecordBatches.java
│ │ ├── MongoDbSinkTask.java
│ │ ├── VersionUtil.java
│ │ ├── cdc
│ │ ├── CdcHandler.java
│ │ ├── CdcOperation.java
│ │ └── debezium
│ │ │ ├── DebeziumCdcHandler.java
│ │ │ ├── OperationType.java
│ │ │ ├── mongodb
│ │ │ ├── MongoDbDelete.java
│ │ │ ├── MongoDbHandler.java
│ │ │ ├── MongoDbInsert.java
│ │ │ ├── MongoDbNoOp.java
│ │ │ └── MongoDbUpdate.java
│ │ │ └── rdbms
│ │ │ ├── RdbmsDelete.java
│ │ │ ├── RdbmsHandler.java
│ │ │ ├── RdbmsInsert.java
│ │ │ ├── RdbmsNoOp.java
│ │ │ ├── RdbmsUpdate.java
│ │ │ ├── mysql
│ │ │ └── MysqlHandler.java
│ │ │ └── postgres
│ │ │ └── PostgresHandler.java
│ │ ├── converter
│ │ ├── AvroJsonSchemafulRecordConverter.java
│ │ ├── FieldConverter.java
│ │ ├── JsonRawStringRecordConverter.java
│ │ ├── JsonSchemalessRecordConverter.java
│ │ ├── RecordConverter.java
│ │ ├── SinkConverter.java
│ │ ├── SinkDocument.java
│ │ ├── SinkFieldConverter.java
│ │ └── types
│ │ │ └── sink
│ │ │ └── bson
│ │ │ ├── BooleanFieldConverter.java
│ │ │ ├── BytesFieldConverter.java
│ │ │ ├── Float32FieldConverter.java
│ │ │ ├── Float64FieldConverter.java
│ │ │ ├── Int16FieldConverter.java
│ │ │ ├── Int32FieldConverter.java
│ │ │ ├── Int64FieldConverter.java
│ │ │ ├── Int8FieldConverter.java
│ │ │ ├── StringFieldConverter.java
│ │ │ └── logical
│ │ │ ├── DateFieldConverter.java
│ │ │ ├── DecimalFieldConverter.java
│ │ │ ├── TimeFieldConverter.java
│ │ │ └── TimestampFieldConverter.java
│ │ ├── processor
│ │ ├── BlacklistKeyProjector.java
│ │ ├── BlacklistValueProjector.java
│ │ ├── DocumentIdAdder.java
│ │ ├── KafkaMetaAdder.java
│ │ ├── PostProcessor.java
│ │ ├── WhitelistKeyProjector.java
│ │ ├── WhitelistValueProjector.java
│ │ ├── field
│ │ │ ├── projection
│ │ │ │ ├── BlacklistProjector.java
│ │ │ │ ├── FieldProjector.java
│ │ │ │ └── WhitelistProjector.java
│ │ │ └── renaming
│ │ │ │ ├── FieldnameMapping.java
│ │ │ │ ├── RegExpSettings.java
│ │ │ │ ├── RenameByMapping.java
│ │ │ │ ├── RenameByRegExp.java
│ │ │ │ └── Renamer.java
│ │ └── id
│ │ │ └── strategy
│ │ │ ├── BsonOidStrategy.java
│ │ │ ├── FullKeyStrategy.java
│ │ │ ├── IdStrategy.java
│ │ │ ├── KafkaMetaDataStrategy.java
│ │ │ ├── PartialKeyStrategy.java
│ │ │ ├── PartialValueStrategy.java
│ │ │ ├── ProvidedInKeyStrategy.java
│ │ │ ├── ProvidedInValueStrategy.java
│ │ │ ├── ProvidedStrategy.java
│ │ │ └── UuidStrategy.java
│ │ └── writemodel
│ │ └── strategy
│ │ ├── DeleteOneDefaultStrategy.java
│ │ ├── MonotonicWritesDefaultStrategy.java
│ │ ├── ReplaceOneBusinessKeyStrategy.java
│ │ ├── ReplaceOneDefaultStrategy.java
│ │ ├── UpdateOneTimestampsStrategy.java
│ │ └── WriteModelStrategy.java
└── resources
│ ├── kafka-connect-mongodb-version.properties
│ └── logback.xml
└── test
├── java
└── at
│ └── grahsl
│ └── kafka
│ └── connect
│ └── mongodb
│ ├── MongoDbSinkConnectorConfigTest.java
│ ├── MongoDbSinkRecordBatchesTest.java
│ ├── MongoDbSinkTaskTest.java
│ ├── ValidatorWithOperatorsTest.java
│ ├── cdc
│ └── debezium
│ │ ├── OperationTypeTest.java
│ │ ├── mongodb
│ │ ├── MongoDbDeleteTest.java
│ │ ├── MongoDbHandlerTest.java
│ │ ├── MongoDbInsertTest.java
│ │ ├── MongoDbNoOpTest.java
│ │ └── MongoDbUpdateTest.java
│ │ └── rdbms
│ │ ├── RdbmsDeleteTest.java
│ │ ├── RdbmsHandlerTest.java
│ │ ├── RdbmsInsertTest.java
│ │ ├── RdbmsNoOpTest.java
│ │ └── RdbmsUpdateTest.java
│ ├── converter
│ ├── RecordConverterTest.java
│ ├── SinkConverterTest.java
│ ├── SinkDocumentTest.java
│ └── SinkFieldConverterTest.java
│ ├── data
│ └── avro
│ │ └── TweetMsg.java
│ ├── end2end
│ └── MinimumViableIT.java
│ ├── processor
│ ├── DocumentIdAdderTest.java
│ ├── field
│ │ ├── projection
│ │ │ └── FieldProjectorTest.java
│ │ └── renaming
│ │ │ └── RenamerTest.java
│ └── id
│ │ └── strategy
│ │ └── IdStrategyTest.java
│ └── writemodel
│ └── strategy
│ └── WriteModelStrategyTest.java
└── resources
├── avro
└── tweetmsg.avsc
├── config
└── sink_connector.json
└── docker
└── compose-env.yml
/.gitignore:
--------------------------------------------------------------------------------
1 | .idea
2 | *.iml
3 | target/
4 |
5 | .gradle/
6 | build/
7 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | sudo: required
2 |
3 | services:
4 | - docker
5 |
6 | env:
7 | DOCKER_COMPOSE_VERSION=1.25.0
8 |
9 | addons:
10 | hosts:
11 | - zookeeper
12 | - kafkabroker
13 | - kafkaconnect
14 | - schemaregistry
15 | - mongodb
16 |
17 | language: java
18 |
19 | jdk:
20 | - openjdk8
21 | - openjdk11
22 |
23 | before_install:
24 | - sudo rm /usr/local/bin/docker-compose
25 | - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
26 | - chmod +x docker-compose
27 | - sudo mv docker-compose /usr/local/bin
28 |
29 | script:
30 | - mvn install -Dmaven.javadoc.skip=true -Dgpg.skip -B -V
31 |
32 | after_success:
33 | - mvn jacoco:prepare-agent test jacoco:report
34 | - mvn com.gavinmogan:codacy-maven-plugin:coverage -DcoverageReportFile=target/site/jacoco/jacoco.xml -DprojectToken=$CODACY_PROJECT_TOKEN -DapiToken=$CODACY_API_TOKEN
35 |
--------------------------------------------------------------------------------
/assets/mongodb-leaf-only.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hpgrahsl/kafka-connect-mongodb/772f543b363bf708fa4b2af44b148ac2474d492d/assets/mongodb-leaf-only.png
--------------------------------------------------------------------------------
/build.gradle:
--------------------------------------------------------------------------------
1 | apply plugin: 'java'
2 | apply plugin: 'maven'
3 | apply plugin: 'com.github.johnrengelman.shadow'
4 |
5 | group = 'at.grahsl.kafka.connect'
6 | version = '1.4.0'
7 |
8 | description = """kafka-connect-mongodb"""
9 |
10 | sourceCompatibility = 1.8
11 | targetCompatibility = 1.8
12 |
13 | tasks.withType(JavaCompile) {
14 | options.encoding = 'UTF-8'
15 | }
16 |
17 | buildscript {
18 | repositories {
19 | jcenter()
20 | }
21 | dependencies {
22 | classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.3'
23 | }
24 | }
25 |
26 | processResources {
27 | expand(project.properties)
28 | }
29 |
30 | def skipIntegrationTest = 'true'
31 | test {
32 | if (skipIntegrationTest.toBoolean()) {
33 | exclude '**/*IT.class'
34 | }
35 | }
36 |
37 | task copyJarToTarget(type: Copy, dependsOn:[jar, shadowJar]) {
38 | description 'Copies jar files to target directory'
39 | copy {
40 | from 'build/libs'
41 | into 'target'
42 | include '**/*.jar'
43 | }
44 | }
45 |
46 | repositories {
47 | maven { url "http://packages.confluent.io/maven/" }
48 | maven { url "http://repo.maven.apache.org/maven2" }
49 | }
50 |
51 | ext {
52 | kafkaVersion='2.4.0'
53 | mongodbDriverVersion='3.12.0'
54 | logbackVersion='1.2.3'
55 | jacksonVersion='2.10.1'
56 | confluentSerializerVersion='5.3.2'
57 | confluentConnectPluginVersion='0.11.2'
58 | junitJupiterVersion='5.5.2'
59 | junitPlatformVersion='1.5.2'
60 | hamcrestVersion='2.0.0.0'
61 | mockitoVersion='3.2.4'
62 | testcontainersVersion='1.12.3'
63 | avroVersion='1.9.1'
64 | okHttpVersion='3.14.4'
65 | yamlBeansVersion='1.13'
66 | }
67 |
68 | dependencies {
69 | compile "org.apache.kafka:connect-api:${kafkaVersion}"
70 | compile "org.mongodb:mongodb-driver:${mongodbDriverVersion}"
71 | compile "ch.qos.logback:logback-classic:${logbackVersion}"
72 | testCompile "com.github.jcustenborder.kafka.connect:connect-utils:[0.2.31,0.2.1000)"
73 | compile "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}"
74 | compile "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}"
75 | compile "io.confluent:kafka-avro-serializer:${confluentSerializerVersion}"
76 | compile "io.confluent:kafka-connect-maven-plugin:${confluentConnectPluginVersion}"
77 | testCompile "org.junit.jupiter:junit-jupiter-engine:${junitJupiterVersion}"
78 | testCompile "org.junit.jupiter:junit-jupiter-params:${junitJupiterVersion}"
79 | testCompile "org.junit.vintage:junit-vintage-engine:${junitJupiterVersion}"
80 | testCompile "org.junit.platform:junit-platform-runner:${junitPlatformVersion}"
81 | testCompile "org.junit.platform:junit-platform-console:${junitPlatformVersion}"
82 | testCompile "org.hamcrest:hamcrest-junit:${hamcrestVersion}"
83 | testCompile "org.mockito:mockito-core:${mockitoVersion}"
84 | testCompile "org.testcontainers:testcontainers:${testcontainersVersion}"
85 | testCompile "org.apache.avro:avro:${avroVersion}"
86 | testCompile "org.apache.avro:avro-maven-plugin:${avroVersion}"
87 | testCompile "com.squareup.okhttp3:okhttp:${okHttpVersion}"
88 | testCompile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jacksonVersion}"
89 | testCompile "com.esotericsoftware.yamlbeans:yamlbeans:${yamlBeansVersion}"
90 | }
91 |
--------------------------------------------------------------------------------
/config/MongoDbSinkConnector.properties:
--------------------------------------------------------------------------------
1 | ##
2 | # Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com)
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 | ##
16 |
17 | name=MyMongoDbSinkConnector
18 | topics=test
19 | tasks.max=1
20 |
21 | key.converter=io.confluent.connect.avro.AvroConverter
22 | key.converter.schema.registry.url=http://localhost:8081
23 | value.converter=io.confluent.connect.avro.AvroConverter
24 | value.converter.schema.registry.url=http://localhost:8081
25 |
26 | connector.class=at.grahsl.kafka.connect.mongodb.MongoDbSinkConnector
27 |
28 | #specific MongoDB sink connector props
29 | #listed below are the defaults
30 | mongodb.connection.uri=mongodb://localhost:27017/kafkaconnect?w=1&journal=true
31 | mongodb.collection=
32 | mongodb.max.num.retries=3
33 | mongodb.retries.defer.timeout=5000
34 | mongodb.value.projection.type=none
35 | mongodb.value.projection.list=
36 | mongodb.document.id.strategy=at.grahsl.kafka.connect.mongodb.processor.id.strategy.BsonOidStrategy
37 | mongodb.document.id.strategies=
38 | mongodb.key.projection.type=none
39 | mongodb.key.projection.list=
40 | mongodb.field.renamer.mapping=[]
41 | mongodb.field.renamer.regexp=[]
42 | mongodb.post.processor.chain=at.grahsl.kafka.connect.mongodb.processor.DocumentIdAdder
43 | mongodb.change.data.capture.handler=
44 | mongodb.change.data.capture.handler.operations=c,r,u,d
45 | mongodb.delete.on.null.values=false
46 | mongodb.writemodel.strategy=at.grahsl.kafka.connect.mongodb.writemodel.strategy.ReplaceOneDefaultStrategy
47 | mongodb.max.batch.size=0
48 | mongodb.rate.limiting.timeout=0
49 | mongodb.rate.limiting.every.n=0
50 |
--------------------------------------------------------------------------------
/docs/logos/qudosoft.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hpgrahsl/kafka-connect-mongodb/772f543b363bf708fa4b2af44b148ac2474d492d/docs/logos/qudosoft.png
--------------------------------------------------------------------------------
/docs/logos/runtitle.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hpgrahsl/kafka-connect-mongodb/772f543b363bf708fa4b2af44b148ac2474d492d/docs/logos/runtitle.png
--------------------------------------------------------------------------------
/src/main/assembly/package.xml:
--------------------------------------------------------------------------------
1 |
5 |
6 | package
7 |
8 | dir
9 |
10 | false
11 |
12 |
13 | ${project.basedir}
14 | share/doc/${project.name}/
15 |
16 | README*
17 | LICENSE*
18 | NOTICE*
19 | licenses/
20 |
21 |
22 |
23 | ${project.basedir}/config
24 | etc/${project.name}
25 |
26 | *
27 |
28 |
29 |
30 |
31 |
32 | share/java/${project.name}
33 | true
34 | true
35 |
36 | org.apache.kafka:connect-api
37 |
38 |
39 |
40 |
41 |
--------------------------------------------------------------------------------
/src/main/java/at/grahsl/kafka/connect/mongodb/CollectionAwareConfig.java:
--------------------------------------------------------------------------------
1 | package at.grahsl.kafka.connect.mongodb;
2 |
3 | import org.apache.kafka.common.config.AbstractConfig;
4 | import org.apache.kafka.common.config.ConfigDef;
5 | import org.apache.kafka.common.config.ConfigException;
6 |
7 | import java.util.HashMap;
8 | import java.util.Map;
9 |
10 | public class CollectionAwareConfig extends AbstractConfig {
11 |
12 | //NOTE: the merging of values() and originals() is a workaround
13 | //in order to allow for properties not being given in the
14 | //ConfigDef at compile time to be picked up and available as well...
15 | private final Map collectionAwareSettings;
16 |
17 | public CollectionAwareConfig(ConfigDef definition, Map, ?> originals, boolean doLog) {
18 | super(definition, originals, doLog);
19 | collectionAwareSettings = new HashMap<>(256);
20 | collectionAwareSettings.putAll(values());
21 | collectionAwareSettings.putAll(originals());
22 | }
23 |
24 | public CollectionAwareConfig(ConfigDef definition, Map, ?> originals) {
25 | super(definition, originals);
26 | collectionAwareSettings = new HashMap<>(256);
27 | collectionAwareSettings.putAll(values());
28 | collectionAwareSettings.putAll(originals());
29 | }
30 |
31 | protected Object get(String property, String collection) {
32 | String fullProperty = property+"."+collection;
33 | if(collectionAwareSettings.containsKey(fullProperty)) {
34 | return collectionAwareSettings.get(fullProperty);
35 | }
36 | return collectionAwareSettings.get(property);
37 | }
38 |
39 | public String getString(String property, String collection) {
40 | if(collection == null || collection.isEmpty()) {
41 | return (String) get(property);
42 | }
43 | return (String) get(property,collection);
44 | }
45 |
46 | //NOTE: in the this topic aware map, everything is currently stored as
47 | //type String so direct casting won't work which is why the
48 | //*.parse*(String value) methods are to be used for now.
49 | public Boolean getBoolean(String property, String collection) {
50 | Object obj;
51 |
52 | if(collection == null || collection.isEmpty()) {
53 | obj = get(property);
54 | } else {
55 | obj = get(property,collection);
56 | }
57 |
58 | if(obj instanceof Boolean)
59 | return (Boolean) obj;
60 |
61 | if(obj instanceof String)
62 | return Boolean.parseBoolean((String)obj);
63 |
64 | throw new ConfigException("error: unsupported property type for '"+obj+"' where Boolean expected");
65 | }
66 |
67 | public Integer getInt(String property, String collection) {
68 |
69 | Object obj;
70 |
71 | if(collection == null || collection.isEmpty()) {
72 | obj = get(property);
73 | } else {
74 | obj = get(property,collection);
75 | }
76 |
77 | if(obj instanceof Integer)
78 | return (Integer) obj;
79 |
80 | if(obj instanceof String)
81 | return Integer.parseInt((String)obj);
82 |
83 | throw new ConfigException("error: unsupported property type for '"+obj+"' where Integer expected");
84 | }
85 |
86 | }
87 |
--------------------------------------------------------------------------------
/src/main/java/at/grahsl/kafka/connect/mongodb/MongoDbSinkConnector.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2017. Hans-Peter Grahsl (grahslhp@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package at.grahsl.kafka.connect.mongodb;
18 |
19 | import org.apache.kafka.common.config.ConfigDef;
20 | import org.apache.kafka.connect.connector.Task;
21 | import org.apache.kafka.connect.sink.SinkConnector;
22 |
23 | import java.util.ArrayList;
24 | import java.util.List;
25 | import java.util.Map;
26 |
27 | public class MongoDbSinkConnector extends SinkConnector {
28 |
29 | private Map settings;
30 |
31 | @Override
32 | public String version() {
33 | return VersionUtil.getVersion();
34 | }
35 |
36 | @Override
37 | public void start(Map map) {
38 | settings = map;
39 | }
40 |
41 | @Override
42 | public Class extends Task> taskClass() {
43 | return MongoDbSinkTask.class;
44 | }
45 |
46 | @Override
47 | public List