├── docs └── README.adoc ├── assets └── couchbase_logo.png ├── src ├── main │ ├── resources │ │ ├── META-INF │ │ │ └── services │ │ │ │ ├── org.apache.kafka.connect.sink.SinkConnector │ │ │ │ ├── org.apache.kafka.connect.source.SourceConnector │ │ │ │ └── org.apache.kafka.connect.transforms.Transformation │ │ └── kafka-connect-couchbase-version.properties │ └── java │ │ └── com │ │ └── couchbase │ │ └── connect │ │ └── kafka │ │ ├── config │ │ ├── common │ │ │ ├── CommonConfig.java │ │ │ ├── LoggingConfig.java │ │ │ ├── ConnectionConfig.java │ │ │ └── SecurityConfig.java │ │ ├── source │ │ │ ├── CouchbaseSourceConfig.java │ │ │ ├── CouchbaseSourceTaskConfig.java │ │ │ └── SchemaConfig.java │ │ └── sink │ │ │ ├── CouchbaseSinkConfig.java │ │ │ ├── AnalyticsSinkHandlerConfig.java │ │ │ ├── DurabilityConfig.java │ │ │ ├── SubDocumentSinkHandlerConfig.java │ │ │ └── N1qlSinkHandlerConfig.java │ │ ├── filter │ │ ├── AllPassIncludingSystemFilter.java │ │ ├── Filter.java │ │ └── AllPassFilter.java │ │ ├── util │ │ ├── config │ │ │ ├── annotation │ │ │ │ ├── Dependents.java │ │ │ │ ├── Default.java │ │ │ │ ├── EnvironmentVariable.java │ │ │ │ ├── DisplayName.java │ │ │ │ ├── Width.java │ │ │ │ ├── Group.java │ │ │ │ ├── Importance.java │ │ │ │ └── ContextDocumentation.java │ │ │ ├── DataSize.java │ │ │ ├── BooleanParentRecommender.java │ │ │ ├── DataSizeValidator.java │ │ │ ├── Contextual.java │ │ │ ├── EnumRecommender.java │ │ │ ├── DurationValidator.java │ │ │ ├── EnumValidator.java │ │ │ ├── AbstractInvocationHandler.java │ │ │ ├── DataSizeParser.java │ │ │ ├── ConfigHelper.java │ │ │ ├── HtmlRenderer.java │ │ │ ├── DurationParser.java │ │ │ └── LookupTable.java │ │ ├── FirstCallTracker.java │ │ ├── Version.java │ │ ├── ConnectHelper.java │ │ ├── N1qlData.java │ │ ├── Schemas.java │ │ ├── DocumentIdExtractor.java │ │ ├── ListHelper.java │ │ ├── CouchbaseHelper.java │ │ ├── DurabilitySetter.java │ │ ├── JsonHelper.java │ │ ├── ScopeAndCollection.java │ │ ├── TopicMap.java │ │ ├── Watchdog.java │ │ └── BatchBuilder.java │ │ ├── handler │ │ ├── source │ │ │ ├── MutationMetadata.java │ │ │ ├── CollectionMetadata.java │ │ │ ├── SourceHandlerParams.java │ │ │ ├── CouchbaseSourceRecord.java │ │ │ ├── SourceHandler.java │ │ │ ├── ConfigurableSchemaSourceHandler.java │ │ │ ├── DefaultSchemaSourceHandler.java │ │ │ ├── CouchbaseHeaderSetter.java │ │ │ └── MultiSourceHandler.java │ │ └── sink │ │ │ ├── UpsertSinkHandler.java │ │ │ ├── SinkHandlerContext.java │ │ │ ├── SinkDocument.java │ │ │ ├── ConcurrencyHint.java │ │ │ └── SinkHandler.java │ │ ├── transform │ │ ├── DropIfNullValue.java │ │ └── DeserializeJson.java │ │ ├── StreamFrom.java │ │ ├── CouchbaseSinkConnector.java │ │ ├── SourceOffset.java │ │ └── ConnectorLifecycle.java └── test │ └── java │ └── com │ └── couchbase │ └── connect │ └── kafka │ ├── util │ ├── ScopeAndCollectionTest.java │ ├── Resources.java │ ├── config │ │ ├── SizeParserTest.java │ │ ├── HtmlRendererTest.java │ │ └── DurationParserTest.java │ ├── ListHelperTest.java │ ├── TopicMapTest.java │ └── KafkaRetryHelperTest.java │ ├── handler │ ├── source │ │ └── RawJsonSourceHandlerTest.java │ └── sink │ │ └── AnalyticsSinkHandlerTest.java │ └── CouchbaseSinkTaskTest.java ├── .gitignore ├── examples ├── custom-extensions │ ├── src │ │ └── main │ │ │ ├── resources │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── org.apache.kafka.connect.transforms.Transformation │ │ │ └── java │ │ │ └── com │ │ │ └── couchbase │ │ │ └── connect │ │ │ └── kafka │ │ │ └── example │ │ │ ├── CustomSourceHandler.java │ │ │ └── CustomSinkHandler.java │ ├── README.md │ └── pom.xml └── json-producer │ ├── README.md │ ├── pom.xml │ └── src │ └── main │ └── java │ └── com │ └── couchbase │ └── connect │ └── kafka │ └── example │ └── JsonProducerExample.java ├── .mvn └── wrapper │ └── maven-wrapper.properties ├── config ├── quickstart-couchbase-sink.json ├── quickstart-couchbase-source.json └── migrate-config-3-to-4.sh ├── HOWTO-DOCS-AND-BRANCHES.adoc ├── .github └── workflows │ ├── deploy-snapshot.yml │ └── deploy-release.yml ├── README.adoc └── CONTRIBUTING.adoc /docs/README.adoc: -------------------------------------------------------------------------------- 1 | Documentation repository: https://github.com/couchbase/docs-kafka 2 | -------------------------------------------------------------------------------- /assets/couchbase_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/couchbase/kafka-connect-couchbase/HEAD/assets/couchbase_logo.png -------------------------------------------------------------------------------- /src/main/resources/META-INF/services/org.apache.kafka.connect.sink.SinkConnector: -------------------------------------------------------------------------------- 1 | com.couchbase.connect.kafka.CouchbaseSinkConnector 2 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/services/org.apache.kafka.connect.source.SourceConnector: -------------------------------------------------------------------------------- 1 | com.couchbase.connect.kafka.CouchbaseSourceConnector 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Build products 2 | target/ 3 | .flattened-pom.xml 4 | 5 | # IntelliJ data 6 | *.iml 7 | .idea/ 8 | .ipr 9 | 10 | .DS_Store 11 | -------------------------------------------------------------------------------- /examples/custom-extensions/src/main/resources/META-INF/services/org.apache.kafka.connect.transforms.Transformation: -------------------------------------------------------------------------------- 1 | com.couchbase.connect.kafka.example.CustomTransform 2 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/services/org.apache.kafka.connect.transforms.Transformation: -------------------------------------------------------------------------------- 1 | com.couchbase.connect.kafka.transform.DropIfNullValue 2 | com.couchbase.connect.kafka.transform.DeserializeJson 3 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | wrapperVersion=3.3.4 2 | distributionType=only-script 3 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip 4 | -------------------------------------------------------------------------------- /src/main/resources/kafka-connect-couchbase-version.properties: -------------------------------------------------------------------------------- 1 | ## 2 | # Copyright 2016 Couchbase, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | ## 16 | 17 | version=${project.version} 18 | -------------------------------------------------------------------------------- /config/quickstart-couchbase-sink.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test-couchbase-sink", 3 | "config": { 4 | "name": "test-couchbase-sink", 5 | "connector.class": "com.couchbase.connect.kafka.CouchbaseSinkConnector", 6 | "tasks.max": "2", 7 | "topics": "couchbase-sink-example", 8 | "couchbase.seed.nodes": "127.0.0.1", 9 | "couchbase.bootstrap.timeout": "10s", 10 | "couchbase.bucket": "default", 11 | "couchbase.username": "Administrator", 12 | "couchbase.password": "password", 13 | "couchbase.durability": "NONE", 14 | "key.converter": "org.apache.kafka.connect.storage.StringConverter", 15 | "value.converter": "org.apache.kafka.connect.json.JsonConverter", 16 | "value.converter.schemas.enable": "false" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/config/common/CommonConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.config.common; 18 | 19 | public interface CommonConfig extends 20 | ConnectionConfig, 21 | SecurityConfig, 22 | LoggingConfig { 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/config/source/CouchbaseSourceConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.config.source; 18 | 19 | import com.couchbase.connect.kafka.config.common.CommonConfig; 20 | 21 | public interface CouchbaseSourceConfig extends 22 | CommonConfig, 23 | SourceBehaviorConfig, 24 | SchemaConfig, 25 | DcpConfig { 26 | } 27 | -------------------------------------------------------------------------------- /config/quickstart-couchbase-source.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test-couchbase-source", 3 | "config": { 4 | "name": "test-couchbase-source", 5 | "connector.class": "com.couchbase.connect.kafka.CouchbaseSourceConnector", 6 | "tasks.max": "2", 7 | "couchbase.topic": "test-default", 8 | "couchbase.seed.nodes": "127.0.0.1", 9 | "couchbase.bootstrap.timeout": "10s", 10 | "couchbase.bucket": "default", 11 | "couchbase.username": "Administrator", 12 | "couchbase.password": "password", 13 | "key.converter": "org.apache.kafka.connect.storage.StringConverter", 14 | "couchbase.source.handler": "com.couchbase.connect.kafka.handler.source.RawJsonSourceHandler", 15 | "value.converter": "org.apache.kafka.connect.converters.ByteArrayConverter", 16 | "couchbase.event.filter": "com.couchbase.connect.kafka.filter.AllPassFilter", 17 | "couchbase.stream.from": "SAVED_OFFSET_OR_BEGINNING", 18 | "couchbase.compression": "ENABLED", 19 | "couchbase.flow.control.buffer": "16m", 20 | "couchbase.persistence.polling.interval": "100ms" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/config/sink/CouchbaseSinkConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.config.sink; 18 | 19 | import com.couchbase.connect.kafka.config.common.CommonConfig; 20 | 21 | public interface CouchbaseSinkConfig extends 22 | CommonConfig, 23 | SinkBehaviorConfig, 24 | DurabilityConfig, 25 | N1qlSinkHandlerConfig, 26 | AnalyticsSinkHandlerConfig, 27 | SubDocumentSinkHandlerConfig { 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/filter/AllPassIncludingSystemFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.filter; 18 | 19 | import com.couchbase.connect.kafka.handler.source.DocumentEvent; 20 | 21 | /** 22 | * Allows publication of any event, including transaction metadata and events in system scopes. 23 | */ 24 | public class AllPassIncludingSystemFilter implements Filter { 25 | @Override 26 | public boolean pass(final DocumentEvent event) { 27 | return true; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/annotation/Dependents.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config.annotation; 18 | 19 | import java.lang.annotation.ElementType; 20 | import java.lang.annotation.Retention; 21 | import java.lang.annotation.RetentionPolicy; 22 | import java.lang.annotation.Target; 23 | 24 | /** 25 | * Indicates the named config keys are dependents of the annotated key. 26 | */ 27 | @Retention(RetentionPolicy.RUNTIME) 28 | @Target(ElementType.METHOD) 29 | public @interface Dependents { 30 | String[] value(); 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/annotation/Default.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config.annotation; 18 | 19 | import java.lang.annotation.ElementType; 20 | import java.lang.annotation.Retention; 21 | import java.lang.annotation.RetentionPolicy; 22 | import java.lang.annotation.Target; 23 | 24 | /** 25 | * Indicates the config key is optional, and specifies the default value. 26 | */ 27 | @Retention(RetentionPolicy.RUNTIME) 28 | @Target(ElementType.METHOD) 29 | public @interface Default { 30 | String value() default ""; 31 | } 32 | -------------------------------------------------------------------------------- /HOWTO-DOCS-AND-BRANCHES.adoc: -------------------------------------------------------------------------------- 1 | = Branch Management 2 | 3 | [abstract] 4 | A brief overview of how this project uses branches for code and documentation. 5 | 6 | The `master` branch is used for day-to-day development. 7 | 8 | There is one `release/*` branch for each previous minor release version. 9 | For example, if the current minor version is `2.1`, the branches might be `release/1.0`, `release/1.1`, `release/2.0`. 10 | 11 | The `master` branch is for the latest minor version. 12 | 13 | Patch versions (for example, `2.1.3`) are tagged on the `master` branch (or one of the release branches, in the rare case you need to do a new release on an older minor version). 14 | 15 | There's one set of documentation for each minor version. 16 | The documentation lives in a separate repository: https://github.com/couchbase/docs-kafka 17 | 18 | The documentation repository uses the same branch strategy. 19 | 20 | CAUTION: Changes to the docs are automatically published to `docs.couchbase.com`, including docs in the `main` branch. 21 | If you're writing docs for an upcoming minor version, you might want to do that in a temporary branch instead of `main`, and/or only commit the changes after the minor version is released. 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/FirstCallTracker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util; 18 | 19 | import com.couchbase.client.core.annotation.Stability; 20 | 21 | import java.util.concurrent.atomic.AtomicBoolean; 22 | 23 | @Stability.Internal 24 | public final class FirstCallTracker { 25 | private final AtomicBoolean alreadyCalled = new AtomicBoolean(false); 26 | 27 | /** 28 | * Return false the first time this method is called, and true on all subsequent calls. 29 | */ 30 | public boolean alreadyCalled() { 31 | return alreadyCalled.getAndSet(true); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/annotation/EnvironmentVariable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config.annotation; 18 | 19 | import java.lang.annotation.ElementType; 20 | import java.lang.annotation.Retention; 21 | import java.lang.annotation.RetentionPolicy; 22 | import java.lang.annotation.Target; 23 | 24 | /** 25 | * Indicates the config key's value should be obtained from 26 | * the named environment variable (if the variable is set). 27 | */ 28 | @Retention(RetentionPolicy.RUNTIME) 29 | @Target(ElementType.METHOD) 30 | public @interface EnvironmentVariable { 31 | String value(); 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/annotation/DisplayName.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config.annotation; 18 | 19 | import java.lang.annotation.ElementType; 20 | import java.lang.annotation.Retention; 21 | import java.lang.annotation.RetentionPolicy; 22 | import java.lang.annotation.Target; 23 | 24 | /** 25 | * Specifies the display name of the config key, overriding 26 | * the default inference strategy (which is to capitalize 27 | * the method name and insert a space between capital letters). 28 | */ 29 | @Retention(RetentionPolicy.RUNTIME) 30 | @Target(ElementType.METHOD) 31 | public @interface DisplayName { 32 | String value(); 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/DataSize.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config; 18 | 19 | public class DataSize { 20 | private final long byteCount; 21 | 22 | private DataSize(long byteCount) { 23 | this.byteCount = byteCount; 24 | } 25 | 26 | public static DataSize ofBytes(long bytes) { 27 | return new DataSize(bytes); 28 | } 29 | 30 | public long getByteCount() { 31 | return byteCount; 32 | } 33 | 34 | public int getByteCountAsSaturatedInt() { 35 | return (int) Math.min(Integer.MAX_VALUE, byteCount); 36 | } 37 | 38 | @Override 39 | public String toString() { 40 | return byteCount + " bytes"; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/filter/Filter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.filter; 18 | 19 | 20 | import com.couchbase.connect.kafka.handler.source.DocumentEvent; 21 | 22 | import java.util.Map; 23 | 24 | /** 25 | * Determines which Couchbase document changes get published to Kafka. 26 | */ 27 | public interface Filter { 28 | /** 29 | * Called when the filter is instantiated. 30 | * 31 | * @param configProperties the connector configuration. 32 | */ 33 | default void init(Map configProperties) { 34 | } 35 | 36 | /** 37 | * Returns true if the event should be published to Kafka, otherwise false. 38 | */ 39 | boolean pass(DocumentEvent event); 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/handler/source/MutationMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.handler.source; 18 | 19 | import com.couchbase.client.dcp.highlevel.Mutation; 20 | 21 | import static java.util.Objects.requireNonNull; 22 | 23 | /** 24 | * Extra metadata associated with mutation events. 25 | */ 26 | public class MutationMetadata { 27 | private final Mutation mutation; 28 | 29 | public MutationMetadata(Mutation mutation) { 30 | this.mutation = requireNonNull(mutation); 31 | } 32 | 33 | public int expiry() { 34 | return mutation.getExpiry(); 35 | } 36 | 37 | public int lockTime() { 38 | return mutation.getLockTime(); 39 | } 40 | 41 | public int flags() { 42 | return mutation.getFlagsAsInt(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/handler/sink/UpsertSinkHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.handler.sink; 18 | 19 | /** 20 | * Upserts each incoming Kafka record to Couchbase as a JSON document 21 | * using the Key/Value (Data) service. 22 | */ 23 | public class UpsertSinkHandler implements SinkHandler { 24 | @Override 25 | public SinkAction handle(SinkHandlerParams params) { 26 | String documentId = getDocumentId(params); 27 | SinkDocument doc = params.document().orElse(null); 28 | return doc == null 29 | ? SinkAction.remove(params, params.collection(), documentId) 30 | : SinkAction.upsertJson(params, params.collection(), documentId, doc.content()); 31 | } 32 | 33 | @Override 34 | public String toString() { 35 | return "UpsertSinkHandler{}"; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/Version.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util; 18 | 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | import java.util.Properties; 23 | 24 | public class Version { 25 | private static final Logger LOGGER = LoggerFactory.getLogger(Version.class); 26 | private static String version = "unknown"; 27 | 28 | static { 29 | try { 30 | Properties props = new Properties(); 31 | props.load(Version.class.getResourceAsStream("/kafka-connect-couchbase-version.properties")); 32 | version = props.getProperty("version", version).trim(); 33 | } catch (Exception e) { 34 | LOGGER.warn("Error while loading version:", e); 35 | } 36 | } 37 | 38 | public static String getVersion() { 39 | return version; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /.github/workflows/deploy-snapshot.yml: -------------------------------------------------------------------------------- 1 | name: Maven Deploy Snapshot 2 | 3 | concurrency: 4 | group: ${{ github.workflow }}-${{ github.ref }} 5 | 6 | on: 7 | push: 8 | branches: 9 | - master 10 | paths-ignore: 11 | - '*.md' 12 | - '*.adoc' 13 | - '.gitignore' 14 | - '.editorconfig' 15 | - 'examples/**' 16 | - '.github/workflows/**' 17 | - '!.github/workflows/deploy-snapshot.yml' 18 | 19 | jobs: 20 | build: 21 | runs-on: ubuntu-latest 22 | permissions: 23 | contents: read 24 | 25 | steps: 26 | - uses: actions/checkout@v4 27 | - uses: actions/setup-java@v4 28 | with: 29 | java-version: '21' 30 | distribution: 'temurin' 31 | 32 | server-id: 'central' 33 | server-username: MAVEN_USERNAME 34 | server-password: MAVEN_PASSWORD 35 | 36 | cache: 'maven' 37 | 38 | - name: Printing maven version 39 | run: ./mvnw --version 40 | 41 | - name: Build and deploy to Maven Central 42 | run: ./mvnw deploy --batch-mode -Dgpg.signer=bc -Psnapshot 43 | env: 44 | MAVEN_USERNAME: ${{ vars.MAVEN_USERNAME }} 45 | MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }} 46 | MAVEN_GPG_KEY: ${{ secrets.SDK_ROBOT_GPG_PRIVATE_KEY }} 47 | MAVEN_GPG_PASSPHRASE: '' 48 | 49 | - name: Remove artifacts from Maven repo so they're not cached 50 | run: rm -rfv ~/.m2/repository/com/couchbase/client/ 51 | -------------------------------------------------------------------------------- /examples/json-producer/README.md: -------------------------------------------------------------------------------- 1 | # JSON Producer Example 2 | 3 | This example code publishes randomly generated airport weather reports to a 4 | Kafka topic in JSON format (without schema). Make sure any sink connectors 5 | subscribed to the topic are configured with the following properties: 6 | 7 | key.converter=org.apache.kafka.connect.storage.StringConverter 8 | value.converter=org.apache.kafka.connect.json.JsonConverter 9 | value.converter.schemas.enable=false 10 | 11 | These properties may be applied as global defaults in the connect worker config, 12 | and may be overridden on a per-connector basis in the connect plugin config. 13 | 14 | 15 | ## Configurability (Lack Thereof) 16 | 17 | This example assumes there is a Kafka broker running on localhost, listening 18 | on the default port. The topic name is hard-coded. Feel free to edit the 19 | source code if you want to change any of these settings. 20 | 21 | 22 | ## Running the Example 23 | 24 | From this directory, run the producer with the Maven command: 25 | 26 | mvn compile exec:java 27 | 28 | The producer will publish some random messages, then terminate. 29 | 30 | 31 | ## What About Avro? 32 | 33 | Kafka Connect Couchbase works with Avro, too! Just configure the 34 | `key.converter` and `value.converter` properties to match 35 | the format of the Kafka messages. The Confluent team have some 36 | [very nice examples](https://github.com/confluentinc/examples) 37 | showing how to produce Kafka messages in Avro format. 38 | -------------------------------------------------------------------------------- /README.adoc: -------------------------------------------------------------------------------- 1 | https://docs.couchbase.com/kafka-connector/current/release-notes.html[*Download*] 2 | | https://docs.couchbase.com/kafka-connector/current/index.html[*Documentation*] 3 | | https://issues.couchbase.com/projects/KAFKAC[*Issues*] 4 | | https://forums.couchbase.com/c/Kafka-Connector[*Discussion*] 5 | 6 | = Kafka Connect Couchbase 7 | 8 | [abstract] 9 | A plug-in for the https://kafka.apache.org/documentation.html#connect[Kafka Connect framework]. 10 | 11 | == What does it do? 12 | 13 | The plugin includes a "source connector" for publishing document change notifications from Couchbase to a Kafka topic, as well as a "sink connector" that subscribes to one or more Kafka topics and writes the messages to Couchbase. 14 | 15 | See the documentation linked above for more details and a quickstart guide. 16 | 17 | == Customizing the message format 18 | 19 | The example project in `examples/custom-extensions` shows how to extend the source connector and customize the format of messages published to Kafka. 20 | 21 | == Building the connector from source 22 | 23 | Pre-built distributions are available from the download link above. 24 | The following instructions are intended for developers working on the connector. 25 | 26 | JDK 9 or later is required when building the connector. 27 | 28 | . Clone this GitHub repository. 29 | . Run `./mvnw package` in the project's root directory to generate the connector archive. 30 | . Look for `couchbase-kafka-connect-couchbase-.zip` under the `target` directory. 31 | -------------------------------------------------------------------------------- /src/test/java/com/couchbase/connect/kafka/util/ScopeAndCollectionTest.java: -------------------------------------------------------------------------------- 1 | package com.couchbase.connect.kafka.util; 2 | 3 | 4 | import org.junit.jupiter.api.Test; 5 | 6 | import static org.junit.jupiter.api.Assertions.assertEquals; 7 | import static org.junit.jupiter.api.Assertions.assertThrows; 8 | 9 | public class ScopeAndCollectionTest { 10 | 11 | public String scope = "Ascope"; 12 | public String collection = "Acollection"; 13 | 14 | @Test 15 | public void parse() { 16 | ScopeAndCollection scopeAndCollection = ScopeAndCollection.parse(createFQCN(scope, collection)); 17 | 18 | assertEquals(scope, scopeAndCollection.getScope()); 19 | assertEquals(collection, scopeAndCollection.getCollection()); 20 | } 21 | 22 | @Test 23 | public void parseMissingDelim() { 24 | assertThrows(IllegalArgumentException.class, () -> ScopeAndCollection.parse(scope + collection)); 25 | } 26 | 27 | @Test 28 | public void parseExtraDelim() { 29 | assertThrows(IllegalArgumentException.class, () -> ScopeAndCollection.parse(scope + ".." + collection)); 30 | } 31 | 32 | @Test 33 | public void checkEqual() { 34 | ScopeAndCollection scopeAndCollection1 = ScopeAndCollection.parse(createFQCN(scope, collection)); 35 | ScopeAndCollection scopeAndCollection2 = new ScopeAndCollection(scope, collection); 36 | assertEquals(scopeAndCollection1, scopeAndCollection2); 37 | } 38 | 39 | public String createFQCN(String scope, String collection) { 40 | return scope + "." + collection; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/BooleanParentRecommender.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config; 18 | 19 | import org.apache.kafka.common.config.ConfigDef; 20 | 21 | import java.util.LinkedList; 22 | import java.util.List; 23 | import java.util.Map; 24 | 25 | public class BooleanParentRecommender implements ConfigDef.Recommender { 26 | protected String parentConfigName; 27 | 28 | public BooleanParentRecommender(String parentConfigName) { 29 | this.parentConfigName = parentConfigName; 30 | } 31 | 32 | @Override 33 | public List validValues(String name, Map connectorConfigs) { 34 | return new LinkedList<>(); 35 | } 36 | 37 | @Override 38 | public boolean visible(String name, Map connectorConfigs) { 39 | return (Boolean) connectorConfigs.get(parentConfigName); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /examples/custom-extensions/README.md: -------------------------------------------------------------------------------- 1 | # Custom Extensions Example 2 | 3 | This Maven project is a template you can use for developing your own 4 | custom extension classes for Kafka Connect Couchbase. 5 | 6 | You can customize the behavior of the Couchbase Source Connector 7 | by providing your own `SourceHandler` implementation. A custom `SourceHandler` 8 | can skip certain messages, route messages to different topics, or 9 | change the content and format of the messages. 10 | 11 | The Sink Connector can modify incoming documents by applying 12 | [Single Message Transforms](https://kafka.apache.org/documentation/#connect_transforms). 13 | This project includes an example of a custom transform you can modify to suit your needs 14 | in case Kafka's built-in transforms are insufficient. 15 | 16 | It's also possible to completely customize the behavior of the Sink Connector by providing your own `SinkHandler`. 17 | 18 | ## Build and Install the Example Extensions 19 | 20 | Run `mvn package` to build the custom extensions JAR. Copy the JAR from 21 | the `target` directory into the same directory as `kafka-connect-couchabse-.jar`. 22 | (In the Quickstart guide this directory is referred to as `$KAFKA_CONNECT_COUCHBASE_HOME`.) 23 | 24 | NOTE: Don't forget to copy the JAR every time you change the Java code. 25 | 26 | The source code of each extension includes Javadoc with instructions 27 | for configuring the connector to use the extension. 28 | 29 | If you get stuck, help is available on the 30 | [Couchbase Forum](https://forums.couchbase.com/c/Kafka-Connector). 31 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/transform/DropIfNullValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.transform; 18 | 19 | import org.apache.kafka.common.config.ConfigDef; 20 | import org.apache.kafka.connect.connector.ConnectRecord; 21 | import org.apache.kafka.connect.transforms.Transformation; 22 | 23 | import java.util.Map; 24 | 25 | public class DropIfNullValue> implements Transformation { 26 | 27 | public static final String OVERVIEW_DOC = 28 | "Propagate a record only if its value is non-null."; 29 | 30 | @Override 31 | public R apply(R record) { 32 | return record.value() == null ? null : record; 33 | } 34 | 35 | @Override 36 | public ConfigDef config() { 37 | return new ConfigDef(); 38 | } 39 | 40 | @Override 41 | public void close() { 42 | } 43 | 44 | @Override 45 | public void configure(Map configs) { 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/handler/source/CollectionMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.handler.source; 18 | 19 | import com.couchbase.client.dcp.highlevel.DocumentChange; 20 | 21 | import static java.util.Objects.requireNonNull; 22 | 23 | public class CollectionMetadata { 24 | private final DocumentChange documentChange; 25 | 26 | public CollectionMetadata(DocumentChange documentChange) { 27 | this.documentChange = requireNonNull(documentChange); 28 | } 29 | 30 | public String scopeName() { 31 | return documentChange.getCollection().scope().name(); 32 | } 33 | 34 | public long scopeId() { 35 | return documentChange.getCollection().scope().id(); 36 | } 37 | 38 | public String collectionName() { 39 | return documentChange.getCollection().name(); 40 | } 41 | 42 | public long collectionId() { 43 | return documentChange.getCollection().id(); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/annotation/Width.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config.annotation; 18 | 19 | import org.apache.kafka.common.config.ConfigDef; 20 | 21 | import java.lang.annotation.ElementType; 22 | import java.lang.annotation.Retention; 23 | import java.lang.annotation.RetentionPolicy; 24 | import java.lang.annotation.Target; 25 | 26 | /** 27 | * Specifies the width of the config key, overriding the 28 | * default of {@link ConfigDef.Width#NONE}. 29 | *

30 | * When applied to an interface, sets the width of all 31 | * methods defined in the interface. 32 | *

33 | * If this annotation is present on an interface as well as on a 34 | * method in the interface, the method annotation takes precedence. 35 | */ 36 | @Retention(RetentionPolicy.RUNTIME) 37 | @Target({ElementType.METHOD, ElementType.TYPE}) 38 | public @interface Width { 39 | ConfigDef.Width value(); 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/DataSizeValidator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config; 18 | 19 | import org.apache.kafka.common.config.ConfigDef; 20 | import org.apache.kafka.common.config.ConfigException; 21 | 22 | public class DataSizeValidator implements ConfigDef.Validator { 23 | @Override 24 | public void ensureValid(String name, Object value) { 25 | try { 26 | if (value != null && !((String) value).isEmpty()) { 27 | DataSizeParser.parseDataSize((String) value); 28 | } 29 | } catch (IllegalArgumentException e) { 30 | throw new ConfigException("Failed to parse config property '" + name + "' -- " + e.getMessage()); 31 | } 32 | } 33 | 34 | public String toString() { 35 | return "An integer followed by a size unit (b = bytes, k = kilobytes, m = megabytes, g = gigabytes)." + 36 | " For example, to specify 64 megabytes: 64m"; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/Contextual.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config; 18 | 19 | import java.util.Map; 20 | 21 | /** 22 | * Holds a fallback value and zero or more context-specific values. 23 | *

24 | * Specified in the connector config like this: 25 | * 

26 |  * some.property=fallback-value
27 |  * some.property[context-a]=value-a
28 |  * some.property[context-b]=value-b
29 |  *
30 | * 31 | * @param Type of the held values. Must be one of the other supported config types; 32 | * see {@link KafkaConfigProxyFactory}. 33 | * @see com.couchbase.connect.kafka.util.config.annotation.ContextDocumentation 34 | */ 35 | public final class Contextual extends LookupTable { 36 | public Contextual( 37 | String propertyName, 38 | T fallback, 39 | Map contextToValue 40 | ) { 41 | super(propertyName, fallback, contextToValue); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/annotation/Group.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config.annotation; 18 | 19 | import java.lang.annotation.ElementType; 20 | import java.lang.annotation.Retention; 21 | import java.lang.annotation.RetentionPolicy; 22 | import java.lang.annotation.Target; 23 | 24 | /** 25 | * Specifies the name of the group the config key belongs to, 26 | * overriding the default inference strategy (which is to 27 | * derive the group name from the name of the method's class). 28 | *

29 | * When applied to an interface, sets the group name for all 30 | * methods defined in the interface. 31 | *

32 | * If this annotation is present on an interface as well as on a 33 | * method in the interface, the method annotation takes precedence. 34 | */ 35 | @Retention(RetentionPolicy.RUNTIME) 36 | @Target({ElementType.METHOD, ElementType.TYPE}) 37 | public @interface Group { 38 | String value(); 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/annotation/Importance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config.annotation; 18 | 19 | import org.apache.kafka.common.config.ConfigDef; 20 | 21 | import java.lang.annotation.ElementType; 22 | import java.lang.annotation.Retention; 23 | import java.lang.annotation.RetentionPolicy; 24 | import java.lang.annotation.Target; 25 | 26 | /** 27 | * Specifies the importance of the config key, overriding the 28 | * default of {@link ConfigDef.Importance#MEDIUM}. 29 | *

30 | * When applied to an interface, sets the importance of all 31 | * methods defined in the interface. 32 | *

33 | * If this annotation is present on an interface as well as on a 34 | * method in the interface, the method annotation takes precedence. 35 | */ 36 | @Retention(RetentionPolicy.RUNTIME) 37 | @Target({ElementType.METHOD, ElementType.TYPE}) 38 | public @interface Importance { 39 | ConfigDef.Importance value(); 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/handler/sink/SinkHandlerContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.handler.sink; 18 | 19 | import com.couchbase.client.java.ReactiveCluster; 20 | 21 | import java.util.Map; 22 | 23 | import static java.util.Collections.unmodifiableMap; 24 | import static java.util.Objects.requireNonNull; 25 | 26 | public class SinkHandlerContext { 27 | private final ReactiveCluster cluster; 28 | private final Map configProperties; 29 | 30 | public SinkHandlerContext(ReactiveCluster cluster, Map configProperties) { 31 | this.cluster = requireNonNull(cluster); 32 | this.configProperties = requireNonNull(unmodifiableMap(configProperties)); 33 | } 34 | 35 | public ReactiveCluster cluster() { 36 | return cluster; 37 | } 38 | 39 | /** 40 | * Returns the connector configuration properties as an unmodifiable map. 41 | */ 42 | public Map configProperties() { 43 | return configProperties; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/EnumRecommender.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config; 18 | 19 | import org.apache.kafka.common.config.ConfigDef; 20 | 21 | import java.util.ArrayList; 22 | import java.util.Collections; 23 | import java.util.List; 24 | import java.util.Map; 25 | 26 | public class EnumRecommender implements ConfigDef.Recommender { 27 | private final List validValues; 28 | 29 | public EnumRecommender(Class streamFromClass) { 30 | List names = new ArrayList<>(); 31 | for (Enum value : streamFromClass.getEnumConstants()) { 32 | names.add(value.name()); 33 | } 34 | this.validValues = Collections.unmodifiableList(names); 35 | } 36 | 37 | @Override 38 | public List validValues(String name, Map parsedConfig) { 39 | return validValues; 40 | } 41 | 42 | @Override 43 | public boolean visible(String name, Map parsedConfig) { 44 | return true; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/config/source/CouchbaseSourceTaskConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.config.source; 18 | 19 | import com.couchbase.client.dcp.util.PartitionSet; 20 | import com.couchbase.connect.kafka.util.config.annotation.Default; 21 | 22 | public interface CouchbaseSourceTaskConfig extends CouchbaseSourceConfig { 23 | /** 24 | * Set of partitions for this task to watch for changes. 25 | *

26 | * Parse using {@link PartitionSet#parse(String)}. 27 | */ 28 | String partitions(); 29 | 30 | /** 31 | * The task ID... probably. Kafka 2.3.0 and later expose the task ID 32 | * in the logging context, but for earlier versions we have to assume 33 | * the task IDs are assigned in the same order as the configs returned 34 | * by Connector.taskConfigs(int). 35 | */ 36 | String maybeTaskId(); 37 | 38 | /** 39 | * The unique identifier of the Couchbase Server cluster, or empty string if unknown. 40 | */ 41 | @Default 42 | String clusterUuid(); 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/DurationValidator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config; 18 | 19 | import org.apache.kafka.common.config.ConfigDef; 20 | import org.apache.kafka.common.config.ConfigException; 21 | 22 | import static java.util.concurrent.TimeUnit.MILLISECONDS; 23 | 24 | public class DurationValidator implements ConfigDef.Validator { 25 | @Override 26 | public void ensureValid(String name, Object value) { 27 | try { 28 | if (value != null && !((String) value).isEmpty()) { 29 | DurationParser.parseDuration((String) value, MILLISECONDS); 30 | } 31 | } catch (IllegalArgumentException e) { 32 | throw new ConfigException("Failed to parse config property '" + name + "' -- " + e.getMessage()); 33 | } 34 | } 35 | 36 | public String toString() { 37 | return "An integer followed by a time unit (ms = milliseconds, s = seconds, m = minutes, h = hours, d = days)." + 38 | " For example, to specify 30 minutes: 30m"; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/com/couchbase/connect/kafka/util/Resources.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.couchbase.connect.kafka.util; 17 | 18 | import java.io.InputStream; 19 | 20 | /** 21 | * Helper class for various resource handling mechanisms. 22 | * 23 | * @author Michael Nitschinger 24 | * @since 1.0 25 | */ 26 | public class Resources { 27 | 28 | /** 29 | * Reads a file from the resources folder (in the same path as the requesting test class). 30 | *

31 | * The class will be automatically loaded relative to the namespace and converted to a string. 32 | * 33 | * @param filename the filename of the resource. 34 | * @param clazz the reference class. 35 | * @return the loaded string. 36 | */ 37 | public static String read(final String filename, final Class clazz) { 38 | String path = "/" + clazz.getPackage().getName().replace(".", "/") + "/" + filename; 39 | InputStream stream = clazz.getResourceAsStream(path); 40 | java.util.Scanner s = new java.util.Scanner(stream).useDelimiter("\\A"); 41 | return s.hasNext() ? s.next() : ""; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/EnumValidator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config; 18 | 19 | import org.apache.kafka.common.config.ConfigDef; 20 | import org.apache.kafka.common.config.ConfigException; 21 | 22 | import java.util.Arrays; 23 | 24 | public class EnumValidator implements ConfigDef.Validator { 25 | private final Class enumClass; 26 | 27 | public EnumValidator(Class enumClass) { 28 | this.enumClass = enumClass; 29 | } 30 | 31 | @Override 32 | public void ensureValid(String name, Object value) { 33 | try { 34 | //noinspection unchecked 35 | Enum.valueOf(enumClass, (String) value); 36 | } catch (IllegalArgumentException e) { 37 | throw new ConfigException("Invalid value '" + value + "' for config key '" + name + "'" + 38 | "; must be one of " + Arrays.toString(enumClass.getEnumConstants())); 39 | } 40 | } 41 | 42 | @Override 43 | public String toString() { 44 | return "One of " + Arrays.toString(enumClass.getEnumConstants()); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/annotation/ContextDocumentation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config.annotation; 18 | 19 | import java.lang.annotation.ElementType; 20 | import java.lang.annotation.Retention; 21 | import java.lang.annotation.RetentionPolicy; 22 | import java.lang.annotation.Target; 23 | 24 | /** 25 | * When applied to a config method that returns {@link com.couchbase.connect.kafka.util.config.Contextual}, 26 | * describes the meaning of the context string. 27 | */ 28 | @Retention(RetentionPolicy.RUNTIME) 29 | @Target(ElementType.METHOD) 30 | public @interface ContextDocumentation { 31 | /** 32 | * Describes the context's meaning. Should start with an article, if appropriate. 33 | * Example: "the source topic". 34 | */ 35 | String contextDescription(); 36 | 37 | /** 38 | * A sample context to include in generated documentation. 39 | */ 40 | String sampleContext(); 41 | 42 | /** 43 | * A sample property value to include in generated documentation. 44 | */ 45 | String sampleValue(); 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/handler/source/SourceHandlerParams.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.handler.source; 18 | 19 | /** 20 | * Parameter block for document event handling. 21 | */ 22 | public class SourceHandlerParams { 23 | private final DocumentEvent documentEvent; 24 | private final String topic; 25 | private final boolean noValue; 26 | 27 | public SourceHandlerParams(DocumentEvent documentEvent, String topic, boolean noValue) { 28 | this.documentEvent = documentEvent; 29 | this.topic = topic; 30 | this.noValue = noValue; 31 | } 32 | 33 | /** 34 | * Returns the event to be converted to a {@link SourceRecordBuilder}. 35 | */ 36 | public DocumentEvent documentEvent() { 37 | return documentEvent; 38 | } 39 | 40 | /** 41 | * Returns the Kafka topic name from the connector configuration. 42 | */ 43 | public String topic() { 44 | return topic; 45 | } 46 | 47 | /** 48 | * Returns true if the connector was configured to omit document values. 49 | */ 50 | public boolean noValue() { 51 | return noValue; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/filter/AllPassFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.filter; 18 | 19 | import com.couchbase.connect.kafka.handler.source.DocumentEvent; 20 | 21 | /** 22 | * Allows publication of any event, except for transaction metadata 23 | * and events in Couchbase system scopes. 24 | *

25 | * Transaction metadata is any document whose key starts with "_txn:". 26 | *

27 | * A system scope is any scope whose name start with percent or underscore, 28 | * except for the default scope (whose name is "_default"). 29 | * 30 | * @see AllPassIncludingSystemFilter 31 | */ 32 | public class AllPassFilter implements Filter { 33 | 34 | @Override 35 | public boolean pass(final DocumentEvent event) { 36 | return !isSystemScope(event) && !isTransactionMetadata(event); 37 | } 38 | 39 | private static boolean isSystemScope(DocumentEvent event) { 40 | String scopeName = event.collectionMetadata().scopeName(); 41 | return scopeName.startsWith("%") || (scopeName.startsWith("_") && !scopeName.equals("_default")); 42 | } 43 | 44 | private static boolean isTransactionMetadata(DocumentEvent event) { 45 | return event.key().startsWith("_txn:"); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/ConnectHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util; 18 | 19 | import org.slf4j.MDC; 20 | 21 | import java.util.Optional; 22 | import java.util.regex.Matcher; 23 | import java.util.regex.Pattern; 24 | 25 | public class ConnectHelper { 26 | private ConnectHelper() { 27 | throw new AssertionError("not instantiable"); 28 | } 29 | 30 | private static final Pattern taskIdPattern = Pattern.compile("\\|task-(\\d+)"); 31 | 32 | public static Optional getConnectorContextFromLoggingContext() { 33 | // This should be present in Kafka 2.3.0 and later. 34 | // See https://issues.apache.org/jira/browse/KAFKA-3816 35 | return Optional.ofNullable(MDC.get("connector.context")).map(String::trim); 36 | } 37 | 38 | /** 39 | * Returns the connector's task ID, or an empty optional if it could not 40 | * be determined from the logging context. 41 | */ 42 | public static Optional getTaskIdFromLoggingContext() { 43 | return getConnectorContextFromLoggingContext() 44 | .map(context -> { 45 | Matcher m = taskIdPattern.matcher(context); 46 | return m.find() ? m.group(1) : null; 47 | }); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /examples/json-producer/pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 4.0.0 5 | 6 | jar 7 | 8 | com.couchbase.connect.kafka.example 9 | json-producer-example 10 | 1.0-SNAPSHOT 11 | 12 | 13 | UTF-8 14 | 1.8 15 | 1.8 16 | 17 | 18 | 19 | 20 | org.slf4j 21 | slf4j-simple 22 | 1.7.36 23 | 24 | 25 | org.apache.kafka 26 | kafka-clients 27 | 2.8.2 28 | 29 | 30 | com.fasterxml.jackson.core 31 | jackson-databind 32 | 2.16.1 33 | 34 | 35 | 36 | 37 | 38 | 39 | org.codehaus.mojo 40 | exec-maven-plugin 41 | 1.2.1 42 | 43 | com.couchbase.connect.kafka.example.JsonProducerExample 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/handler/sink/SinkDocument.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.handler.sink; 18 | 19 | import org.jspecify.annotations.Nullable; 20 | 21 | import java.util.Optional; 22 | 23 | import static java.util.Objects.requireNonNull; 24 | 25 | /** 26 | * Holds the content of a document, and optionally a document ID 27 | * derived from fields of the document. 28 | */ 29 | public class SinkDocument { 30 | private final @Nullable String id; 31 | private final byte[] content; 32 | 33 | public SinkDocument(@Nullable String id, byte[] content) { 34 | this.id = id; 35 | this.content = requireNonNull(content); 36 | } 37 | 38 | /** 39 | * Returns the document ID extracted from the message body, 40 | * or an empty optional if ID extraction is disabled or failed 41 | * because the expected field(s) are missing. 42 | */ 43 | public Optional id() { 44 | return Optional.ofNullable(id); 45 | } 46 | 47 | /** 48 | * Returns the document content. If the connector is configured 49 | * to remove fields used to generate the document ID, those fields 50 | * will not be present in the returned content. 51 | */ 52 | public byte[] content() { 53 | return content; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/N1qlData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util; 18 | 19 | import com.couchbase.connect.kafka.handler.sink.ConcurrencyHint; 20 | 21 | import static java.util.Objects.requireNonNull; 22 | 23 | public class N1qlData { 24 | private final String keyspace; 25 | private final String data; 26 | private final OperationType type; 27 | private final ConcurrencyHint hint; 28 | 29 | public N1qlData(String keyspace, String data, OperationType type, ConcurrencyHint hint) { 30 | this.keyspace = requireNonNull(keyspace); 31 | this.data = requireNonNull(data); 32 | this.type = requireNonNull(type); 33 | this.hint = requireNonNull(hint); 34 | } 35 | 36 | public String getData() { 37 | return data; 38 | } 39 | 40 | public OperationType getType() { 41 | return type; 42 | } 43 | 44 | public ConcurrencyHint getHint() { 45 | return hint; 46 | } 47 | 48 | public String getKeyspace() { 49 | return keyspace; 50 | } 51 | 52 | @Override 53 | public String toString() { 54 | return "[ " + "keyspace = " + keyspace + " data = " + data + " type = " + type + " hint = " + hint + " ] "; 55 | } 56 | 57 | public enum OperationType { 58 | UPSERT, 59 | DELETE 60 | } 61 | 62 | 63 | } 64 | -------------------------------------------------------------------------------- /src/test/java/com/couchbase/connect/kafka/util/config/SizeParserTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config; 18 | 19 | 20 | import org.junit.jupiter.api.Test; 21 | 22 | import static org.junit.jupiter.api.Assertions.assertEquals; 23 | import static org.junit.jupiter.api.Assertions.assertThrows; 24 | 25 | public class SizeParserTest { 26 | @Test 27 | public void parseSize() { 28 | assertEquals(0, parseBytes("0")); 29 | assertEquals(0, parseBytes("0k")); 30 | assertEquals(3, parseBytes("3b")); 31 | assertEquals(3 * 1024, parseBytes("3k")); 32 | assertEquals(3 * 1024 * 1024, parseBytes("3m")); 33 | assertEquals(3L * 1024 * 1024 * 1024, parseBytes("3g")); 34 | 35 | assertEquals(0, parseBytes("0b")); 36 | assertEquals(0, parseBytes("0k")); 37 | assertEquals(0, parseBytes("0m")); 38 | assertEquals(0, parseBytes("0g")); 39 | } 40 | 41 | @Test 42 | public void missingNumber() { 43 | assertThrows(IllegalArgumentException.class, () -> parseBytes("k")); 44 | } 45 | 46 | @Test 47 | public void missingUnit() { 48 | assertThrows(IllegalArgumentException.class, () -> parseBytes("300")); 49 | } 50 | 51 | private static long parseBytes(String s) { 52 | return DataSizeParser.parseDataSize(s).getByteCount(); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/Schemas.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util; 18 | 19 | import org.apache.kafka.connect.data.Schema; 20 | import org.apache.kafka.connect.data.SchemaBuilder; 21 | 22 | public enum Schemas { 23 | ; 24 | 25 | public static final Schema KEY_SCHEMA = Schema.STRING_SCHEMA; 26 | public static final Schema VALUE_DEFAULT_SCHEMA = 27 | SchemaBuilder.struct().name("com.couchbase.DcpMessage") 28 | .field("event", Schema.STRING_SCHEMA) 29 | .field("partition", Schema.INT16_SCHEMA) // Couchbase "vBucket ID" 30 | .field("key", Schema.STRING_SCHEMA) 31 | .field("cas", Schema.INT64_SCHEMA) 32 | .field("bySeqno", Schema.INT64_SCHEMA) 33 | .field("revSeqno", Schema.INT64_SCHEMA) 34 | .field("expiration", Schema.OPTIONAL_INT32_SCHEMA) 35 | .field("flags", Schema.OPTIONAL_INT32_SCHEMA) 36 | .field("lockTime", Schema.OPTIONAL_INT32_SCHEMA) 37 | .field("content", Schema.OPTIONAL_BYTES_SCHEMA) 38 | 39 | // Added in 3.2.0. Marked as optional to support schema evolution. 40 | .field("bucket", SchemaBuilder.string().optional().build()) 41 | .field("vBucketUuid", SchemaBuilder.int64().optional().build()) 42 | 43 | .build(); 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/StreamFrom.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka; 18 | 19 | /** 20 | * The resume modes supported by the Kafka connector. 21 | */ 22 | public enum StreamFrom { 23 | SAVED_OFFSET_OR_BEGINNING, 24 | SAVED_OFFSET_OR_NOW, 25 | BEGINNING, 26 | NOW; 27 | 28 | /** 29 | * Returns the "fallback" mode for modes that prefer saved offsets, otherwise returns {@code this}. 30 | */ 31 | public StreamFrom withoutSavedOffset() { 32 | switch (this) { 33 | case SAVED_OFFSET_OR_BEGINNING: 34 | case BEGINNING: 35 | return BEGINNING; 36 | 37 | case SAVED_OFFSET_OR_NOW: 38 | case NOW: 39 | return NOW; 40 | 41 | default: 42 | throw new AssertionError(); 43 | } 44 | } 45 | 46 | /** 47 | * Returns true if this mode prefers to use saved offset. 48 | */ 49 | public boolean isSavedOffset() { 50 | return this != withoutSavedOffset(); 51 | } 52 | 53 | public com.couchbase.client.dcp.StreamFrom asDcpStreamFrom() { 54 | switch (this) { 55 | case BEGINNING: 56 | return com.couchbase.client.dcp.StreamFrom.BEGINNING; 57 | case NOW: 58 | return com.couchbase.client.dcp.StreamFrom.NOW; 59 | default: 60 | throw new IllegalStateException(this + " has no DCP counterpart"); 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/DocumentIdExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util; 18 | 19 | import com.couchbase.connect.kafka.handler.sink.SinkDocument; 20 | 21 | import java.io.IOException; 22 | 23 | /** 24 | * Locates a document ID using a JSON pointer, optionally removing the ID from the document. 25 | *

26 | * Immutable. 27 | */ 28 | public interface DocumentIdExtractor { 29 | 30 | /** 31 | * @param json The document content encoded as UTF-8. If this method returns normally, 32 | * it may modify the contents of the array to remove the fields used by the document ID. 33 | */ 34 | SinkDocument extractDocumentId(final byte[] json, boolean removeDocumentId) throws IOException, DocumentPathExtractor.DocumentPathNotFoundException; 35 | 36 | static DocumentIdExtractor from(String documentIdFormat) { 37 | if (documentIdFormat.isEmpty()) { 38 | return (json, removeDocumentPath) -> new SinkDocument(null, json); 39 | } 40 | 41 | DocumentPathExtractor pathExtractor = new DocumentPathExtractor(documentIdFormat); 42 | 43 | return (json, removeDocumentId) -> { 44 | DocumentPathExtractor.DocumentExtraction extraction = pathExtractor.extractDocumentPath(json, removeDocumentId); 45 | return new SinkDocument(extraction.getPathValue(), extraction.getData()); 46 | }; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /CONTRIBUTING.adoc: -------------------------------------------------------------------------------- 1 | = Contributing 2 | 3 | == Bug reports and feature requests 4 | 5 | If you come across a bug or find something unintuitive, let us know and we’ll work to fix it in an upcoming release. 6 | 7 | To file a bug report or feature request, go to the *Issues* tab of this GitHub repository. 8 | 9 | When filing a bug report, please include: 10 | 11 | * Couchbase Server version 12 | * Kafka version 13 | * Connector version 14 | * Operating system 15 | * A description of the problem 16 | * Detailed steps to reproduce the problem 17 | 18 | == Contributing code 19 | 20 | Whether you have a fix for a typo in a component, a bugfix, or a new feature, we love to receive code from the community! 21 | 22 | It takes a lot of work to get from a potential new bug fix or feature idea to well-tested shipping code. 23 | Our engineers want to help you get there. 24 | 25 | === One-time setup 26 | The first step is to sign our Contributor License Agreement. 27 | 28 | NOTE: Contributor License Agreements (CLAs) are common for projects under the Apache license, and typically serve to grant control of the code to a central entity. 29 | Because our code is available under the Apache License, signing the CLA doesn’t prevent you from using your code however you like, but it does give Couchbase the ability to defend the source legally and build and maintain a business around the technology. 30 | 31 | Create an account on our https://review.couchbase.org/[Gerrit code review site]. 32 | Make sure the email address you register with matches the email address on your git commits. 33 | You can associate additional email addresses with your Gerrit account later if needed. 34 | 35 | Fill out the agreement under **Settings > Agreements**. 36 | 37 | === Submitting code 38 | 39 | We encourage you to submit patch sets directly to the Gerrit server. 40 | That makes things easier for us, but if you're new to Gerrit it might be intimidating. 41 | Alternatively, free to submit a pull request on GitHub. 42 | A robot will automatically import your change into Gerrit. 43 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/config/common/LoggingConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.config.common; 18 | 19 | import com.couchbase.client.core.annotation.Stability; 20 | import com.couchbase.client.core.logging.RedactionLevel; 21 | import com.couchbase.connect.kafka.util.config.annotation.Default; 22 | 23 | import java.time.Duration; 24 | 25 | public interface LoggingConfig { 26 | /** 27 | * Determines which kinds of sensitive log messages from the Couchbase connector 28 | * will be tagged for later redaction by the Couchbase log redaction tool. 29 | * NONE = no tagging; PARTIAL = user data is tagged; FULL = user, meta, and system data is tagged. 30 | */ 31 | @Default("NONE") 32 | RedactionLevel logRedaction(); 33 | 34 | /** 35 | * If true, document lifecycle milestones will be logged at INFO level 36 | * instead of DEBUG. Enabling this feature lets you watch documents 37 | * flow through the connector. Disabled by default because it generates 38 | * many log messages. 39 | */ 40 | @Default("false") 41 | boolean logDocumentLifecycle(); 42 | 43 | /** 44 | * The connector writes metrics to the log at this interval. 45 | *

46 | * Disable metric logging by setting this to `0`. 47 | * 48 | * @since 4.2.3 49 | */ 50 | @Stability.Uncommitted 51 | @Default("10m") 52 | Duration metricsInterval(); 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/AbstractInvocationHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config; 18 | 19 | import java.lang.reflect.InvocationHandler; 20 | import java.lang.reflect.Method; 21 | 22 | abstract class AbstractInvocationHandler implements InvocationHandler { 23 | private static final Object[] EMPTY_OBJECT_ARRAY = new Object[0]; 24 | 25 | private final String toStringPrefix; 26 | 27 | public AbstractInvocationHandler(String toStringPrefix) { 28 | this.toStringPrefix = toStringPrefix; 29 | } 30 | 31 | @Override 32 | public Object invoke(Object proxy, Method method, Object[] argsMaybeNull) throws Throwable { 33 | Object[] args = argsMaybeNull == null ? EMPTY_OBJECT_ARRAY : argsMaybeNull; 34 | 35 | if ("equals".equals(method.getName()) 36 | && args.length == 1 37 | && method.getParameterTypes()[0].equals(Object.class)) { 38 | return proxy == args[0]; 39 | } 40 | 41 | if ("hashCode".equals(method.getName()) && args.length == 0) { 42 | return System.identityHashCode(proxy); 43 | } 44 | 45 | if ("toString".equals(method.getName()) && args.length == 0) { 46 | return toStringPrefix + "@" + Integer.toHexString(proxy.hashCode()); 47 | } 48 | 49 | return doInvoke(proxy, method, args); 50 | } 51 | 52 | protected abstract Object doInvoke(Object proxy, Method method, Object[] args); 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/ListHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | import static java.util.Objects.requireNonNull; 23 | 24 | public class ListHelper { 25 | private ListHelper() { 26 | throw new AssertionError("not instantiable"); 27 | } 28 | 29 | /** 30 | * Splits the given list into the requested number of chunks. 31 | * The smallest and largest chunks are guaranteed to differ in size by no more than 1. 32 | * If the requested number of chunks is greater than the number of items, 33 | * some chunks will be empty. 34 | */ 35 | public static List> chunks(List items, int chunks) { 36 | if (chunks <= 0) { 37 | throw new IllegalArgumentException("chunks must be > 0"); 38 | } 39 | requireNonNull(items); 40 | 41 | final int maxChunkSize = ((items.size() - 1) / chunks) + 1; // size / chunks, rounded up 42 | final int numFullChunks = chunks - (maxChunkSize * chunks - items.size()); 43 | 44 | final List> result = new ArrayList<>(chunks); 45 | 46 | int startIndex = 0; 47 | for (int i = 0; i < chunks; i++) { 48 | int endIndex = startIndex + maxChunkSize; 49 | if (i >= numFullChunks) { 50 | endIndex--; 51 | } 52 | result.add(items.subList(startIndex, endIndex)); 53 | startIndex = endIndex; 54 | } 55 | return result; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/test/java/com/couchbase/connect/kafka/handler/source/RawJsonSourceHandlerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.handler.source; 18 | 19 | 20 | import org.junit.jupiter.api.Test; 21 | 22 | import static com.couchbase.connect.kafka.util.JsonHelper.isValidJson; 23 | import static java.nio.charset.StandardCharsets.UTF_8; 24 | import static org.junit.jupiter.api.Assertions.assertFalse; 25 | import static org.junit.jupiter.api.Assertions.assertTrue; 26 | 27 | public class RawJsonSourceHandlerTest { 28 | 29 | @Test 30 | public void jsonValidation() { 31 | assertValid("true"); 32 | assertValid("0"); 33 | assertValid("1.0"); 34 | assertValid("null"); 35 | assertValid("\"foo\""); 36 | assertValid("{}"); 37 | assertValid("[]"); 38 | assertValid("{\"foo\":{}}"); 39 | assertValid("[[],[],[]]"); 40 | 41 | assertInvalid(""); 42 | assertInvalid("foo"); 43 | assertInvalid("{}{}"); 44 | assertInvalid("[][]"); 45 | assertInvalid("{}true"); 46 | assertInvalid("[]true"); 47 | assertInvalid("{}}"); 48 | assertInvalid("[]]"); 49 | assertInvalid("{\"foo\":true"); 50 | assertInvalid("[1,2,3"); 51 | } 52 | 53 | private static void assertValid(String input) { 54 | assertTrue(isValidJson(input.getBytes(UTF_8)), "should be valid: " + input); 55 | } 56 | 57 | private static void assertInvalid(String input) { 58 | assertFalse(isValidJson(input.getBytes(UTF_8)), "should be invalid: " + input); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/handler/source/CouchbaseSourceRecord.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.handler.source; 18 | 19 | import com.couchbase.client.dcp.highlevel.DocumentChange; 20 | import org.apache.kafka.connect.data.Schema; 21 | import org.apache.kafka.connect.header.Header; 22 | import org.apache.kafka.connect.source.SourceRecord; 23 | 24 | import java.util.Map; 25 | 26 | /** 27 | * A source record with some metadata tacked on for document lifecycle tracing. 28 | */ 29 | public class CouchbaseSourceRecord extends SourceRecord { 30 | private final String qualifiedKey; 31 | private final long tracingToken; 32 | 33 | public CouchbaseSourceRecord(DocumentChange change, 34 | Map sourcePartition, Map sourceOffset, 35 | String topic, Integer partition, 36 | Schema keySchema, Object key, 37 | Schema valueSchema, Object value, 38 | Long timestamp, Iterable

headers) { 39 | super(sourcePartition, sourceOffset, topic, partition, keySchema, key, valueSchema, value, timestamp, headers); 40 | this.qualifiedKey = change.getQualifiedKey(); 41 | this.tracingToken = change.getTracingToken(); 42 | } 43 | 44 | public String getCouchbaseDocumentId() { 45 | return qualifiedKey; 46 | } 47 | 48 | public long getTracingToken() { 49 | return tracingToken; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/CouchbaseSinkConnector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka; 18 | 19 | import com.couchbase.connect.kafka.config.sink.CouchbaseSinkConfig; 20 | import com.couchbase.connect.kafka.util.Version; 21 | import com.couchbase.connect.kafka.util.config.ConfigHelper; 22 | import org.apache.kafka.common.config.ConfigDef; 23 | import org.apache.kafka.connect.connector.Task; 24 | import org.apache.kafka.connect.sink.SinkConnector; 25 | 26 | import java.util.ArrayList; 27 | import java.util.List; 28 | import java.util.Map; 29 | 30 | public class CouchbaseSinkConnector extends SinkConnector { 31 | private Map configProperties; 32 | 33 | @Override 34 | public String version() { 35 | return Version.getVersion(); 36 | } 37 | 38 | @Override 39 | public void start(Map properties) { 40 | configProperties = properties; 41 | } 42 | 43 | @Override 44 | public void stop() { 45 | } 46 | 47 | @Override 48 | public ConfigDef config() { 49 | return ConfigHelper.define(CouchbaseSinkConfig.class); 50 | } 51 | 52 | @Override 53 | public Class taskClass() { 54 | return CouchbaseSinkTask.class; 55 | } 56 | 57 | @Override 58 | public List> taskConfigs(int maxTasks) { 59 | List> taskConfigs = new ArrayList<>(maxTasks); 60 | for (int i = 0; i < maxTasks; i++) { 61 | taskConfigs.add(configProperties); 62 | } 63 | return taskConfigs; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/CouchbaseHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util; 18 | 19 | import com.couchbase.client.core.Core; 20 | import com.couchbase.client.core.topology.ClusterTopologyWithBucket; 21 | import com.couchbase.client.core.topology.CouchbaseBucketTopology; 22 | import com.couchbase.client.java.Bucket; 23 | import reactor.core.publisher.Mono; 24 | 25 | import java.time.Duration; 26 | 27 | public class CouchbaseHelper { 28 | public static Mono getConfig(Core core, String bucketName) { 29 | return core 30 | .configurationProvider() 31 | .configs() 32 | .flatMap(clusterConfig -> 33 | Mono.justOrEmpty(clusterConfig.bucketTopology(bucketName))) 34 | .filter(CouchbaseHelper::hasPartitionInfo) 35 | .next(); 36 | } 37 | 38 | /** 39 | * Returns true unless the topology is from a newly-created bucket 40 | * whose partition count is not yet available. 41 | */ 42 | private static boolean hasPartitionInfo(ClusterTopologyWithBucket topology) { 43 | CouchbaseBucketTopology bucketTopology = (CouchbaseBucketTopology) topology.bucket(); 44 | return bucketTopology.numberOfPartitions() > 0; 45 | } 46 | 47 | 48 | public static Mono getConfig(Bucket bucket) { 49 | return getConfig(bucket.core(), bucket.name()); 50 | } 51 | 52 | public static ClusterTopologyWithBucket getConfig(Bucket bucket, Duration timeout) { 53 | return getConfig(bucket).block(timeout); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/com/couchbase/connect/kafka/util/config/HtmlRendererTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config; 18 | 19 | 20 | import org.junit.jupiter.api.Test; 21 | 22 | import static com.couchbase.connect.kafka.util.config.HtmlRenderer.PARAGRAPH_SEPARATOR; 23 | import static com.couchbase.connect.kafka.util.config.HtmlRenderer.htmlToPlaintext; 24 | import static org.junit.jupiter.api.Assertions.assertEquals; 25 | 26 | public class HtmlRendererTest { 27 | @Test 28 | public void paragraphsRenderedNicely() { 29 | String plain = htmlToPlaintext("hello

world"); 30 | assertEquals("hello" + PARAGRAPH_SEPARATOR + "world", plain); 31 | } 32 | 33 | @Test 34 | public void tagWhitespace() { 35 | assertEquals("helloworld", htmlToPlaintext("helloworld")); 36 | assertEquals("helloworldagain", htmlToPlaintext("helloworldagain")); 37 | assertEquals("hello world", htmlToPlaintext("hello world")); 38 | assertEquals("hello world", htmlToPlaintext("hello world")); 39 | assertEquals("hello world", htmlToPlaintext("hello world ")); 40 | assertEquals("hello world again", htmlToPlaintext("hello world again")); 41 | } 42 | 43 | @Test 44 | public void paragraphsBreaksAreMerged() { 45 | assertEquals( 46 | htmlToPlaintext("hello

world"), 47 | htmlToPlaintext("hello

world")); 48 | } 49 | 50 | @Test 51 | public void leadingAndTrailingParagraphsIgnored() { 52 | assertEquals("hello", htmlToPlaintext("

hello

")); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/DataSizeParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config; 18 | 19 | import java.util.Collections; 20 | import java.util.HashMap; 21 | import java.util.Locale; 22 | import java.util.Map; 23 | import java.util.regex.Matcher; 24 | import java.util.regex.Pattern; 25 | 26 | public class DataSizeParser { 27 | private DataSizeParser() { 28 | throw new AssertionError("not instantiable"); 29 | } 30 | 31 | private static final Pattern PATTERN = Pattern.compile("(\\d+)(.+)"); 32 | 33 | private static final Map qualifierToScale; 34 | 35 | static { 36 | final Map temp = new HashMap<>(); 37 | temp.put("b", 1); 38 | temp.put("k", 1024); 39 | temp.put("m", 1024 * 1024); 40 | temp.put("g", 1024 * 1024 * 1024); 41 | qualifierToScale = Collections.unmodifiableMap(temp); 42 | } 43 | 44 | public static DataSize parseDataSize(String s) { 45 | s = s.trim().toLowerCase(Locale.ROOT); 46 | 47 | if (s.equals("0")) { 48 | return DataSize.ofBytes(0); 49 | } 50 | 51 | final Matcher m = PATTERN.matcher(s); 52 | if (!m.matches() || !qualifierToScale.containsKey(m.group(2))) { 53 | throw new IllegalArgumentException("Unable to parse size '" + s + "'." + 54 | " Please specify an integer followed by a size unit (b = bytes, k = kilobytes, m = megabytes, g = gigabytes)." + 55 | " For example, to specify 64 megabytes: 64m"); 56 | } 57 | 58 | final long value = Long.parseLong(m.group(1)); 59 | final Integer unit = qualifierToScale.get(m.group(2)); 60 | return DataSize.ofBytes(value * unit); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/ConfigHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config; 18 | 19 | import org.apache.kafka.common.config.ConfigDef; 20 | import org.apache.kafka.common.config.ConfigException; 21 | 22 | import java.util.Map; 23 | import java.util.function.Consumer; 24 | 25 | public class ConfigHelper { 26 | private static final KafkaConfigProxyFactory factory = 27 | new KafkaConfigProxyFactory("couchbase"); 28 | 29 | private ConfigHelper() { 30 | throw new AssertionError("not instantiable"); 31 | } 32 | 33 | public static ConfigDef define(Class configClass) { 34 | return factory.define(configClass); 35 | } 36 | 37 | public static T parse(Class configClass, Map props) { 38 | return factory.newProxy(configClass, props); 39 | } 40 | 41 | public static String keyName(Class configClass, Consumer methodInvoker) { 42 | return factory.keyName(configClass, methodInvoker); 43 | } 44 | 45 | public interface SimpleValidator { 46 | void validate(T value) throws Exception; 47 | } 48 | 49 | @SuppressWarnings("unchecked") 50 | public static ConfigDef.Validator validate(SimpleValidator validator, String description) { 51 | return new ConfigDef.Validator() { 52 | @Override 53 | public String toString() { 54 | return description; 55 | } 56 | 57 | @Override 58 | public void ensureValid(String name, Object value) { 59 | try { 60 | validator.validate((T) value); 61 | } catch (Exception e) { 62 | throw new ConfigException(name, value, e.getMessage()); 63 | } 64 | } 65 | }; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/test/java/com/couchbase/connect/kafka/util/ListHelperTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util; 18 | 19 | import com.couchbase.client.core.deps.com.fasterxml.jackson.core.type.TypeReference; 20 | import com.couchbase.client.dcp.core.utils.DefaultObjectMapper; 21 | import org.junit.jupiter.api.Test; 22 | 23 | import java.io.IOException; 24 | import java.util.List; 25 | import java.util.stream.IntStream; 26 | 27 | import static com.couchbase.client.core.util.CbCollections.listOf; 28 | import static com.couchbase.connect.kafka.util.ListHelper.chunks; 29 | import static java.util.stream.Collectors.toList; 30 | import static org.junit.jupiter.api.Assertions.assertEquals; 31 | import static org.junit.jupiter.api.Assertions.assertThrows; 32 | 33 | public class ListHelperTest { 34 | 35 | @Test 36 | public void chunkIntoZero() { 37 | assertThrows(IllegalArgumentException.class, () -> chunks(listOf(1, 2, 3, 4, 5), 0)); 38 | } 39 | 40 | @Test 41 | public void chunkIntoVarious() throws Exception { 42 | check(0, 1, "[[]]"); 43 | check(0, 2, "[[],[]]"); 44 | check(4, 2, "[[1,2],[3,4]]"); 45 | check(3, 1, "[[1,2,3]]"); 46 | check(3, 2, "[[1,2],[3]]"); 47 | check(3, 3, "[[1],[2],[3]]"); 48 | check(3, 5, "[[1],[2],[3],[],[]]"); 49 | check(5, 3, "[[1,2],[3,4],[5]]"); 50 | } 51 | 52 | private static void check(int listSize, int numChunks, String expectedJson) throws IOException { 53 | final List list = IntStream.range(1, listSize + 1).boxed().collect(toList()); 54 | List> expected = DefaultObjectMapper.readValue(expectedJson, new TypeReference>>() { 55 | }); 56 | assertEquals(expected, chunks(list, numChunks)); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/DurabilitySetter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util; 18 | 19 | import com.couchbase.client.core.msg.kv.DurabilityLevel; 20 | import com.couchbase.client.java.kv.CommonDurabilityOptions; 21 | import com.couchbase.client.java.kv.PersistTo; 22 | import com.couchbase.client.java.kv.ReplicateTo; 23 | import com.couchbase.connect.kafka.config.sink.DurabilityConfig; 24 | import org.apache.kafka.connect.errors.ConnectException; 25 | 26 | import java.util.function.Consumer; 27 | 28 | import static com.couchbase.connect.kafka.util.config.ConfigHelper.keyName; 29 | 30 | public interface DurabilitySetter extends Consumer> { 31 | static DurabilitySetter create(DurabilityConfig config) { 32 | DurabilityLevel durabilityLevel = config.durability(); 33 | if (durabilityLevel != DurabilityLevel.NONE) { 34 | if (config.persistTo() != PersistTo.NONE || config.replicateTo() != ReplicateTo.NONE) { 35 | String durabilityKey = keyName(DurabilityConfig.class, DurabilityConfig::durability); 36 | String replicateToKey = keyName(DurabilityConfig.class, DurabilityConfig::replicateTo); 37 | String persistToKey = keyName(DurabilityConfig.class, DurabilityConfig::persistTo); 38 | 39 | throw new ConnectException("Invalid durability config. When '" + durabilityKey + "' is set," + 40 | " you must not set '" + replicateToKey + "' or '" + persistToKey + "'."); 41 | } 42 | 43 | return options -> options.durability(durabilityLevel); 44 | } 45 | 46 | PersistTo persistTo = config.persistTo(); 47 | ReplicateTo replicateTo = config.replicateTo(); 48 | return options -> options.durability(persistTo, replicateTo); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/test/java/com/couchbase/connect/kafka/util/config/DurationParserTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config; 18 | 19 | 20 | import org.junit.jupiter.api.Test; 21 | 22 | import static java.util.concurrent.TimeUnit.MILLISECONDS; 23 | import static java.util.concurrent.TimeUnit.SECONDS; 24 | import static org.junit.jupiter.api.Assertions.assertEquals; 25 | import static org.junit.jupiter.api.Assertions.assertThrows; 26 | 27 | public class DurationParserTest { 28 | @Test 29 | public void parseDuration() throws Exception { 30 | assertEquals(0, DurationParser.parseDuration("0", MILLISECONDS)); 31 | assertEquals(0, DurationParser.parseDuration("0s", MILLISECONDS)); 32 | 33 | assertEquals(12345, DurationParser.parseDuration("12345ms", MILLISECONDS)); 34 | 35 | // should round up to nearest second 36 | assertEquals(1, DurationParser.parseDuration("1ms", SECONDS)); 37 | 38 | assertEquals(1, DurationParser.parseDuration("1s", SECONDS)); 39 | assertEquals(2, DurationParser.parseDuration("2s", SECONDS)); 40 | assertEquals(60, DurationParser.parseDuration("1m", SECONDS)); 41 | assertEquals(60 * 60, DurationParser.parseDuration("1h", SECONDS)); 42 | assertEquals(60 * 60 * 24, DurationParser.parseDuration("1d", SECONDS)); 43 | 44 | assertEquals(0, DurationParser.parseDuration("0ms", MILLISECONDS)); 45 | assertEquals(0, DurationParser.parseDuration("0ms", SECONDS)); 46 | } 47 | 48 | @Test 49 | public void missingNumber() { 50 | assertThrows(IllegalArgumentException.class, () -> DurationParser.parseDuration("ms", MILLISECONDS)); 51 | } 52 | 53 | @Test 54 | public void missingUnit() { 55 | assertThrows(IllegalArgumentException.class, () -> DurationParser.parseDuration("300", MILLISECONDS)); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/config/sink/AnalyticsSinkHandlerConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.couchbase.connect.kafka.config.sink; 17 | 18 | import com.couchbase.client.core.annotation.Stability; 19 | import com.couchbase.connect.kafka.util.config.DataSize; 20 | import com.couchbase.connect.kafka.util.config.annotation.Default; 21 | 22 | import java.time.Duration; 23 | 24 | public interface AnalyticsSinkHandlerConfig { 25 | /** 26 | * Every Batch consists of an UPSERT or a DELETE statement, 27 | * based on mutations. 28 | * This property determines the maximum number of records 29 | * in the UPSERT or DELETE statement in the batch. Users can configure 30 | * this parameter based on the capacity of their analytics cluster. 31 | *

32 | * This property is specific to `AnalyticsSinkHandler`. 33 | * 34 | * @since 4.1.14 35 | */ 36 | @Stability.Uncommitted 37 | @Default("100") 38 | int analyticsMaxRecordsInBatch(); 39 | 40 | /** 41 | * Every Batch consists of an UPSERT or a DELETE statement, 42 | * based on mutations. 43 | * This property defines the max size of all docs in bytes in an UPSERT statement in a batch. 44 | * Users can configure this parameter based on the capacity of their analytics cluster. 45 | *

46 | * This property is specific to `AnalyticsSinkHandler`. 47 | * 48 | * @since 4.2.0 49 | */ 50 | @Stability.Uncommitted 51 | @Default("5m") 52 | DataSize analyticsMaxSizeInBatch(); 53 | 54 | /** 55 | * This property determines the time period after which client cancels the Query request for Analytics. 56 | *

57 | * This property is specific to `AnalyticsSinkHandler`. 58 | * 59 | * @since 4.2.0 60 | */ 61 | @Stability.Uncommitted 62 | @Default("5m") 63 | Duration analyticsQueryTimeout(); 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/config/sink/DurabilityConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.config.sink; 18 | 19 | import com.couchbase.client.core.msg.kv.DurabilityLevel; 20 | import com.couchbase.client.java.kv.PersistTo; 21 | import com.couchbase.client.java.kv.ReplicateTo; 22 | import com.couchbase.connect.kafka.util.config.annotation.Default; 23 | 24 | public interface DurabilityConfig { 25 | /** 26 | * The preferred way to specify an enhanced durability requirement 27 | * when using Couchbase Server 6.5 or later. 28 | *

29 | * The default value of `NONE` means a write is considered 30 | * successful as soon as it reaches the memory of the active node. 31 | *

32 | * NOTE: If you set this to anything other than `NONE`, then you 33 | * must not set `couchbase.persist.to` or `couchbase.replicate.to`. 34 | */ 35 | @Default("NONE") 36 | DurabilityLevel durability(); 37 | 38 | /** 39 | * For Couchbase Server versions prior to 6.5, this is how you require 40 | * the connector to verify a write is persisted to disk on a certain 41 | * number of replicas before considering the write successful. 42 | *

43 | * If you're using Couchbase Server 6.5 or later, we recommend 44 | * using the `couchbase.durability` property instead. 45 | */ 46 | @Default("NONE") 47 | PersistTo persistTo(); 48 | 49 | /** 50 | * For Couchbase Server versions prior to 6.5, this is how you require 51 | * the connector to verify a write has reached the memory of a certain 52 | * number of replicas before considering the write successful. 53 | *

54 | * If you're using Couchbase Server 6.5 or later, we recommend 55 | * using the `couchbase.durability` property instead. 56 | */ 57 | @Default("NONE") 58 | ReplicateTo replicateTo(); 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/config/sink/SubDocumentSinkHandlerConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.config.sink; 18 | 19 | import com.couchbase.connect.kafka.handler.sink.SubDocumentSinkHandler; 20 | import com.couchbase.connect.kafka.util.config.annotation.Default; 21 | 22 | /** 23 | * Config properties used only by {@link SubDocumentSinkHandler}. 24 | */ 25 | public interface SubDocumentSinkHandlerConfig { 26 | /** 27 | * JSON Pointer to the property of the Kafka message whose value is 28 | * the subdocument path to use when modifying the Couchbase document. 29 | *

30 | * This property is specific to `SubDocumentSinkHandler`. 31 | */ 32 | @Default 33 | String subdocumentPath(); 34 | 35 | /** 36 | * Setting to indicate the type of update to a sub-document. 37 | *

38 | * This property is specific to `SubDocumentSinkHandler`. 39 | */ 40 | @Default("UPSERT") 41 | Operation subdocumentOperation(); 42 | 43 | /** 44 | * Whether to add the parent paths if they are missing in the document. 45 | *

46 | * This property is specific to `SubDocumentSinkHandler`. 47 | */ 48 | @Default("true") 49 | boolean subdocumentCreatePath(); 50 | 51 | /** 52 | * This property controls whether to create the document if it does not exist. 53 | *

54 | * This property is specific to `SubDocumentSinkHandler`. 55 | */ 56 | @Default("true") 57 | boolean subdocumentCreateDocument(); 58 | 59 | enum Operation { 60 | /** 61 | * Replaces the value at the subdocument path with the Kafka message. 62 | */ 63 | UPSERT, 64 | 65 | /** 66 | * Prepend the Kafka message to the array at the subdocument path. 67 | */ 68 | ARRAY_PREPEND, 69 | 70 | /** 71 | * Append the Kafka message to the array at the subdocument path. 72 | */ 73 | ARRAY_APPEND, 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/JsonHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util; 18 | 19 | import com.fasterxml.jackson.core.JsonFactory; 20 | import com.fasterxml.jackson.core.JsonParser; 21 | import com.fasterxml.jackson.core.JsonToken; 22 | 23 | import java.io.IOException; 24 | 25 | public class JsonHelper { 26 | 27 | private JsonHelper() { 28 | throw new AssertionError("not instantiable"); 29 | } 30 | 31 | private static final JsonFactory jsonFactory = new JsonFactory(); 32 | 33 | public static boolean isValidJson(byte[] bytes) { 34 | try { 35 | final JsonParser parser = jsonFactory.createParser(bytes); 36 | final JsonToken firstToken = parser.nextToken(); 37 | 38 | final JsonToken incrementDepthToken; 39 | final JsonToken decrementDepthToken; 40 | 41 | if (firstToken == JsonToken.START_OBJECT) { 42 | incrementDepthToken = JsonToken.START_OBJECT; 43 | decrementDepthToken = JsonToken.END_OBJECT; 44 | 45 | } else if (firstToken == JsonToken.START_ARRAY) { 46 | incrementDepthToken = JsonToken.START_ARRAY; 47 | decrementDepthToken = JsonToken.END_ARRAY; 48 | 49 | } else { 50 | // valid if there's exactly one token. 51 | return firstToken != null && parser.nextToken() == null; 52 | } 53 | 54 | int depth = 1; 55 | JsonToken token; 56 | while ((token = parser.nextToken()) != null) { 57 | if (token == incrementDepthToken) { 58 | depth++; 59 | } else if (token == decrementDepthToken) { 60 | depth--; 61 | if (depth == 0 && parser.nextToken() != null) { 62 | // multiple JSON roots, or trailing garbage 63 | return false; 64 | } 65 | } 66 | } 67 | } catch (IOException e) { 68 | // malformed 69 | return false; 70 | } 71 | 72 | return true; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/HtmlRenderer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config; 18 | 19 | import org.jsoup.Jsoup; 20 | import org.jsoup.nodes.Element; 21 | import org.jsoup.nodes.Node; 22 | import org.jsoup.nodes.TextNode; 23 | 24 | public class HtmlRenderer { 25 | static final String PARAGRAPH_SEPARATOR = 26 | System.lineSeparator() + System.lineSeparator(); 27 | 28 | public static String htmlToPlaintext(String html) { 29 | StringBuilder result = new StringBuilder(); 30 | html = html.replaceAll("\\s+", " "); 31 | renderAsPlaintext(Jsoup.parse(html).body(), result); 32 | trimRight(result); 33 | return result.toString(); 34 | } 35 | 36 | private static void renderAsPlaintext(Node node, StringBuilder out) { 37 | if (node instanceof TextNode) { 38 | String text = ((TextNode) node).text(); 39 | if (out.length() == 0 || endsWithWhitespace(out)) { 40 | text = trimLeft(text); 41 | } 42 | out.append(text); 43 | return; 44 | } 45 | 46 | if (node instanceof Element) { 47 | Element e = (Element) node; 48 | 49 | if (e.tagName().equals("p") || e.tagName().equals("br")) { 50 | trimRight(out); 51 | if (out.length() > 0) { 52 | out.append(PARAGRAPH_SEPARATOR); 53 | } 54 | } 55 | 56 | for (Node child : e.childNodes()) { 57 | renderAsPlaintext(child, out); 58 | } 59 | } 60 | } 61 | 62 | private static void trimRight(StringBuilder out) { 63 | while (endsWithWhitespace(out)) { 64 | out.setLength(out.length() - 1); 65 | } 66 | } 67 | 68 | private static String trimLeft(String text) { 69 | return text.replaceFirst("^\\s+", ""); 70 | } 71 | 72 | private static boolean endsWithWhitespace(CharSequence out) { 73 | return out.length() > 0 && Character.isWhitespace(out.charAt(out.length() - 1)); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/test/java/com/couchbase/connect/kafka/handler/sink/AnalyticsSinkHandlerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.handler.sink; 18 | 19 | import com.couchbase.client.java.json.JsonArray; 20 | import com.couchbase.client.java.json.JsonObject; 21 | import com.couchbase.connect.kafka.handler.sink.AnalyticsSinkHandler.StatementAndArgs; 22 | import org.junit.jupiter.api.Test; 23 | 24 | import java.util.Arrays; 25 | import java.util.Collections; 26 | 27 | import static org.junit.jupiter.api.Assertions.*; 28 | 29 | 30 | public class AnalyticsSinkHandlerTest { 31 | @Test 32 | public void testDeleteQuery() { 33 | 34 | String expectedDeleteStatement = "DELETE FROM bucket.scope.collection WHERE `UserID`=? AND `Name`=?;"; 35 | JsonArray expectedPositionalParameters = JsonArray.from(Arrays.asList(27, "Jinesh")); 36 | StatementAndArgs obtainedDeleteQuery = AnalyticsSinkHandler.deleteQuery( 37 | "bucket.scope.collection", JsonObject.fromJson("{\"UserID\":27,\"Name\":\"Jinesh\"}")); 38 | 39 | assertEquals(expectedDeleteStatement, obtainedDeleteQuery.statement()); 40 | assertEquals(expectedPositionalParameters, obtainedDeleteQuery.args()); 41 | 42 | expectedDeleteStatement = "DELETE FROM bucket.scope.collection WHERE `array with spaces`=?;"; 43 | expectedPositionalParameters = JsonArray.from(Collections.singletonList(Arrays.asList("a", "b"))); 44 | obtainedDeleteQuery = AnalyticsSinkHandler.deleteQuery( 45 | "bucket.scope.collection", JsonObject.fromJson("{\"array with spaces\":[\"a\",\"b\"]}")); 46 | 47 | assertEquals(expectedDeleteStatement, obtainedDeleteQuery.statement()); 48 | assertEquals(expectedPositionalParameters, obtainedDeleteQuery.args()); 49 | 50 | } 51 | 52 | @Test 53 | public void testGetJsonObject() { 54 | assertNull(AnalyticsSinkHandler.getJsonObject("invalid JSON")); 55 | assertNotNull(AnalyticsSinkHandler.getJsonObject("{\"UserID\":27,\"Name\":\"Jinesh\"}")); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/DurationParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config; 18 | 19 | import java.time.Duration; 20 | import java.util.Collections; 21 | import java.util.HashMap; 22 | import java.util.Locale; 23 | import java.util.Map; 24 | import java.util.concurrent.TimeUnit; 25 | import java.util.regex.Matcher; 26 | import java.util.regex.Pattern; 27 | 28 | public class DurationParser { 29 | private DurationParser() { 30 | throw new AssertionError("not instantiable"); 31 | } 32 | 33 | private static final Pattern DURATION_PATTERN = Pattern.compile("(\\d+)(.+)"); 34 | 35 | private static final Map qualifierToTimeUnit; 36 | 37 | static { 38 | final Map temp = new HashMap<>(); 39 | temp.put("ms", TimeUnit.MILLISECONDS); 40 | temp.put("s", TimeUnit.SECONDS); 41 | temp.put("m", TimeUnit.MINUTES); 42 | temp.put("h", TimeUnit.HOURS); 43 | temp.put("d", TimeUnit.DAYS); 44 | qualifierToTimeUnit = Collections.unmodifiableMap(temp); 45 | } 46 | 47 | public static Duration parseDuration(String s) { 48 | return Duration.ofMillis(parseDuration(s, TimeUnit.MILLISECONDS)); 49 | } 50 | 51 | public static long parseDuration(String s, TimeUnit resultUnit) { 52 | s = s.trim().toLowerCase(Locale.ROOT); 53 | if (s.equals("0")) { 54 | return 0; 55 | } 56 | final Matcher m = DURATION_PATTERN.matcher(s); 57 | if (!m.matches() || !qualifierToTimeUnit.containsKey(m.group(2))) { 58 | throw new IllegalArgumentException("Unable to parse duration '" + s + "'." + 59 | " Please specify an integer followed by a time unit (ms = milliseconds, s = seconds, m = minutes, h = hours, d = days)." + 60 | " For example, to specify 30 minutes: 30m"); 61 | } 62 | 63 | final long value = Long.parseLong(m.group(1)); 64 | final TimeUnit unit = qualifierToTimeUnit.get(m.group(2)); 65 | return divideRoundUp(unit.toMillis(value), resultUnit.toMillis(1)); 66 | } 67 | 68 | private static long divideRoundUp(long num, long divisor) { 69 | // assume both inputs are positive 70 | return (num + divisor - 1) / divisor; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/test/java/com/couchbase/connect/kafka/util/TopicMapTest.java: -------------------------------------------------------------------------------- 1 | package com.couchbase.connect.kafka.util; 2 | 3 | 4 | import org.junit.jupiter.api.Test; 5 | 6 | import java.util.Arrays; 7 | import java.util.List; 8 | 9 | import static com.couchbase.client.core.util.CbCollections.listOf; 10 | import static com.couchbase.client.core.util.CbCollections.mapOf; 11 | import static org.junit.jupiter.api.Assertions.assertEquals; 12 | import static org.junit.jupiter.api.Assertions.assertThrows; 13 | 14 | public class TopicMapTest { 15 | @Test 16 | public void parseTopicToCollection() { 17 | assertEquals( 18 | mapOf( 19 | "topic1", new Keyspace("default-bucket", "my-scope", "collection1"), 20 | "topic2", new Keyspace("default-bucket", "their-scope", "collection2") 21 | ), 22 | TopicMap.parseTopicToCollection(listOf( 23 | "topic1=my-scope.collection1", 24 | "topic2=their-scope.collection2" 25 | ), "default-bucket") 26 | ); 27 | } 28 | 29 | @Test 30 | public void parseTopicToCollectionWithDatabaseName() { 31 | assertEquals( 32 | mapOf( 33 | "topic1", new Keyspace("my-bucket", "my-scope", "collection1"), 34 | "topic2", new Keyspace("their-bucket", "their-scope", "collection2") 35 | ), 36 | TopicMap.parseTopicToCollection(listOf( 37 | "topic1=my-bucket.my-scope.collection1", 38 | "topic2=their-bucket.their-scope.collection2" 39 | ), "default-bucket") 40 | ); 41 | } 42 | 43 | @Test 44 | public void parseCollectionToTopic() { 45 | assertEquals( 46 | mapOf( 47 | new ScopeAndCollection("my-scope", "collection1"), "topic1", 48 | new ScopeAndCollection("their-scope", "collection2"), "topic2" 49 | ), 50 | TopicMap.parseCollectionToTopic(listOf( 51 | "my-scope.collection1=topic1", 52 | "their-scope.collection2=topic2" 53 | )) 54 | ); 55 | } 56 | 57 | @Test 58 | public void parseMissingDelim() { 59 | List topicsToCollections = Arrays.asList( 60 | "topic1=myscope.collection1", 61 | "topic2.theirscope.collection2" 62 | ); 63 | assertThrows(IllegalArgumentException.class, () -> TopicMap.parseTopicToCollection(topicsToCollections, "default-bucket")); 64 | } 65 | 66 | @Test 67 | public void parseExtraDelim() { 68 | List topicsToCollections = Arrays.asList( 69 | "topic1=myscope.collection1", 70 | "topic2==theirscope.collection2" 71 | ); 72 | assertThrows(IllegalArgumentException.class, () -> TopicMap.parseTopicToCollection(topicsToCollections, "default-bucket")); 73 | } 74 | 75 | @Test 76 | public void parseMalformedCollection() { 77 | assertThrows(IllegalArgumentException.class, () -> TopicMap.parseTopicToCollection(listOf("foo=bar"), "default-bucket")); 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /examples/custom-extensions/src/main/java/com/couchbase/connect/kafka/example/CustomSourceHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.example; 18 | 19 | import com.couchbase.connect.kafka.handler.source.RawJsonSourceHandler; 20 | import com.couchbase.connect.kafka.handler.source.SourceHandlerParams; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | import java.util.Map; 25 | 26 | /** 27 | * An example of extending {@link RawJsonSourceHandler} to add custom behavior. 28 | * To use this handler, build this project and move the resulting JAR to 29 | * the same location as the kafka-connect-couchbase JAR. Then use these connector 30 | * config properties: 31 | * 

32 |  * couchbase.source.handler=com.couchbase.connect.kafka.example.CustomSourceHandler
33 |  * value.converter=org.apache.kafka.connect.converters.ByteArrayConverter
34 |  *
35 | * For more customization ideas, please see the source code for {@link RawJsonSourceHandler} 36 | * and {@link com.couchbase.connect.kafka.handler.source.DefaultSchemaSourceHandler}. 37 | */ 38 | public class CustomSourceHandler extends RawJsonSourceHandler { 39 | private static final Logger LOGGER = LoggerFactory.getLogger(CustomSourceHandler.class); 40 | 41 | private String specialKeyPrefix; 42 | 43 | @Override 44 | public void init(Map connectorConfig) { 45 | super.init(connectorConfig); 46 | 47 | // read a value from the connector config and save it for later 48 | specialKeyPrefix = connectorConfig.getOrDefault("example.special.key.prefix", "xyzzy"); 49 | } 50 | 51 | @Override 52 | protected boolean passesFilter(SourceHandlerParams params) { 53 | // Ignore deletions and expirations instead of sending message with null value. 54 | // NOTE: another way to achieve this result would be to use a DropIfNullValue transform. 55 | return params.documentEvent().isMutation(); 56 | } 57 | 58 | @Override 59 | protected String getTopic(SourceHandlerParams params) { 60 | // Alter the topic based on document key / content: 61 | if (params.documentEvent().key().startsWith(specialKeyPrefix)) { 62 | return params.topic() + "-" + specialKeyPrefix; 63 | } 64 | 65 | // Or use the default topic 66 | return null; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/SourceOffset.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka; 18 | 19 | import com.couchbase.client.dcp.highlevel.SnapshotMarker; 20 | import com.couchbase.client.dcp.highlevel.StreamOffset; 21 | 22 | import java.util.Map; 23 | 24 | import static com.couchbase.client.core.util.CbCollections.mapOf; 25 | import static java.util.Objects.requireNonNull; 26 | 27 | /** 28 | * Info required for resuming a DCP stream. 29 | */ 30 | class SourceOffset { 31 | private final StreamOffset streamOffset; 32 | 33 | public SourceOffset(StreamOffset offset) { 34 | this.streamOffset = requireNonNull(offset); 35 | } 36 | 37 | public SourceOffset withVbucketUuid(long vbucketUuid) { 38 | return new SourceOffset( 39 | new StreamOffset( 40 | vbucketUuid, 41 | streamOffset.getSeqno(), 42 | streamOffset.getSnapshot(), 43 | streamOffset.getCollectionsManifestUid() 44 | ) 45 | ); 46 | } 47 | 48 | public StreamOffset asStreamOffset() { 49 | return streamOffset; 50 | } 51 | 52 | public Map toMap() { 53 | return mapOf( 54 | "bySeqno", streamOffset.getSeqno(), 55 | "vbuuid", streamOffset.getVbuuid(), 56 | "snapshotStartSeqno", streamOffset.getSnapshot().getStartSeqno(), 57 | "snapshotEndSeqno", streamOffset.getSnapshot().getEndSeqno(), 58 | "collectionsManifestUid", streamOffset.getCollectionsManifestUid() 59 | ); 60 | } 61 | 62 | public static SourceOffset fromMap(Map map) { 63 | // Only the sequence number is guaranteed to be present. 64 | // All other properties were added in later connector versions. 65 | long seqno = (Long) map.get("bySeqno"); 66 | 67 | return new SourceOffset( 68 | new StreamOffset( 69 | (Long) map.getOrDefault("vbuuid", 0L), 70 | seqno, 71 | new SnapshotMarker( 72 | (Long) map.getOrDefault("snapshotStartSeqno", seqno), 73 | (Long) map.getOrDefault("snapshotEndSeqno", seqno) 74 | ), 75 | (Long) map.getOrDefault("collectionsManifestUid", 0L) 76 | )); 77 | } 78 | 79 | @Override 80 | public String toString() { 81 | return streamOffset.toString(); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /.github/workflows/deploy-release.yml: -------------------------------------------------------------------------------- 1 | name: Maven Deploy Release 2 | 3 | concurrency: 4 | group: ${{ github.workflow }}-${{ github.ref }} 5 | 6 | on: 7 | workflow_dispatch: 8 | inputs: 9 | tag: 10 | type: string 11 | description: Tag to release. Must already exist. 12 | required: true 13 | 14 | jobs: 15 | build: 16 | runs-on: ubuntu-latest 17 | permissions: 18 | contents: read 19 | id-token: write 20 | 21 | steps: 22 | - uses: actions/checkout@v4 23 | with: 24 | ref: ${{ inputs.tag }} 25 | 26 | - name: Verify the ref is actually a tag 27 | run: git tag --list | grep --line-regexp ${{ inputs.tag }} 28 | 29 | - uses: actions/setup-java@v4 30 | with: 31 | java-version: '21' 32 | distribution: 'temurin' 33 | 34 | server-id: 'central' 35 | server-username: MAVEN_USERNAME 36 | server-password: MAVEN_PASSWORD 37 | 38 | cache: 'maven' 39 | 40 | - name: Printing maven version 41 | run: ./mvnw --version 42 | 43 | - name: Build and deploy to Maven Central 44 | run: ./mvnw deploy --batch-mode -Dgpg.signer=bc -Prelease 45 | env: 46 | MAVEN_USERNAME: ${{ vars.MAVEN_USERNAME }} 47 | MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }} 48 | MAVEN_GPG_KEY: ${{ secrets.SDK_ROBOT_GPG_PRIVATE_KEY }} 49 | MAVEN_GPG_PASSPHRASE: '' 50 | 51 | - uses: aws-actions/configure-aws-credentials@v3 52 | with: 53 | role-to-assume: arn:aws:iam::786014483886:role/SDK_GHA 54 | aws-region: us-west-1 55 | 56 | - name: Sign and publish connector package 57 | # There isn't a nice way to sign an arbitrary file using the Maven GPG plugin, 58 | # so cheat by "deploying" to a fake repository. We expect gpg:sign-and-deploy-file 59 | # to fail, but not before it generates the signature. 60 | run: | 61 | VERS=${{ inputs.tag }} 62 | ARTIFACT=couchbase-kafka-connect-couchbase-${VERS}.zip 63 | cd target/components/packages 64 | ls ${ARTIFACT} 65 | ../../../mvnw gpg:sign-and-deploy-file --batch-mode -Dgpg.signer=bc -Dfile=${ARTIFACT} -Durl=fake -DgroupId=fake -DartifactId=fake -Dversion=fake -DgeneratePom=false || true 66 | echo "Don't worry, we expected that to print a bunch of error messages. All that matters is that it created the signature file." 67 | SIGNATURE=${ARTIFACT}.asc 68 | ls ${SIGNATURE} 69 | aws s3 cp --acl public-read ${ARTIFACT} s3://packages.couchbase.com/clients/kafka/${VERS}/${ARTIFACT} 70 | aws s3 cp --acl public-read ${SIGNATURE} s3://packages.couchbase.com/clients/kafka/${VERS}/${SIGNATURE} 71 | env: 72 | MAVEN_GPG_KEY: ${{ secrets.SDK_ROBOT_GPG_PRIVATE_KEY }} 73 | MAVEN_GPG_PASSPHRASE: '' 74 | 75 | - name: Remove artifacts from Maven repo so they're not cached 76 | run: rm -rfv ~/.m2/repository/com/couchbase/client/ 77 | -------------------------------------------------------------------------------- /examples/custom-extensions/src/main/java/com/couchbase/connect/kafka/example/CustomSinkHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.example; 18 | 19 | import com.couchbase.connect.kafka.handler.sink.SinkAction; 20 | import com.couchbase.connect.kafka.handler.sink.SinkDocument; 21 | import com.couchbase.connect.kafka.handler.sink.SinkHandler; 22 | import com.couchbase.connect.kafka.handler.sink.SinkHandlerContext; 23 | import com.couchbase.connect.kafka.handler.sink.SinkHandlerParams; 24 | import com.fasterxml.jackson.databind.JsonNode; 25 | import com.fasterxml.jackson.databind.ObjectMapper; 26 | import com.fasterxml.jackson.databind.node.ObjectNode; 27 | import org.apache.kafka.common.config.ConfigException; 28 | 29 | import java.io.IOException; 30 | 31 | /** 32 | * An example sink handler that reads a field name from the 33 | * {@code "example.field.to.remove"} connector config property and removes 34 | * that field from every message before writing the document to Couchbase. 35 | */ 36 | public class CustomSinkHandler implements SinkHandler { 37 | private static final ObjectMapper mapper = new ObjectMapper(); 38 | 39 | private String fieldNameToRemove; 40 | 41 | @Override 42 | public void init(SinkHandlerContext context) { 43 | String configPropertyName = "example.field.to.remove"; 44 | fieldNameToRemove = context.configProperties().get(configPropertyName); 45 | if (fieldNameToRemove == null) { 46 | throw new ConfigException("Missing required connector config property: " + configPropertyName); 47 | } 48 | } 49 | 50 | @Override 51 | public SinkAction handle(SinkHandlerParams params) { 52 | String documentId = getDocumentId(params); 53 | SinkDocument doc = params.document().orElse(null); 54 | return doc == null 55 | ? SinkAction.ignore() // ignore Kafka records with null values 56 | : SinkAction.upsertJson(params, params.collection(), documentId, transform(doc.content())); 57 | } 58 | 59 | private byte[] transform(byte[] content) { 60 | try { 61 | JsonNode node = mapper.readTree(content); 62 | 63 | if (node.has(fieldNameToRemove)) { 64 | ((ObjectNode) node).remove(fieldNameToRemove); 65 | return mapper.writeValueAsBytes(node); 66 | } 67 | 68 | return content; 69 | 70 | } catch (IOException e) { 71 | throw new RuntimeException(e); 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/handler/source/SourceHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.handler.source; 18 | 19 | import com.couchbase.client.core.annotation.Stability; 20 | import com.couchbase.connect.kafka.config.common.LoggingConfig; 21 | import org.apache.kafka.clients.producer.RecordMetadata; 22 | import org.apache.kafka.connect.source.SourceRecord; 23 | import org.apache.kafka.connect.source.SourceTask; 24 | import org.jspecify.annotations.Nullable; 25 | 26 | import java.util.Map; 27 | 28 | /** 29 | * Primary extension point for customizing how the Source Connector publishes messages to Kafka. 30 | */ 31 | public interface SourceHandler { 32 | /** 33 | * Called one time when the filter is instantiated. 34 | * 35 | * @param configProperties the connector configuration. 36 | */ 37 | default void init(Map configProperties) { 38 | } 39 | 40 | /** 41 | * Translates a DocumentEvent into a SourceRecord for publication to a Kafka topic. 42 | *

43 | * The document event is specified by the SourceHandlerParams parameter block, along with 44 | * other bits of info that may be useful to custom implementations. 45 | *

46 | * The handler may route the message to an arbitrary topic by setting the {@code topic} 47 | * property of the returned builder, or may leave it null to use the default topic 48 | * from the connector configuration. 49 | *

50 | * The handler may filter the event stream by returning {@code null} to skip this event. 51 | * 52 | * @param params A parameter block containing input to the handler, 53 | * most notably the {@link DocumentEvent}. 54 | * @return (nullable) The record to publish to Kafka, or {@code null} to skip this event. 55 | */ 56 | SourceRecordBuilder handle(SourceHandlerParams params); 57 | 58 | /** 59 | * Called whenever the Kafka Connect framework calls {@link SourceTask#commitRecord(SourceRecord, RecordMetadata)}. 60 | *

61 | * Subclasses may override this method to do something special with committed record metadata 62 | * beyond the usual document lifecycle logging. 63 | *

64 | * This method should return quickly. 65 | *

66 | * The default implementation does nothing, which is always okay. 67 | * 68 | * @see LoggingConfig#logDocumentLifecycle() 69 | */ 70 | @Stability.Volatile 71 | default void onRecordCommitted(SourceRecord record, @Nullable RecordMetadata metadata) { 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/ScopeAndCollection.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util; 18 | 19 | import com.couchbase.connect.kafka.handler.source.DocumentEvent; 20 | 21 | import java.util.Objects; 22 | 23 | import static java.util.Objects.requireNonNull; 24 | 25 | /** 26 | * Like a {@link Keyspace} where the database component is always null. 27 | * It's a distinct type (at least for now) to reinforce the idea that 28 | * the source connector can stream from only one bucket. 29 | *

30 | * In other words, the source connector uses ScopeAndCollection because 31 | * it reads from only one bucket, and the sink connector uses Keyspace 32 | * because it can write to multiple buckets. 33 | */ 34 | public class ScopeAndCollection { 35 | private final String scope; 36 | private final String collection; 37 | 38 | public static ScopeAndCollection parse(String scopeAndCollection) { 39 | try { 40 | Keyspace ks = Keyspace.parse(scopeAndCollection, null); 41 | if (ks.getBucket() != null) { 42 | throw new IllegalArgumentException("Expected 2 components, but got 3; a bucket name is not valid in this context."); 43 | } 44 | return new ScopeAndCollection(ks.getScope(), ks.getCollection()); 45 | } catch (Exception e) { 46 | throw new IllegalArgumentException( 47 | "Expected a qualified collection name (scope.collection) with no bucket component, but got: " + scopeAndCollection, 48 | e 49 | ); 50 | } 51 | } 52 | 53 | public static ScopeAndCollection from(DocumentEvent docEvent) { 54 | return new ScopeAndCollection(docEvent.collectionMetadata().scopeName(), docEvent.collectionMetadata().collectionName()); 55 | } 56 | 57 | public ScopeAndCollection(String scope, String collection) { 58 | this.scope = requireNonNull(scope); 59 | this.collection = requireNonNull(collection); 60 | } 61 | 62 | public String getScope() { 63 | return scope; 64 | } 65 | 66 | public String getCollection() { 67 | return collection; 68 | } 69 | 70 | @Override 71 | public boolean equals(Object o) { 72 | if (this == o) return true; 73 | if (o == null || getClass() != o.getClass()) return false; 74 | ScopeAndCollection that = (ScopeAndCollection) o; 75 | return scope.equals(that.scope) && 76 | collection.equals(that.collection); 77 | } 78 | 79 | @Override 80 | public int hashCode() { 81 | return Objects.hash(scope, collection); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/TopicMap.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util; 18 | 19 | import org.jspecify.annotations.Nullable; 20 | 21 | import java.util.HashMap; 22 | import java.util.List; 23 | import java.util.Map; 24 | import java.util.function.Function; 25 | 26 | import static java.util.stream.Collectors.toMap; 27 | 28 | /** 29 | * Helper methods for parsing legacy map-like config options. 30 | * 31 | * @deprecated New map-like config options should use contextual overrides. 32 | * See {@link com.couchbase.connect.kafka.util.config.Contextual}. 33 | */ 34 | @Deprecated 35 | public class TopicMap { 36 | private TopicMap() { 37 | throw new AssertionError("not instantiable"); 38 | } 39 | 40 | public static Map parseTopicToDocumentId(List topicToDocumentIdFormat) { 41 | return mapValues(parseCommon(topicToDocumentIdFormat), DocumentIdExtractor::from); 42 | } 43 | 44 | public static Map parseTopicToCollection( 45 | List topicToCollection, 46 | @Nullable String defaultBucket 47 | ) { 48 | return mapValues(parseCommon(topicToCollection), it -> Keyspace.parse(it, defaultBucket)); 49 | } 50 | 51 | public static Map parseCollectionToTopic(List collectionToTopic) { 52 | return mapKeys(parseCommon(collectionToTopic), ScopeAndCollection::parse); 53 | } 54 | 55 | private static Map parseCommon(List map) { 56 | Map result = new HashMap<>(); 57 | for (String entry : map) { 58 | String[] components = entry.split("=", -1); 59 | if (components.length != 2) { 60 | throw new IllegalArgumentException("Bad entry: '" + entry + "'. Expected exactly one equals (=) character separating key and value."); 61 | } 62 | result.put(components[0], components[1]); 63 | } 64 | return result; 65 | } 66 | 67 | private static Map mapValues(Map map, Function valueTransformer) { 68 | return map.entrySet().stream() 69 | .collect(toMap( 70 | Map.Entry::getKey, 71 | entry -> valueTransformer.apply(entry.getValue()) 72 | )); 73 | } 74 | 75 | private static Map mapKeys(Map map, Function keyTransformer) { 76 | return map.entrySet().stream() 77 | .collect(toMap( 78 | entry -> keyTransformer.apply(entry.getKey()), 79 | Map.Entry::getValue 80 | )); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/transform/DeserializeJson.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.transform; 18 | 19 | import com.fasterxml.jackson.databind.DeserializationFeature; 20 | import com.fasterxml.jackson.databind.ObjectMapper; 21 | import org.apache.kafka.common.config.ConfigDef; 22 | import org.apache.kafka.common.utils.ByteBufferInputStream; 23 | import org.apache.kafka.connect.connector.ConnectRecord; 24 | import org.apache.kafka.connect.errors.DataException; 25 | import org.apache.kafka.connect.transforms.Transformation; 26 | 27 | import java.io.IOException; 28 | import java.nio.ByteBuffer; 29 | import java.util.Map; 30 | 31 | public class DeserializeJson> implements Transformation { 32 | private static final ObjectMapper objectMapper = new ObjectMapper(); 33 | 34 | static { 35 | objectMapper.enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS); 36 | } 37 | 38 | public static final String OVERVIEW_DOC = 39 | "Convert JSON values from a byte[] or ByteBuffer to a Map so downstream transforms can operate on the value." + 40 | " Any null values are passed through unmodified." + 41 | " The value schema is passed through unmodified (may be null)."; 42 | 43 | @Override 44 | public R apply(R record) { 45 | final Object value = record.value(); 46 | final Map newValue; 47 | 48 | try { 49 | if (value == null) { 50 | return record; 51 | 52 | } else if (value instanceof byte[]) { 53 | newValue = objectMapper.readValue((byte[]) value, Map.class); 54 | 55 | } else if (value instanceof ByteBuffer) { 56 | try (ByteBufferInputStream in = new ByteBufferInputStream((ByteBuffer) value)) { 57 | newValue = objectMapper.readValue(in, Map.class); 58 | } 59 | 60 | } else { 61 | throw new DataException(getClass().getSimpleName() + " transform expected value to be a byte array or ByteBuffer but got " + value.getClass().getName()); 62 | } 63 | 64 | return record.newRecord(record.topic(), record.kafkaPartition(), 65 | record.keySchema(), record.key(), 66 | record.valueSchema(), newValue, 67 | record.timestamp()); 68 | 69 | } catch (IOException e) { 70 | throw new DataException(getClass().getSimpleName() + " transform expected value to be JSON but got something else.", e); 71 | } 72 | } 73 | 74 | @Override 75 | public ConfigDef config() { 76 | return new ConfigDef(); 77 | } 78 | 79 | @Override 80 | public void close() { 81 | } 82 | 83 | @Override 84 | public void configure(Map configs) { 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/Watchdog.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util; 18 | 19 | import com.couchbase.client.core.annotation.Stability; 20 | import com.couchbase.client.core.env.CouchbaseThreadFactory; 21 | import com.couchbase.client.core.util.NanoTimestamp; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import java.time.Duration; 26 | import java.util.concurrent.Executors; 27 | import java.util.concurrent.ScheduledExecutorService; 28 | import java.util.concurrent.ThreadFactory; 29 | 30 | import static com.couchbase.connect.kafka.util.ConnectHelper.getConnectorContextFromLoggingContext; 31 | import static java.util.Objects.requireNonNull; 32 | import static java.util.concurrent.TimeUnit.SECONDS; 33 | 34 | @Stability.Internal 35 | public final class Watchdog { 36 | private static final Logger log = LoggerFactory.getLogger(Watchdog.class); 37 | private static final ThreadFactory threadFactory = new CouchbaseThreadFactory("cb-watchdog-"); 38 | 39 | private static class State { 40 | private final NanoTimestamp startTime = NanoTimestamp.now(); 41 | private final String name; 42 | 43 | private State(String name) { 44 | this.name = requireNonNull(name); 45 | } 46 | } 47 | 48 | private final String taskUuid; 49 | private ScheduledExecutorService executor; 50 | private volatile State lastObservedState = new State("stopped"); 51 | private volatile State currentState = new State("initial"); 52 | 53 | public Watchdog(String taskUuid) { 54 | this.taskUuid = requireNonNull(taskUuid); 55 | } 56 | 57 | public void enterState(String s) { 58 | this.currentState = new State(s); 59 | log.debug("Transitioned to state: {}; taskUuid={}", s, taskUuid); 60 | } 61 | 62 | public synchronized void start() { 63 | stop(); 64 | 65 | executor = Executors.newSingleThreadScheduledExecutor(threadFactory); 66 | getConnectorContextFromLoggingContext().ifPresent(ctx -> 67 | executor.execute(() -> Thread.currentThread().setName(Thread.currentThread().getName() + ctx)) 68 | ); 69 | 70 | currentState = new State("starting"); 71 | lastObservedState = currentState; 72 | 73 | executor.scheduleWithFixedDelay(() -> { 74 | if (currentState == lastObservedState) { 75 | Duration elapsed = lastObservedState.startTime.elapsed(); 76 | log.warn("SourceTask has been in same state ({}) for {}; taskUuid={} ", lastObservedState.name, elapsed, taskUuid); 77 | } 78 | lastObservedState = currentState; 79 | }, 10, 10, SECONDS); 80 | } 81 | 82 | public synchronized void stop() { 83 | if (executor != null) { 84 | executor.shutdownNow(); 85 | executor = null; 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/config/common/ConnectionConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.config.common; 18 | 19 | import com.couchbase.connect.kafka.util.config.annotation.Default; 20 | import com.couchbase.connect.kafka.util.config.annotation.EnvironmentVariable; 21 | import com.couchbase.connect.kafka.util.config.annotation.Importance; 22 | import com.couchbase.connect.kafka.util.config.annotation.Width; 23 | import org.apache.kafka.common.config.types.Password; 24 | 25 | import java.time.Duration; 26 | import java.util.List; 27 | 28 | import static org.apache.kafka.common.config.ConfigDef.Importance.HIGH; 29 | import static org.apache.kafka.common.config.ConfigDef.Width.LONG; 30 | 31 | public interface ConnectionConfig { 32 | /** 33 | * Addresses of Couchbase Server nodes, delimited by commas. 34 | *

35 | * If a custom port is specified, it must be the KV port 36 | * (which is normally 11210 for insecure connections, 37 | * or 11207 for secure connections). 38 | */ 39 | @Width(LONG) 40 | @Importance(HIGH) 41 | List seedNodes(); 42 | 43 | /** 44 | * Name of the Couchbase user to authenticate as. 45 | */ 46 | @Importance(HIGH) 47 | String username(); 48 | 49 | /** 50 | * Password of the Couchbase user. 51 | */ 52 | @Importance(HIGH) 53 | @EnvironmentVariable("KAFKA_COUCHBASE_PASSWORD") 54 | Password password(); 55 | 56 | /** 57 | * Name of the Couchbase bucket to use. 58 | *

59 | * This property is required unless using the experimental AnalyticsSinkHandler. 60 | */ 61 | @Default 62 | @Width(LONG) 63 | @Importance(HIGH) 64 | String bucket(); 65 | 66 | /** 67 | * The network selection strategy for connecting to a Couchbase Server cluster 68 | * that advertises alternate addresses. 69 | *

70 | * A Couchbase node running inside a container environment (like Docker or Kubernetes) 71 | * might be configured to advertise both its address within the container environment 72 | * (known as its "default" address) as well as an "external" address for use by clients 73 | * connecting from outside the environment. 74 | *

75 | * Setting the 'couchbase.network' config property to 'default' or 'external' forces 76 | * the selection of the respective addresses. Setting the value to 'auto' tells the 77 | * connector to select whichever network contains the addresses specified in the 78 | * 'couchbase.seed.nodes' config property. 79 | */ 80 | @Default("auto") 81 | String network(); 82 | 83 | /** 84 | * On startup, the connector will wait this long for a Couchbase connection to be established. 85 | * If a connection is not established before the timeout expires, the connector will terminate. 86 | */ 87 | @Default("30s") 88 | Duration bootstrapTimeout(); 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/handler/source/ConfigurableSchemaSourceHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.handler.source; 18 | 19 | import com.couchbase.client.core.annotation.Stability; 20 | import com.couchbase.connect.kafka.config.source.CouchbaseSourceTaskConfig; 21 | import com.couchbase.connect.kafka.util.SchemaHelper; 22 | import com.couchbase.connect.kafka.util.ScopeAndCollection; 23 | import com.couchbase.connect.kafka.util.config.ConfigHelper; 24 | import com.couchbase.connect.kafka.util.config.LookupTable; 25 | import org.apache.kafka.connect.data.Schema; 26 | import org.apache.kafka.connect.data.Struct; 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | import java.util.Map; 31 | 32 | import static com.couchbase.connect.kafka.util.SchemaHelper.*; 33 | 34 | /** 35 | * This handler enforces a custom schema using the {@link org.apache.kafka.connect.json.JsonConverter}. 36 | * It only supports documents where the root is a JSON object. Documents that do not match the schema or are not a JSON object will be filtered out 37 | * The schema is provided in the format of an Avro Schema Record, stored in a Json Object 38 | * The schema is read from the {@link com.couchbase.connect.kafka.config.source.SchemaConfig#valueSchema()} configuration property 39 | */ 40 | @Stability.Uncommitted 41 | public class ConfigurableSchemaSourceHandler implements SourceHandler{ 42 | 43 | private static final Logger LOGGER = LoggerFactory.getLogger(ConfigurableSchemaSourceHandler.class); 44 | private LookupTable valueSchemas; 45 | 46 | @Override 47 | public void init(Map configProperties) { 48 | CouchbaseSourceTaskConfig config = ConfigHelper.parse(CouchbaseSourceTaskConfig.class, configProperties); 49 | 50 | valueSchemas = config.valueSchema() 51 | .mapKeys(ScopeAndCollection::parse) 52 | .mapValues(SchemaHelper::parseSchema); 53 | } 54 | 55 | @Override 56 | public SourceRecordBuilder handle(SourceHandlerParams params) { 57 | SourceRecordBuilder builder = new SourceRecordBuilder(); 58 | 59 | Schema schemaToUse = valueSchemas.get(ScopeAndCollection.from(params.documentEvent())); 60 | 61 | builder.topic(params.topic()); 62 | 63 | // No schema provided is shorthand for String schema 64 | builder.key(params.documentEvent().key()); 65 | 66 | try { 67 | Struct record = buildStruct(schemaToUse, params.documentEvent().content()); 68 | 69 | checkStruct(schemaToUse, record); 70 | 71 | builder.value(schemaToUse, record); 72 | } catch (Exception e) { 73 | LOGGER.debug("Document {} doesn't match specified schema. Will not be pushed to Kafka", params.documentEvent().key(), e); 74 | return null; 75 | } 76 | 77 | return builder; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/ConnectorLifecycle.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka; 18 | 19 | import com.couchbase.client.core.json.Mapper; 20 | import com.couchbase.client.dcp.metrics.LogLevel; 21 | import com.couchbase.client.dcp.util.PartitionSet; 22 | import com.couchbase.connect.kafka.util.Version; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | import java.util.Collections; 27 | import java.util.LinkedHashMap; 28 | import java.util.List; 29 | import java.util.Map; 30 | import java.util.UUID; 31 | 32 | import static com.couchbase.connect.kafka.util.ConnectHelper.getTaskIdFromLoggingContext; 33 | 34 | public class ConnectorLifecycle { 35 | 36 | public enum Milestone { 37 | CONNECTOR_STARTED, 38 | CONNECTOR_STOPPED, 39 | 40 | /** 41 | * The connector has created a configuration for each task, and specified 42 | * which Couchbase partitions each task should stream from. 43 | */ 44 | PARTITIONS_ASSIGNED, 45 | ; 46 | 47 | private final Logger logger = LoggerFactory.getLogger(ConnectorLifecycle.class.getName() + "." + this.name()); 48 | } 49 | 50 | private final LogLevel logLevel = LogLevel.INFO; 51 | 52 | private final String uuid = UUID.randomUUID().toString(); 53 | 54 | public void logConnectorStarted(String name) { 55 | Map details = new LinkedHashMap<>(); 56 | details.put("connectorVersion", Version.getVersion()); 57 | details.put("connectorName", name); 58 | logMilestone(Milestone.CONNECTOR_STARTED, details); 59 | } 60 | 61 | public void logPartitionsAssigned(List partitions) { 62 | Map details = new LinkedHashMap<>(); 63 | for (int i = 0; i < partitions.size(); i++) { 64 | details.put("task" + i, partitions.get(i).format()); 65 | } 66 | logMilestone(Milestone.PARTITIONS_ASSIGNED, details); 67 | } 68 | 69 | public void logConnectorStopped() { 70 | logMilestone(Milestone.CONNECTOR_STOPPED, Collections.emptyMap()); 71 | } 72 | 73 | private void logMilestone(ConnectorLifecycle.Milestone milestone, Map milestoneDetails) { 74 | if (enabled()) { 75 | LinkedHashMap message = new LinkedHashMap<>(); 76 | message.put("milestone", milestone); 77 | message.put("connectorUuid", uuid); 78 | getTaskIdFromLoggingContext().ifPresent(id -> message.put("taskId", id)); 79 | message.putAll(milestoneDetails); 80 | doLog(milestone.logger, message); 81 | } 82 | } 83 | 84 | private void doLog(Logger logger, Object message) { 85 | try { 86 | logLevel.log(logger, Mapper.encodeAsString(message)); 87 | } catch (Exception e) { 88 | logLevel.log(logger, message.toString()); 89 | } 90 | } 91 | 92 | private boolean enabled() { 93 | return true; 94 | } 95 | 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/config/LookupTable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util.config; 18 | 19 | import org.apache.kafka.common.config.ConfigException; 20 | 21 | import java.util.HashMap; 22 | import java.util.Map; 23 | import java.util.function.Function; 24 | 25 | import static com.couchbase.client.core.util.CbCollections.mapCopyOf; 26 | import static java.util.Objects.requireNonNull; 27 | import static java.util.stream.Collectors.toMap; 28 | 29 | /** 30 | * A read-only map that returns a default value for absent keys. 31 | */ 32 | public class LookupTable { 33 | private final String propertyName; 34 | private final V defaultValue; 35 | private final Map map; 36 | 37 | LookupTable(String propertyName, V defaultValue, Map map) { 38 | this.propertyName = requireNonNull(propertyName); 39 | this.defaultValue = requireNonNull(defaultValue); 40 | this.map = mapCopyOf(map); 41 | } 42 | 43 | public V get(K key) { 44 | return map.getOrDefault(key, defaultValue); 45 | } 46 | 47 | public LookupTable withUnderlay(Map contexts) { 48 | Map newMap = new HashMap<>(contexts); 49 | newMap.putAll(map); 50 | return new LookupTable<>(propertyName, defaultValue, newMap); 51 | } 52 | 53 | public LookupTable mapKeys(Function keyMapper) { 54 | Map newMap = map.entrySet().stream() 55 | .collect(toMap( 56 | entry -> { 57 | try { 58 | return keyMapper.apply((String) entry.getKey()); 59 | } catch (RuntimeException e) { 60 | throw new ConfigException("Invalid configuration " + propertyName + "[" + entry.getKey() + "] ; " + e); 61 | } 62 | }, 63 | Map.Entry::getValue 64 | )); 65 | return new LookupTable<>(propertyName, defaultValue, newMap); 66 | } 67 | 68 | public LookupTable mapValues(Function valueMapper) { 69 | Map newMap = map.entrySet().stream() 70 | .collect(toMap( 71 | Map.Entry::getKey, 72 | entry -> mapOneValue(propertyName + "[" + entry.getKey() + "]", entry.getValue(), valueMapper) 73 | )); 74 | 75 | return new LookupTable<>( 76 | propertyName, 77 | mapOneValue(propertyName, defaultValue, valueMapper), 78 | newMap); 79 | } 80 | 81 | private static R mapOneValue(String propertyNameWithContext, V value, Function valueMapper) { 82 | try { 83 | return valueMapper.apply(value); 84 | } catch (RuntimeException e) { 85 | throw new ConfigException(propertyNameWithContext, value, e.toString()); 86 | } 87 | } 88 | 89 | @Override 90 | public String toString() { 91 | return "LookupTable{" + 92 | "defaultValue=" + defaultValue + 93 | ", map=" + map + 94 | '}'; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /examples/custom-extensions/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 20 | 21 | 4.0.0 22 | 23 | jar 24 | 25 | com.couchbase.client.kafka.example 26 | custom-extensions 27 | 1.0-SNAPSHOT 28 | 29 | 30 | 1.8 31 | UTF-8 32 | 33 | 4.3.2 34 | 2.8.2 35 | 1.7.36 36 | 37 | 38 | 39 | 40 | 41 | sonatype-snapshots 42 | https://oss.sonatype.org/content/repositories/snapshots 43 | false 44 | true 45 | 46 | 47 | 48 | 49 | 50 | org.apache.kafka 51 | connect-json 52 | ${kafka.version} 53 | provided 54 | 55 | 56 | com.couchbase.client 57 | kafka-connect-couchbase 58 | ${kafka-connect-couchbase.version} 59 | provided 60 | 61 | 62 | org.apache.kafka 63 | connect-api 64 | ${kafka.version} 65 | provided 66 | 67 | 68 | org.slf4j 69 | slf4j-api 70 | ${slf4j.version} 71 | provided 72 | 73 | 74 | 75 | 76 | 77 | 78 | org.apache.maven.plugins 79 | maven-compiler-plugin 80 | 3.8.0 81 | 82 | ${java-compat.version} 83 | ${java-compat.version} 84 | 85 | 86 | 87 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/handler/source/DefaultSchemaSourceHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.handler.source; 18 | 19 | import com.couchbase.connect.kafka.util.Schemas; 20 | import org.apache.kafka.connect.data.Struct; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | /** 25 | * The standard handler. Publishes metadata along with document content. 26 | * 27 | * @see Schemas 28 | * @see RawJsonSourceHandler 29 | * @see RawJsonWithMetadataSourceHandler 30 | */ 31 | public class DefaultSchemaSourceHandler implements SourceHandler { 32 | 33 | private static final Logger LOGGER = LoggerFactory.getLogger(DefaultSchemaSourceHandler.class); 34 | 35 | @Override 36 | public SourceRecordBuilder handle(SourceHandlerParams params) { 37 | SourceRecordBuilder builder = new SourceRecordBuilder(); 38 | 39 | // A handler may choose to route the message to any topic. 40 | // The code shown here sends the message to the topic from the connector configuration. 41 | // This is optional; if no topic is specified, it defaults to the one from the config. 42 | builder.topic(params.topic()); 43 | 44 | buildKey(params, builder); 45 | 46 | if (!buildValue(params, builder)) { 47 | // Don't know how to handle this message; skip it! 48 | // A custom handler may filter the event stream by returning null to skip a message. 49 | return null; 50 | } 51 | 52 | return builder; 53 | } 54 | 55 | protected void buildKey(SourceHandlerParams params, SourceRecordBuilder builder) { 56 | builder.key(Schemas.KEY_SCHEMA, params.documentEvent().key()); 57 | } 58 | 59 | /** 60 | * @return true to publish the message, or false to skip it 61 | */ 62 | protected boolean buildValue(SourceHandlerParams params, SourceRecordBuilder builder) { 63 | final DocumentEvent docEvent = params.documentEvent(); 64 | final DocumentEvent.Type type = docEvent.type(); 65 | 66 | final Struct record = new Struct(Schemas.VALUE_DEFAULT_SCHEMA); 67 | record.put("event", type.schemaName()); 68 | 69 | record.put("bucket", docEvent.bucket()); 70 | record.put("partition", docEvent.partition()); 71 | record.put("vBucketUuid", docEvent.partitionUuid()); 72 | record.put("key", docEvent.key()); 73 | record.put("cas", docEvent.cas()); 74 | record.put("bySeqno", docEvent.bySeqno()); 75 | record.put("revSeqno", docEvent.revisionSeqno()); 76 | 77 | final MutationMetadata mutation = docEvent.mutationMetadata().orElse(null); 78 | if (mutation != null) { 79 | record.put("expiration", mutation.expiry()); 80 | record.put("flags", mutation.flags()); 81 | record.put("lockTime", mutation.lockTime()); 82 | record.put("content", docEvent.content()); 83 | 84 | } else if (type != DocumentEvent.Type.DELETION && type != DocumentEvent.Type.EXPIRATION) { 85 | LOGGER.warn("unexpected event type: {}", type); 86 | return false; 87 | } 88 | 89 | builder.value(Schemas.VALUE_DEFAULT_SCHEMA, record); 90 | return true; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/util/BatchBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util; 18 | 19 | import com.couchbase.connect.kafka.handler.sink.ConcurrencyHint; 20 | 21 | import java.util.ArrayList; 22 | import java.util.HashSet; 23 | import java.util.List; 24 | import java.util.Set; 25 | 26 | import static java.util.Collections.singletonList; 27 | 28 | /** 29 | * Builds a list of batches where items within a single batch 30 | * may be executed concurrently, but batches themselves must be 31 | * executed sequentially. 32 | *

33 | * Batch boundaries are determined by inspecting the provided concurrency hints. 34 | * A resulting batch will contain no duplicate hint values (except for 35 | * {@link ConcurrencyHint#alwaysConcurrent()} 36 | * which may appear any number of times in a single batch). 37 | *

38 | * Each instance of {@link ConcurrencyHint#neverConcurrent()} results in 39 | * a batch containing only the single item associated with that hint. 40 | * 41 | * @param batch element 42 | */ 43 | public class BatchBuilder { 44 | private final List> batches = new ArrayList<>(); 45 | private final Set hintsInCurrentBatch = new HashSet<>(); 46 | private List currentBatch = null; 47 | 48 | /** 49 | * Inspects the concurrency hint to see if a new batch must be started, 50 | * then adds the item to the current batch. 51 | * 52 | * @param item item to add to a batch 53 | * @param hint determines batch boundaries 54 | * @return this builder 55 | */ 56 | public BatchBuilder add(T item, ConcurrencyHint hint) { 57 | if (hint == ConcurrencyHint.neverConcurrent()) { 58 | // Special case so we don't waste tons of space 59 | // if all of the items are "never concurrent" 60 | doAddSingleton(item); 61 | return this; 62 | } 63 | 64 | if (hint != ConcurrencyHint.alwaysConcurrent() && !hintsInCurrentBatch.add(hint)) { 65 | insertBarrier(); 66 | hintsInCurrentBatch.add(hint); 67 | } 68 | 69 | doAdd(item); 70 | return this; 71 | } 72 | 73 | /** 74 | * Returns the list of batches of items from previous calls to {@link #add(Object, ConcurrencyHint)}. 75 | */ 76 | public List> build() { 77 | return batches; 78 | } 79 | 80 | private void insertBarrier() { 81 | currentBatch = null; 82 | hintsInCurrentBatch.clear(); 83 | } 84 | 85 | private void doAdd(T item) { 86 | if (currentBatch == null) { 87 | currentBatch = new ArrayList<>(); 88 | batches.add(currentBatch); 89 | } 90 | currentBatch.add(item); 91 | } 92 | 93 | private void doAddSingleton(T item) { 94 | batches.add(singletonList(item)); 95 | insertBarrier(); 96 | } 97 | 98 | @Override 99 | public String toString() { 100 | return "BatchBuilder{" + 101 | "hintsInCurrentBatch=" + hintsInCurrentBatch + 102 | ", batches=" + batches + 103 | ", currentBatch=" + currentBatch + 104 | '}'; 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /config/migrate-config-3-to-4.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # This shell script modifies a kafka-connect-couchbase version 3.x config file 4 | # (in either *.properties or *.json format) and renames the properties 5 | # so the config file may be used with version 4.x of the connector. 6 | 7 | set -e 8 | set -u 9 | 10 | BACKUP_SUFFIX=.backup 11 | 12 | if [ "$#" -ne 1 ]; then 13 | echo "This script takes one argument: the path of the config file to migrate." 14 | echo "The config file will be modified in place." 15 | echo "A backup will be created with filename suffix '$BACKUP_SUFFIX'." 16 | exit 1 17 | fi 18 | 19 | INPUT_FILE="$1" 20 | 21 | case $INPUT_FILE in *.json | *.properties) true ;; *) 22 | echo "ERROR: Expected filename to end with .json or .properties" 23 | exit 1 24 | esac 25 | 26 | BACKUP_FILE="$INPUT_FILE$BACKUP_SUFFIX" 27 | 28 | if [ -f "$BACKUP_FILE" ]; then 29 | echo "ERROR: Backup file '$BACKUP_FILE' already exists." 30 | exit 1 31 | fi 32 | 33 | sed -i "$BACKUP_SUFFIX" \ 34 | -e "s/connection.cluster_address/couchbase.seed.nodes/g" \ 35 | -e "s/couchbase.network/couchbase.network/g" \ 36 | -e "s/connection.bucket/couchbase.bucket/g" \ 37 | -e "s/connection.username/couchbase.username/g" \ 38 | -e "s/connection.password/couchbase.password/g" \ 39 | -e "s/connection.timeout.ms/couchbase.bootstrap.timeout/g" \ 40 | -e "s/connection.ssl.enabled/couchbase.enable.tls/g" \ 41 | -e "s/connection.ssl.keystore.password/couchbase.trust.store.password/g" \ 42 | -e "s/connection.ssl.keystore.location/couchbase.trust.store.path/g" \ 43 | -e "s/couchbase.log_redaction/couchbase.log.redaction/g" \ 44 | -e "s/topic.name/couchbase.topic/g" \ 45 | -e "s/dcp.message.converter.class/couchbase.source.handler/g" \ 46 | -e "s/event.filter.class/couchbase.event.filter/g" \ 47 | -e "s/batch.size.max/couchbase.batch.size.max/g" \ 48 | -e "s/compat.connector_name_in_offsets/couchbase.connector.name.in.offsets/g" \ 49 | -e "s/couchbase.stream_from/couchbase.stream.from/g" \ 50 | -e "s/couchbase.log_redaction/couchbase.log.redaction/g" \ 51 | -e "s/couchbase.compression/couchbase.compression/g" \ 52 | -e "s/couchbase.persistence_polling_interval/couchbase.persistence.polling.interval/g" \ 53 | -e "s/couchbase.flow_control_buffer/couchbase.flow.control.buffer/g" \ 54 | -e "s/couchbase.document.id/couchbase.document.id/g" \ 55 | -e "s/couchbase.remove.document.id/couchbase.remove.document.id/g" \ 56 | -e "s/couchbase.durability.persist_to/couchbase.persist.to/g" \ 57 | -e "s/couchbase.durability.replicate_to/couchbase.replicate.to/g" \ 58 | -e "s/couchbase.subdocument.path/couchbase.subdocument.path/g" \ 59 | -e "s/couchbase.document.mode/couchbase.document.mode/g" \ 60 | -e "s/couchbase.subdocument.operation/couchbase.subdocument.operation/g" \ 61 | -e "s/couchbase.n1ql.operation/couchbase.n1ql.operation/g" \ 62 | -e "s/couchbase.n1ql.where_fields/couchbase.n1ql.where.fields/g" \ 63 | -e "s/couchbase.subdocument.create_path/couchbase.subdocument.create.path/g" \ 64 | -e "s/couchbase.subdocument.create_document/couchbase.create.document/g" \ 65 | -e "s/couchbase.document.expiration/couchbase.document.expiration/g" \ 66 | "$INPUT_FILE" 67 | 68 | echo "The original config was backed up to '$BACKUP_FILE'" 69 | echo "SUCCESS: The connector config file '$INPUT_FILE' has been migrated successfully." 70 | 71 | for OBSOLETE_PROPERTY in "use_snapshots" "couchbase.forceIPv4" 72 | do 73 | if grep -q "$OBSOLETE_PROPERTY" "$INPUT_FILE"; then 74 | echo "*** Please manually remove the obsolete '$OBSOLETE_PROPERTY' property." 75 | fi 76 | done 77 | 78 | if grep -q couchbase.bootstrap.timeout "$INPUT_FILE"; then 79 | echo "*** Please manually update the 'couchbase.bootstrap.timeout' property and append 'ms' to the value to indicate the value is in milliseconds." 80 | fi 81 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/config/source/SchemaConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.config.source; 18 | 19 | import com.couchbase.client.core.annotation.Stability; 20 | import com.couchbase.connect.kafka.CouchbaseSourceTask; 21 | import com.couchbase.connect.kafka.StreamFrom; 22 | import com.couchbase.connect.kafka.filter.Filter; 23 | import com.couchbase.connect.kafka.handler.source.CouchbaseHeaderSetter; 24 | import com.couchbase.connect.kafka.util.TopicMap; 25 | import com.couchbase.connect.kafka.util.config.Contextual; 26 | import com.couchbase.connect.kafka.util.config.annotation.ContextDocumentation; 27 | import com.couchbase.connect.kafka.util.config.annotation.Default; 28 | import org.apache.kafka.common.config.ConfigDef; 29 | 30 | import java.util.List; 31 | 32 | import static com.couchbase.connect.kafka.util.config.ConfigHelper.validate; 33 | 34 | public interface SchemaConfig { 35 | /** 36 | * The schema to apply to the value of each record. 37 | *

38 | * This property will be ignored unless the source handler specified by `couchbase.source.handler` supports it. 39 | *

40 | * The built-in `com.couchbase.connect.kafka.handler.source.ConfigurableSchemaSourceHandler` supports this property. 41 | * ConfigurableSchemaSourceHandler expects the schema to be an Avro Schema Record, stored in a JSON object. Reference: https://avro.apache.org/docs/1.12.0/specification/ 42 | *

43 | * ConfigurableSchemaSourceHandler will filter out documents that do not have a JSON Object at the root, and any that do not match the specified Schema.. 44 | */ 45 | @Stability.Uncommitted 46 | @Default 47 | @ContextDocumentation( 48 | contextDescription = "the name of the collection to apply the schema to, qualified by scope", 49 | sampleContext = "myScope.myCollection", 50 | sampleValue = "{\"type\": \"typeName\", ...attributes...}" 51 | ) 52 | Contextual valueSchema(); 53 | 54 | /** 55 | * This property will be ignored unless the source handler specified by `couchbase.source.handler` supports it. 56 | * The built-in 'com.couchbase.connect.kafka.handler.source.SchemaRegistrySourceHandler' supports this property. 57 | *

58 | * If set to a non-empty string, Documents that do not match the registered schema for a topic will be sent to the specified schema. 59 | *

60 | * If left empty, Documents that do not match the registered schema for a topic will cause the Connector Task to fail. 61 | */ 62 | @Stability.Uncommitted 63 | @Default 64 | String schemaMismatchTopic(); 65 | 66 | /** 67 | * This property will be ignored unless the source handler specified by `couchbase.source.handler` supports it. 68 | * The built-in 'com.couchbase.connect.kafka.handler.source.SchemaRegistrySourceHandler' supports this property. 69 | *

70 | * If set to a non-empty string, Documents sent to a topic with no registered schema will be sent to the specified missing schema topic 71 | *

72 | * If left empty, Documents sent to a topic with no registered schema will cause the Connector Task to fail. 73 | */ 74 | @Stability.Uncommitted 75 | @Default 76 | String missingSchemaTopic(); 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/handler/source/CouchbaseHeaderSetter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.handler.source; 18 | 19 | import com.couchbase.client.core.annotation.Stability; 20 | import org.apache.kafka.connect.data.Schema; 21 | import org.apache.kafka.connect.data.SchemaAndValue; 22 | import org.apache.kafka.connect.header.Headers; 23 | 24 | import java.util.Collection; 25 | import java.util.LinkedHashMap; 26 | import java.util.LinkedHashSet; 27 | import java.util.Map; 28 | import java.util.Set; 29 | import java.util.function.Function; 30 | 31 | import static java.util.Collections.unmodifiableMap; 32 | 33 | @Stability.Internal 34 | public final class CouchbaseHeaderSetter { 35 | private static final Map> template; 36 | 37 | static { 38 | Map> map = new LinkedHashMap<>(); 39 | map.put("bucket", event -> string(event.bucket())); 40 | map.put("scope", event -> string(event.collectionMetadata().scopeName())); 41 | map.put("collection", event -> string(event.collectionMetadata().collectionName())); 42 | map.put("key", event -> string(event.key())); 43 | map.put("qualifiedKey", event -> string(event.qualifiedKey())); 44 | map.put("cas", event -> int64(event.cas())); 45 | map.put("partition", event -> int32(event.partition())); 46 | map.put("partitionUuid", event -> int64(event.partitionUuid())); 47 | map.put("seqno", event -> int64(event.bySeqno())); 48 | map.put("rev", event -> int64(event.revisionSeqno())); 49 | map.put("expiry", event -> { 50 | MutationMetadata md = event.mutationMetadata().orElse(null); 51 | return md == null || md.expiry() == 0 52 | ? null 53 | : int64(md.expiry()); 54 | } 55 | ); 56 | template = unmodifiableMap(map); 57 | } 58 | 59 | private final Map> headerNameToValueAccessor; 60 | 61 | public static Set validHeaders() { 62 | return template.keySet(); 63 | } 64 | 65 | public CouchbaseHeaderSetter(String prefix, Collection headerNames) { 66 | Set invalidHeaderNames = new LinkedHashSet<>(headerNames); 67 | invalidHeaderNames.removeAll(validHeaders()); 68 | if (!invalidHeaderNames.isEmpty()) { 69 | throw new IllegalArgumentException("Invalid header names: " + invalidHeaderNames + " ; each header name must be one of " + validHeaders()); 70 | } 71 | 72 | Map> map = new LinkedHashMap<>(); 73 | headerNames.forEach(name -> map.put(prefix + name, template.get(name))); 74 | headerNameToValueAccessor = unmodifiableMap(map); 75 | } 76 | 77 | public void setHeaders(Headers headers, DocumentEvent event) { 78 | headerNameToValueAccessor.forEach((name, accessor) -> headers.add(name, accessor.apply(event))); 79 | } 80 | 81 | private static SchemaAndValue string(String value) { 82 | return new SchemaAndValue(Schema.STRING_SCHEMA, value); 83 | } 84 | 85 | private static SchemaAndValue int32(int value) { 86 | return new SchemaAndValue(Schema.INT32_SCHEMA, value); 87 | } 88 | 89 | private static SchemaAndValue int64(long value) { 90 | return new SchemaAndValue(Schema.INT64_SCHEMA, value); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/test/java/com/couchbase/connect/kafka/util/KafkaRetryHelperTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.util; 18 | 19 | import com.couchbase.client.core.error.CouchbaseException; 20 | import org.apache.kafka.connect.errors.RetriableException; 21 | import org.junit.jupiter.api.AfterEach; 22 | import org.junit.jupiter.api.BeforeEach; 23 | import org.junit.jupiter.api.Test; 24 | 25 | import java.time.Duration; 26 | import java.util.concurrent.atomic.AtomicBoolean; 27 | 28 | import static org.junit.jupiter.api.Assertions.assertThrows; 29 | import static org.junit.jupiter.api.Assertions.assertTrue; 30 | 31 | public class KafkaRetryHelperTest { 32 | private TestClock clock; 33 | private KafkaRetryHelper retryHelper; 34 | 35 | @BeforeEach 36 | public void setup() { 37 | clock = new TestClock(); 38 | retryHelper = new KafkaRetryHelper("test", Duration.ofSeconds(5), clock); 39 | } 40 | 41 | @AfterEach 42 | public void after() { 43 | retryHelper.close(); 44 | } 45 | 46 | private static class TestClock implements KafkaRetryHelper.Clock { 47 | public long now = Long.MAX_VALUE; 48 | 49 | @Override 50 | public long nanoTime() { 51 | return now; 52 | } 53 | 54 | public void advance(Duration d) { 55 | now += d.toNanos(); 56 | } 57 | } 58 | 59 | private static void throwCouchbaseException() { 60 | throw new CouchbaseException("oops!"); 61 | } 62 | 63 | private void assertRetriable() { 64 | assertThrows(RetriableException.class, () -> 65 | retryHelper.runWithRetry(KafkaRetryHelperTest::throwCouchbaseException)); 66 | } 67 | 68 | private void assertNotRetriable() { 69 | assertThrows(CouchbaseException.class, () -> 70 | retryHelper.runWithRetry(KafkaRetryHelperTest::throwCouchbaseException)); 71 | } 72 | 73 | private void assertSuccess() { 74 | AtomicBoolean b = new AtomicBoolean(); 75 | retryHelper.runWithRetry(() -> b.set(true)); 76 | assertTrue(b::get); 77 | } 78 | 79 | @Test 80 | public void eventuallyTimesOut() { 81 | assertRetriable(); 82 | 83 | clock.advance(Duration.ofSeconds(1)); 84 | assertRetriable(); 85 | 86 | clock.advance(Duration.ofSeconds(4)); 87 | assertNotRetriable(); 88 | } 89 | 90 | @Test 91 | public void successResetsRetryStartTime() { 92 | assertRetriable(); 93 | 94 | clock.advance(Duration.ofSeconds(1)); 95 | assertRetriable(); 96 | 97 | assertSuccess(); 98 | 99 | clock.advance(Duration.ofSeconds(4)); 100 | assertRetriable(); 101 | 102 | clock.advance(Duration.ofSeconds(1)); 103 | assertRetriable(); 104 | 105 | clock.advance(Duration.ofSeconds(4)); 106 | assertNotRetriable(); 107 | } 108 | 109 | @Test 110 | public void canSucceedImmediately() { 111 | assertSuccess(); 112 | clock.advance(Duration.ofDays(1)); 113 | assertSuccess(); 114 | clock.advance(Duration.ofDays(1)); 115 | assertSuccess(); 116 | } 117 | 118 | @Test 119 | public void zeroRetryDurationMeansNoRetry() { 120 | try (KafkaRetryHelper retryHelper = new KafkaRetryHelper("test", Duration.ZERO, clock)) { 121 | assertThrows(CouchbaseException.class, () -> 122 | retryHelper.runWithRetry(KafkaRetryHelperTest::throwCouchbaseException)); 123 | } 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/handler/sink/ConcurrencyHint.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.handler.sink; 18 | 19 | import java.util.HashSet; 20 | import java.util.Objects; 21 | 22 | import static java.util.Objects.requireNonNull; 23 | 24 | /** 25 | * Used for concurrency control. Determines which actions may 26 | * be performed concurrently. 27 | *

28 | * If two actions have concurrency hints with equal values, 29 | * the first action must complete before the second is started. 30 | *

31 | * Example: Using document IDs as hint values is a way to express 32 | * the constraint that different documents may be updated in parallel, 33 | * but updates to the same document must be applied sequentially. 34 | * 35 | * @see #of(Object) 36 | * @see #alwaysConcurrent() 37 | * @see #neverConcurrent() 38 | */ 39 | public class ConcurrencyHint { 40 | /** 41 | * Indicates the action may be performed concurrently with any other action 42 | * whose hint value is NOT EQUAL to this one. 43 | * 44 | * @param value the value that must be unique withing a batch of concurrent operations. 45 | * Must be the sort of thing you can safely insert in a {@link HashSet}. 46 | */ 47 | public static ConcurrencyHint of(Object value) { 48 | return new ConcurrencyHint(value); 49 | } 50 | 51 | /** 52 | * Returns a special hint indicating the action may be performed concurrently 53 | * with any other action, regardless of the other action's concurrency hint. 54 | */ 55 | public static ConcurrencyHint alwaysConcurrent() { 56 | return ALWAYS_CONCURRENT; 57 | } 58 | 59 | /** 60 | * Returns a special hint indicating the action must never be performed concurrently 61 | * with any other action, regardless of the other action's concurrency hint. 62 | *

63 | * In other words, all prior actions will be completed before this action is performed, 64 | * and this action will be completed before any subsequent actions are performed. 65 | *

66 | * Takes precedence over {@link #alwaysConcurrent()}. 67 | */ 68 | public static ConcurrencyHint neverConcurrent() { 69 | return NEVER_CONCURRENT; 70 | } 71 | 72 | private final Object value; 73 | 74 | private static final ConcurrencyHint NEVER_CONCURRENT = new ConcurrencyHint(new Object()) { 75 | @Override 76 | public String toString() { 77 | return "ConcurrencyHint.NEVER_CONCURRENT"; 78 | } 79 | }; 80 | 81 | private static final ConcurrencyHint ALWAYS_CONCURRENT = new ConcurrencyHint(new Object()) { 82 | @Override 83 | public String toString() { 84 | return "ConcurrencyHint.ALWAYS_CONCURRENT"; 85 | } 86 | }; 87 | 88 | private ConcurrencyHint(Object value) { 89 | this.value = requireNonNull(value); 90 | } 91 | 92 | @Override 93 | public boolean equals(Object o) { 94 | if (this == o) return true; 95 | if (o == null || getClass() != o.getClass()) return false; 96 | ConcurrencyHint that = (ConcurrencyHint) o; 97 | return value.equals(that.value); 98 | } 99 | 100 | @Override 101 | public int hashCode() { 102 | return Objects.hash(value); 103 | } 104 | 105 | @Override 106 | public String toString() { 107 | return "ConcurrencyHint{" + 108 | "value=" + value + 109 | '}'; 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/config/common/SecurityConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.config.common; 18 | 19 | import com.couchbase.connect.kafka.util.config.annotation.Default; 20 | import com.couchbase.connect.kafka.util.config.annotation.Dependents; 21 | import com.couchbase.connect.kafka.util.config.annotation.DisplayName; 22 | import com.couchbase.connect.kafka.util.config.annotation.EnvironmentVariable; 23 | import com.couchbase.connect.kafka.util.config.annotation.Width; 24 | import org.apache.kafka.common.config.types.Password; 25 | 26 | import static org.apache.kafka.common.config.ConfigDef.Width.LONG; 27 | 28 | public interface SecurityConfig { 29 | /** 30 | * Use secure connection to Couchbase Server. 31 | *

32 | * If true, you must also tell the connector which certificate to trust. 33 | * Specify a certificate file with 'couchbase.trust.certificate.path', 34 | * or a Java keystore file with 'couchbase.trust.store.path' and 35 | * 'couchbase.trust.store.password'. 36 | */ 37 | @Dependents({ 38 | "couchbase.trust.certificate.path", 39 | "couchbase.trust.store.path", 40 | "couchbase.trust.store.password", 41 | "couchbase.enable.hostname.verification", 42 | "couchbase.client.certificate.path", 43 | "couchbase.client.certificate.password", 44 | }) 45 | @Default("false") 46 | @DisplayName("Enable TLS") 47 | boolean enableTls(); 48 | 49 | /** 50 | * Set this to `false` to disable TLS hostname verification for Couchbase 51 | * connections. Less secure, but might be required if for some reason you 52 | * can't issue a certificate whose Subject Alternative Names match the 53 | * hostname used to connect to the server. Only disable if you understand 54 | * the impact and can accept the risks. 55 | */ 56 | @Default("true") 57 | @DisplayName("Enable TLS Hostname Verification") 58 | boolean enableHostnameVerification(); 59 | 60 | /** 61 | * Absolute filesystem path to the Java keystore holding the CA certificate 62 | * used by Couchbase Server. 63 | *

64 | * If you want to use a PEM file instead of a Java keystore, 65 | * specify `couchbase.trust.certificate.path` instead. 66 | */ 67 | @Width(LONG) 68 | @Default 69 | String trustStorePath(); 70 | 71 | /** 72 | * Password for accessing the trust store. 73 | */ 74 | @EnvironmentVariable("KAFKA_COUCHBASE_TRUST_STORE_PASSWORD") 75 | @Default 76 | Password trustStorePassword(); 77 | 78 | /** 79 | * Absolute filesystem path to the PEM file containing the 80 | * CA certificate used by Couchbase Server. 81 | *

82 | * If you want to use a Java keystore instead of a PEM file, 83 | * specify `couchbase.trust.store.path` instead. 84 | */ 85 | @Width(LONG) 86 | @Default 87 | String trustCertificatePath(); 88 | 89 | /** 90 | * Absolute filesystem path to a Java keystore or PKCS12 bundle holding 91 | * the private key and certificate chain to use for client certificate 92 | * authentication (mutual TLS). 93 | *

94 | * If you supply a value for this config property, the `couchbase.username` 95 | * and `couchbase.password` properties will be ignored. 96 | */ 97 | @Width(LONG) 98 | @Default 99 | String clientCertificatePath(); 100 | 101 | /** 102 | * Password for accessing the client certificate. 103 | */ 104 | @EnvironmentVariable("KAFKA_COUCHBASE_CLIENT_CERTIFICATE_PASSWORD") 105 | @Default 106 | Password clientCertificatePassword(); 107 | } 108 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/handler/source/MultiSourceHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.handler.source; 18 | 19 | import com.couchbase.client.core.annotation.Stability; 20 | import com.couchbase.connect.kafka.config.common.LoggingConfig; 21 | import org.apache.kafka.clients.producer.RecordMetadata; 22 | import org.apache.kafka.connect.source.SourceRecord; 23 | import org.apache.kafka.connect.source.SourceTask; 24 | import org.jspecify.annotations.NullMarked; 25 | import org.jspecify.annotations.Nullable; 26 | 27 | import java.util.List; 28 | import java.util.Map; 29 | 30 | import static java.util.Collections.emptyList; 31 | import static java.util.Collections.singletonList; 32 | 33 | /** 34 | * An alternative to {@link SourceHandler} that supports generating multiple Kafka records 35 | * from the same Couchbase event. 36 | *

37 | * WARNING: If your handler's {@link #convertToSourceRecords} method ever returns multiple records, 38 | * the Kafka Connect worker SHOULD be configured with {@code exactly.once.source.support=enabled}. 39 | * Otherwise, the connector can only guarantee at least one of the related messages is delivered. 40 | *

41 | * Enabling {@code exactly.once.source.support} requires Apache Kafka 3.3.0 or later. For more details, see 42 | * 43 | * KIP-618: Exactly-Once Support for Source Connectors 44 | * . 45 | */ 46 | @NullMarked 47 | @Stability.Uncommitted 48 | public interface MultiSourceHandler { 49 | /** 50 | * Called one time when the filter is instantiated. 51 | * 52 | * @param configProperties the connector configuration. 53 | */ 54 | default void init(Map configProperties) { 55 | } 56 | 57 | /** 58 | * Translates a DocumentEvent (and associated parameters) into zero or more records 59 | * for publication to Kafka. 60 | *

61 | * The document event is specified by the SourceHandlerParams parameter block, along with 62 | * other bits of info that may be useful to custom implementations. 63 | *

64 | * The handler may route a message to an arbitrary topic by setting the {@code topic} 65 | * property of the returned builder, or may leave it null to use the default topic 66 | * from the connector configuration. 67 | *

68 | * The handler may filter the event stream by returning an empty list to skip this event. 69 | *

70 | * WARNING: See the {@link MultiSourceHandler} Javadoc for an important warning 71 | * about how returning multiple records can affect delivery guarantees. 72 | * 73 | * @param params A parameter block containing input to the handler, 74 | * most notably the {@link DocumentEvent}. 75 | * @return The records to publish to Kafka, or an empty list to skip this event. The list must not contain nulls. 76 | */ 77 | List convertToSourceRecords(SourceHandlerParams params); 78 | 79 | /** 80 | * Called whenever the Kafka Connect framework calls {@link SourceTask#commitRecord(SourceRecord, RecordMetadata)}. 81 | *

82 | * Subclasses may override this method to do something special with committed record metadata 83 | * beyond the usual document lifecycle logging. 84 | *

85 | * This method should return quickly. 86 | *

87 | * The default implementation does nothing, which is always okay. 88 | * 89 | * @see LoggingConfig#logDocumentLifecycle() 90 | */ 91 | @Stability.Volatile 92 | default void onRecordCommitted(SourceRecord record, @Nullable RecordMetadata metadata) { 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /examples/json-producer/src/main/java/com/couchbase/connect/kafka/example/JsonProducerExample.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.example; 18 | 19 | import com.fasterxml.jackson.databind.ObjectMapper; 20 | import com.fasterxml.jackson.databind.node.ObjectNode; 21 | import org.apache.kafka.clients.producer.KafkaProducer; 22 | import org.apache.kafka.clients.producer.Producer; 23 | import org.apache.kafka.clients.producer.ProducerConfig; 24 | import org.apache.kafka.clients.producer.ProducerRecord; 25 | import org.apache.kafka.clients.producer.RecordMetadata; 26 | import org.apache.kafka.common.serialization.ByteArraySerializer; 27 | import org.apache.kafka.common.serialization.StringSerializer; 28 | 29 | import java.util.Arrays; 30 | import java.util.List; 31 | import java.util.Properties; 32 | import java.util.Random; 33 | import java.util.UUID; 34 | 35 | import static java.util.Collections.unmodifiableList; 36 | 37 | public class JsonProducerExample { 38 | private static final String BOOTSTRAP_SERVERS = "localhost:9092"; 39 | private static final String TOPIC = "couchbase-sink-example"; 40 | 41 | private static final ObjectMapper objectMapper = new ObjectMapper(); 42 | private static final Random random = new Random(); 43 | 44 | public static void main(String[] args) throws Exception { 45 | Producer producer = createProducer(); 46 | try { 47 | for (int i = 0; i < 20; i++) { 48 | publishMessage(producer); 49 | Thread.sleep(random.nextInt(500)); 50 | } 51 | } finally { 52 | producer.close(); 53 | } 54 | } 55 | 56 | private static Producer createProducer() { 57 | Properties config = new Properties(); 58 | config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); 59 | config.put(ProducerConfig.CLIENT_ID_CONFIG, "CouchbaseJsonProducerExample"); 60 | config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 61 | config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); 62 | return new KafkaProducer(config); 63 | } 64 | 65 | private static void publishMessage(Producer producer) throws Exception { 66 | // Try setting the key to null and see how the Couchbase Sink Connector behaves. 67 | // For extra fun, try configuring the Couchbase Sink Connector with the property: 68 | // couchbase.document.id=/airport 69 | String key = UUID.randomUUID().toString(); 70 | 71 | ObjectNode weatherReport = randomWeatherReport(); 72 | byte[] valueJson = objectMapper.writeValueAsBytes(weatherReport); 73 | 74 | ProducerRecord record = new ProducerRecord(TOPIC, key, valueJson); 75 | 76 | RecordMetadata md = producer.send(record).get(); 77 | System.out.println("Published " + md.topic() + "/" + md.partition() + "/" + md.offset() 78 | + " (key=" + key + ") : " + weatherReport); 79 | } 80 | 81 | private static final List airports = unmodifiableList(Arrays.asList( 82 | "SFO", "YVR", "LHR", "CDG", "TXL", "VCE", "DME", "DEL", "BJS")); 83 | 84 | private static ObjectNode randomWeatherReport() { 85 | // In a real app you might want to take advantage of Jackson's data binding features. 86 | // Since Jackson is not the focus of this example, let's just build the JSON manually. 87 | ObjectNode report = objectMapper.createObjectNode(); 88 | report.put("airport", airports.get(random.nextInt(airports.size()))); 89 | report.put("degreesF", 70 + (int) (random.nextGaussian() * 20)); 90 | report.put("timestamp", System.currentTimeMillis()); 91 | return report; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/handler/sink/SinkHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.handler.sink; 18 | 19 | import com.couchbase.client.core.annotation.Stability; 20 | import org.apache.kafka.connect.sink.SinkRecord; 21 | 22 | import java.nio.ByteBuffer; 23 | import java.util.ArrayList; 24 | import java.util.List; 25 | 26 | import static java.nio.charset.StandardCharsets.UTF_8; 27 | 28 | /** 29 | * Primary extension point for customizing how the Sink Connector handles messages from Kafka. 30 | */ 31 | public interface SinkHandler { 32 | /** 33 | * Called one time when the filter is instantiated. 34 | * 35 | * @param context provides access to the Couchbase cluster and the connector configuration. 36 | */ 37 | default void init(SinkHandlerContext context) { 38 | } 39 | 40 | /** 41 | * Translates a Kafka Connect {@link SinkRecord} (and associated parameters) 42 | * into an action to perform on the record. 43 | * 44 | * @return (nullable) The action to perform. Return null or {@link SinkAction#ignore()} 45 | * to skip this message. 46 | */ 47 | SinkAction handle(SinkHandlerParams params); 48 | 49 | /** 50 | * Translates List of Kafka Connect {@link SinkRecord} (and associated parameters) 51 | * into List of actions to perform. 52 | */ 53 | default List handleBatch(List params) { 54 | List actions = new ArrayList<>(params.size()); 55 | for (SinkHandlerParams param : params) { 56 | SinkAction action = handle(param); 57 | if (action != null) { 58 | actions.add(action); 59 | } 60 | } 61 | return actions; 62 | } 63 | 64 | /** 65 | * Returns the Couchbase document ID to use for the record 66 | * associated with the given params. 67 | * 68 | * @implNote The default implementation first looks for a document ID 69 | * extracted from the message contents according to the `couchbase.document.id` 70 | * config property. If that fails, it derives the key from the Kafka record 71 | * metadata by calling {@link #getDocumentIdFromKafkaMetadata}. 72 | */ 73 | default String getDocumentId(SinkHandlerParams params) { 74 | return params.document() 75 | .flatMap(SinkDocument::id) 76 | .orElseGet(() -> getDocumentIdFromKafkaMetadata(params.sinkRecord())); 77 | } 78 | 79 | /** 80 | * Returns a document ID derived from the sink record metadata. 81 | * 82 | * @implNote The default implementation first tries to coerce the 83 | * record's key to a String. If that fails, a synthetic key is constructed 84 | * from the record's topic, partition, and offset. 85 | */ 86 | default String getDocumentIdFromKafkaMetadata(SinkRecord record) { 87 | Object key = record.key(); 88 | 89 | if (key instanceof String 90 | || key instanceof Number 91 | || key instanceof Boolean) { 92 | return key.toString(); 93 | } 94 | 95 | if (key instanceof byte[]) { 96 | return new String((byte[]) key, UTF_8); 97 | } 98 | 99 | if (key instanceof ByteBuffer) { 100 | return UTF_8.decode((ByteBuffer) key).toString(); 101 | } 102 | 103 | return record.topic() + "/" + record.kafkaPartition() + "/" + record.kafkaOffset(); 104 | } 105 | 106 | /** 107 | * Returns true if this handler is going to call {@link SinkHandlerParams#collection()}. 108 | *

109 | * Called once, after {@link #init(SinkHandlerContext)}. 110 | *

111 | * This lets the connector avoid opening non-existent KV collections 112 | * when {@link AnalyticsSinkHandler} is used. 113 | */ 114 | @Stability.Internal 115 | default boolean usesKvCollections() { 116 | return true; 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /src/test/java/com/couchbase/connect/kafka/CouchbaseSinkTaskTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka; 18 | 19 | import com.couchbase.connect.kafka.handler.sink.ConcurrencyHint; 20 | import com.couchbase.connect.kafka.handler.sink.SinkAction; 21 | import reactor.core.publisher.Mono; 22 | import reactor.core.publisher.Sinks; 23 | 24 | import java.time.Duration; 25 | import java.util.ArrayList; 26 | import java.util.List; 27 | import java.util.concurrent.CopyOnWriteArrayList; 28 | import java.util.concurrent.atomic.AtomicLong; 29 | 30 | import static com.couchbase.client.core.util.CbCollections.listOf; 31 | import static org.junit.jupiter.api.Assertions.assertEquals; 32 | 33 | import org.junit.jupiter.api.Test; 34 | 35 | public class CouchbaseSinkTaskTest { 36 | 37 | // inspired by https://nitschinger.at/Reactive-Barriers-with-Reactor/ 38 | private static class ReactiveCountDownLatch { 39 | private final Sinks.One inner = Sinks.one(); 40 | private final AtomicLong ready = new AtomicLong(); 41 | private final int needed; 42 | 43 | public ReactiveCountDownLatch(int count) { 44 | if (count < 0) { 45 | throw new IllegalArgumentException("count must be non-negative, but got " + count); 46 | } 47 | this.needed = count; 48 | if (this.needed == 0) { 49 | inner.emitValue(null, Sinks.EmitFailureHandler.FAIL_FAST); 50 | } 51 | } 52 | 53 | public void countDown() { 54 | if (ready.incrementAndGet() >= needed) { 55 | inner.emitValue(null, Sinks.EmitFailureHandler.FAIL_FAST); 56 | } 57 | } 58 | 59 | public Mono await() { 60 | return inner.asMono(); 61 | } 62 | } 63 | 64 | /** 65 | * Expect actions within a batch to be executed concurrently, 66 | * and batches to be executed sequentially. 67 | */ 68 | @Test 69 | public void executionConcurrency() throws Exception { 70 | CopyOnWriteArrayList results = new CopyOnWriteArrayList<>(); 71 | 72 | List actions = new ArrayList<>(); 73 | List expectedResults = new ArrayList<>(); 74 | 75 | for (int i = 0; i < 3; i++) { 76 | // This is the reverse of the action submission order; 77 | // the latch ensures the results appear in the expected order. 78 | expectedResults.addAll(listOf("a", "b")); 79 | 80 | // The latch introduces an artificial dependency between the actions 81 | // to verify they are running concurrently. If they're not, 82 | // there's a deadlock and the test times out. 83 | ReactiveCountDownLatch latch = new ReactiveCountDownLatch(1); 84 | 85 | // The delay causes the test to fail if the executor failed 86 | // to wait for each batch to complete before starting the next. 87 | // 88 | // There's probably a better way to test this without relying on the 89 | // system clock, but I can't figure out how to get StepVerifier 90 | // to work in this context :-/ 91 | Mono delay = Mono.delay(Duration.ofSeconds(1)); 92 | 93 | actions.addAll(listOf( 94 | new SinkAction( 95 | delay 96 | .then(latch.await()) // wait for "a" 97 | .then(Mono.fromRunnable(() -> results.add("b"))), 98 | ConcurrencyHint.of("b")), 99 | 100 | new SinkAction( 101 | delay 102 | .then(Mono.fromRunnable(() -> results.add("a"))) 103 | .then(Mono.fromRunnable(latch::countDown)), 104 | ConcurrencyHint.of("a")) 105 | )); 106 | } 107 | 108 | CouchbaseSinkTask.toMono(actions) 109 | .block(Duration.ofSeconds(30)); 110 | 111 | assertEquals(expectedResults, results); 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connect/kafka/config/sink/N1qlSinkHandlerConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connect.kafka.config.sink; 18 | 19 | import com.couchbase.connect.kafka.handler.sink.N1qlSinkHandler; 20 | import com.couchbase.connect.kafka.util.config.annotation.Default; 21 | 22 | import java.util.List; 23 | 24 | /** 25 | * Config properties used only by {@link N1qlSinkHandler}. 26 | */ 27 | public interface N1qlSinkHandlerConfig { 28 | /** 29 | * The type of update to use when `couchbase.sink.handler` is set to 30 | * `com.couchbase.connect.kafka.handler.sink.N1qlSinkHandler`. 31 | *

32 | * This property is specific to `N1qlSinkHandler`. 33 | */ 34 | @Default("UPDATE") 35 | Operation n1qlOperation(); 36 | 37 | /** 38 | * When using the UPDATE_WHERE operation, this is the list of document fields that must match the Kafka message in order for the document to be updated with the remaining message fields. 39 | * To match against a literal value instead of a message field, use a colon to delimit the document field name and the target value. 40 | * For example, "type:widget,color" matches documents whose 'type' field is 'widget' and whose 'color' field matches the 'color' field of the Kafka message. 41 | *

42 | * This property is specific to `N1qlSinkHandler`. 43 | */ 44 | @Default 45 | List n1qlWhereFields(); 46 | 47 | /** 48 | * Controls whether to create the document if it does not exist. 49 | *

50 | * This property is specific to `N1qlSinkHandler`. 51 | */ 52 | @Default("true") 53 | boolean n1qlCreateDocument(); 54 | 55 | enum Operation { 56 | /** 57 | * Copy all top-level fields of the Kafka message to the target document, 58 | * clobbering any existing top-level fields with the same names. 59 | */ 60 | UPDATE, 61 | 62 | /** 63 | * Target zero or more documents using a WHERE condition to match fields of the message. 64 | *

65 | * Consider the following 3 documents: 66 | * 

 67 |      * {
 68 |      *   "id": "airline_1",
 69 |      *   "type": "airline",
 70 |      *   "name": "airline 1",
 71 |      *   "parent_companycode": "AA",
 72 |      *   "parent_companyname": "airline inc"
 73 |      * }
 74 |      * {
 75 |      *   "id": "airline_2",
 76 |      *   "type": "airline",
 77 |      *   "name": "airline 2",
 78 |      *   "parent_companycode": "AA",
 79 |      *   "parent_companyname": "airline inc"
 80 |      * }
 81 |      * {
 82 |      *   "id": "airline_3",
 83 |      *   "type": "airline",
 84 |      *   "name": "airline 3",
 85 |      *   "parent_companycode": "AA",
 86 |      *   "parent_companyname": "airline inc"
 87 |      * }
 88 |      *
89 | * With the UPDATE mode, it would take 3 Kafka messages to update the 90 | * "parent_companyname" of all the documents, each message using a 91 | * different ID to target one of the 3 documents. 92 | *

93 | * With the UPDATE_WHERE mode, you can send one message and match it on something 94 | * other than the id. For example, if you configure the plugin like this: 95 | * 

 96 |      * ...
 97 |      * "couchbase.document.mode":"N1QL",
 98 |      * "couchbase.n1ql.operation":"UPDATE_WHERE",
 99 |      * "couchbase.n1ql.where_fields":"type:airline,parent_companycode"
100 |      * ...
101 |      *
102 | * the connector will generate an update statement with a WHERE clause 103 | * instead of ON KEYS. For example, when receiving this Kafka message: 104 | * 
105 |      * {
106 |      *   "type":"airline",
107 |      *   "parent_companycode":"AA",
108 |      *   "parent_companyname":"airline ltd"
109 |      * }
110 |      *
111 | * it would generate a N1QL statement along the lines of: 112 | * 
113 |      *   UPDATE `keyspace`
114 |      *   SET `parent_companyname` = $parent_companyname
115 |      *   WHERE `type` = $type AND `parent_companycode` = $parent_companycode
116 |      *   RETURNING meta().id
117 |      *
118 | */ 119 | UPDATE_WHERE, 120 | } 121 | } 122 | --------------------------------------------------------------------------------