├── src ├── test │ ├── resources │ │ └── log4j2-test.properties │ └── java │ │ └── io │ │ └── aiven │ │ └── kafka │ │ └── connect │ │ ├── transforms │ │ ├── ExtractTopicKeyTest.java │ │ ├── ExtractTopicValueTest.java │ │ ├── ExtractTimestampKeyTest.java │ │ ├── ExtractTimestampValueTest.java │ │ ├── MakeTombstoneTest.java │ │ ├── HashKeyTest.java │ │ ├── HashValueTest.java │ │ ├── ConcatFieldsKeyTest.java │ │ ├── ConcatFieldsValueTest.java │ │ ├── ExtractTopicConfigTest.java │ │ ├── TombstoneHandlerTest.java │ │ ├── utils │ │ │ └── HexTest.java │ │ ├── ConcatFieldsConfigTest.java │ │ ├── TombstoneHandlerConfigTest.java │ │ ├── HashConfigTest.java │ │ ├── ExtractTopicFromSchemaNameConfigTest.java │ │ ├── ExtractTimestampConfigTest.java │ │ ├── ExtractTopicFromSchemaNameTest.java │ │ └── FilterByFieldValueTest.java │ │ └── debezium │ │ └── converters │ │ ├── utils │ │ ├── DummyRelationalColumn.java │ │ └── MoneyTestRelationalColumn.java │ │ └── MoneyConverterTest.java ├── main │ ├── resources │ │ └── transforms-for-apache-kafka-connect-version.properties │ └── java │ │ └── io │ │ └── aiven │ │ └── kafka │ │ └── connect │ │ ├── transforms │ │ ├── utils │ │ │ └── Hex.java │ │ ├── MakeTombstone.java │ │ ├── ExtractTopicConfig.java │ │ ├── KeyToValueConfig.java │ │ ├── TombstoneHandler.java │ │ ├── ConcatFieldsConfig.java │ │ ├── CaseTransformConfig.java │ │ ├── TombstoneHandlerConfig.java │ │ ├── ExtractTopicFromSchemaName.java │ │ ├── HashConfig.java │ │ ├── ExtractTopicFromSchemaNameConfig.java │ │ ├── ExtractTimestampConfig.java │ │ ├── ExtractTimestamp.java │ │ ├── ConcatFields.java │ │ ├── FilterByFieldValue.java │ │ ├── CaseTransform.java │ │ ├── Hash.java │ │ └── ExtractTopic.java │ │ └── debezium │ │ └── converters │ │ └── MoneyConverter.java └── integration-test │ ├── resources │ └── log4j2-test.properties │ └── java │ └── io │ └── aiven │ └── kafka │ └── connect │ └── transforms │ ├── AbstractTestSourceConnector.java │ ├── TestSourceConnector.java │ ├── TestCaseTransformConnector.java │ ├── TestKeyToValueConnector.java │ ├── TopicFromValueSchemaConnector.java │ └── ConnectRunner.java ├── gradle.properties ├── settings.gradle ├── .github ├── CODEOWNERS ├── dependabot.yml └── workflows │ ├── master_push_and_pull_request_workflow.yml │ ├── release_pr_workflow.yml │ ├── codeql.yml │ └── create_release.yml ├── .gitignore ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── demo ├── Dockerfile ├── Makefile ├── compose.yml └── README.md ├── config └── checkstyle │ ├── java.header │ └── suppressions.xml ├── SECURITY.md ├── gradlew.bat ├── CONTRIBUTING.md ├── CODE_OF_CONDUCT.md └── gradlew /src/test/resources/log4j2-test.properties: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | version=1.7.0-SNAPSHOT 2 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'transforms-for-apache-kafka-connect' 2 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @aiven-open/team-helpful-husky @aiven-open/aiven-open-source 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | 3 | # Ignore Gradle project-specific cache directory 4 | .gradle 5 | 6 | # Ignore Gradle build output directory 7 | build 8 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aiven-Open/transforms-for-apache-kafka-connect/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gradle" 4 | directory: "/" 5 | schedule: 6 | interval: "monthly" 7 | -------------------------------------------------------------------------------- /demo/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM eclipse-temurin:11-jdk AS base 2 | 3 | ADD ./ transforms 4 | WORKDIR transforms 5 | 6 | RUN ./gradlew installDist 7 | 8 | FROM confluentinc/cp-kafka-connect:7.3.3 9 | 10 | COPY --from=base /transforms/build/install/transforms-for-apache-kafka-connect /transforms 11 | -------------------------------------------------------------------------------- /demo/Makefile: -------------------------------------------------------------------------------- 1 | all: build 2 | 3 | build: 4 | docker compose build 5 | 6 | up: 7 | docker compose up -d 8 | 9 | test: 10 | curl -sSL 'http://localhost:8083/connector-plugins?connectorsOnly=false' \ 11 | | jq '.[] | select(.type=="transformation") | select (.class | startswith("io.aiven"))' 12 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-all.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionSha256Sum=e6d864e3b5bc05cc62041842b306383fc1fefcec359e70cebb1d470a6094ca82 7 | -------------------------------------------------------------------------------- /config/checkstyle/java.header: -------------------------------------------------------------------------------- 1 | /\* 2 | \* Copyright 20(19|2[0-9]) Aiven Oy 3 | \* 4 | \* Licensed under the Apache License, Version 2.0 \(the "License"\); 5 | \* you may not use this file except in compliance with the License. 6 | \* You may obtain a copy of the License at 7 | \* 8 | \* http://www.apache.org/licenses/LICENSE-2.0 9 | \* 10 | \* Unless required by applicable law or agreed to in writing, software 11 | \* distributed under the License is distributed on an "AS IS" BASIS, 12 | \* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | \* See the License for the specific language governing permissions and 14 | \* limitations under the License. 15 | \*/ 16 | -------------------------------------------------------------------------------- /src/main/resources/transforms-for-apache-kafka-connect-version.properties: -------------------------------------------------------------------------------- 1 | ## 2 | # Copyright 2019 Aiven Oy 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | ## 16 | 17 | version=${version ?: 'unknown'} 18 | -------------------------------------------------------------------------------- /src/integration-test/resources/log4j2-test.properties: -------------------------------------------------------------------------------- 1 | status = error 2 | dest = err 3 | name = PropertiesConfig 4 | 5 | filter.threshold.type = ThresholdFilter 6 | filter.threshold.level = debug 7 | 8 | appender.console.type = Console 9 | appender.console.name = STDOUT 10 | appender.console.layout.type = PatternLayout 11 | appender.console.layout.pattern = [%d] %p %m (%c:%L)%n 12 | appender.console.filter.threshold.type = ThresholdFilter 13 | appender.console.filter.threshold.level = info 14 | 15 | rootLogger.level = info 16 | rootLogger.appenderRef.stdout.ref = STDOUT 17 | 18 | logger.org_apache_zookeeper_level.name = org.apache.zookeeper 19 | logger.org_apache_zookeeper_level.level = error 20 | logger.org_reflections.name = org.reflections 21 | logger.org_reflections.level = error 22 | -------------------------------------------------------------------------------- /.github/workflows/master_push_and_pull_request_workflow.yml: -------------------------------------------------------------------------------- 1 | # The workflow to check master after push or pull request. 2 | name: Master and pull request checks 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | jobs: 9 | build: 10 | strategy: 11 | matrix: 12 | java-version: [ 11, 17 ] 13 | runs-on: [ ubuntu-latest ] 14 | name: Build on ${{ matrix.runs-on }} with jdk ${{ matrix.java-version }} 15 | runs-on: ${{ matrix.runs-on }} 16 | steps: 17 | - name: Checkout code 18 | uses: actions/checkout@v4 19 | - name: Set up JDK ${{ matrix.java-version }} 20 | uses: actions/setup-java@v4 21 | with: 22 | java-version: ${{ matrix.java-version }} 23 | distribution: 'temurin' 24 | cache: gradle 25 | - name: Build with Gradle 26 | run: ./gradlew build integrationTest 27 | -------------------------------------------------------------------------------- /config/checkstyle/suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 20 | 21 | 23 | 25 | 27 | 28 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/ExtractTopicKeyTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import org.apache.kafka.connect.data.Schema; 20 | import org.apache.kafka.connect.sink.SinkRecord; 21 | 22 | public class ExtractTopicKeyTest extends ExtractTopicTest { 23 | 24 | @Override 25 | protected String dataPlace() { 26 | return "key"; 27 | } 28 | 29 | @Override 30 | protected ExtractTopic createTransformationObject() { 31 | return new ExtractTopic.Key<>(); 32 | } 33 | 34 | @Override 35 | protected SinkRecord record(final Schema schema, final Object data) { 36 | return record(schema, data, null, null); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/ExtractTopicValueTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import org.apache.kafka.connect.data.Schema; 20 | import org.apache.kafka.connect.sink.SinkRecord; 21 | 22 | public class ExtractTopicValueTest extends ExtractTopicTest { 23 | 24 | @Override 25 | protected String dataPlace() { 26 | return "value"; 27 | } 28 | 29 | @Override 30 | protected ExtractTopic createTransformationObject() { 31 | return new ExtractTopic.Value<>(); 32 | } 33 | 34 | @Override 35 | protected SinkRecord record(final Schema schema, final Object data) { 36 | return record(null, null, schema, data); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/ExtractTimestampKeyTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import org.apache.kafka.connect.data.Schema; 20 | import org.apache.kafka.connect.sink.SinkRecord; 21 | 22 | public class ExtractTimestampKeyTest extends ExtractTimestampTest { 23 | 24 | @Override 25 | protected String keyOrValue() { 26 | return "key"; 27 | } 28 | 29 | @Override 30 | protected ExtractTimestamp createTransformationObject() { 31 | return new ExtractTimestamp.Key<>(); 32 | } 33 | 34 | @Override 35 | protected SinkRecord record(final Schema schema, final Object data) { 36 | return record(schema, data, null, null); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/ExtractTimestampValueTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import org.apache.kafka.connect.data.Schema; 20 | import org.apache.kafka.connect.sink.SinkRecord; 21 | 22 | public class ExtractTimestampValueTest extends ExtractTimestampTest { 23 | 24 | @Override 25 | protected String keyOrValue() { 26 | return "value"; 27 | } 28 | 29 | @Override 30 | protected ExtractTimestamp createTransformationObject() { 31 | return new ExtractTimestamp.Value<>(); 32 | } 33 | 34 | @Override 35 | protected SinkRecord record(final Schema schema, final Object data) { 36 | return record(null, null, schema, data); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/transforms/utils/Hex.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms.utils; 18 | 19 | public class Hex { 20 | private static final char[] HEX_ARRAY = "0123456789abcdef".toCharArray(); 21 | 22 | /** 23 | * Encodes a byte array as a hexadecimal string. 24 | * @implNote https://stackoverflow.com/a/9855338/1781549 25 | */ 26 | public static String encode(final byte[] bytes) { 27 | final char[] hexChars = new char[bytes.length * 2]; 28 | for (int j = 0; j < bytes.length; j++) { 29 | final int v = bytes[j] & 0xFF; 30 | hexChars[j * 2] = HEX_ARRAY[v >>> 4]; // hi nibble 31 | hexChars[j * 2 + 1] = HEX_ARRAY[v & 0x0F]; // lo nibble 32 | } 33 | return new String(hexChars); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/MakeTombstoneTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import org.apache.kafka.connect.data.Schema; 20 | import org.apache.kafka.connect.source.SourceRecord; 21 | 22 | import org.junit.jupiter.api.Test; 23 | 24 | import static org.junit.jupiter.api.Assertions.assertNull; 25 | 26 | class MakeTombstoneTest { 27 | 28 | @Test 29 | void shouldMakeTombstone() { 30 | final MakeTombstone makeTombstone = new MakeTombstone<>(); 31 | final var record = new SourceRecord(null, null, "some_topic", Schema.STRING_SCHEMA, "dummy value"); 32 | 33 | final var actual = makeTombstone.apply(record); 34 | assertNull(actual.valueSchema(), "value schema should be null"); 35 | assertNull(actual.value(), "value should be null"); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | We release patches for security vulnerabilities. Which versions are eligible 6 | receiving such patches depend on the CVSS v3.0 Rating: 7 | 8 | | CVSS v3.0 | Supported Versions | 9 | | --------- | ----------------------------------------- | 10 | | 4.0-10.0 | Most recent release | 11 | 12 | ## Reporting a Vulnerability 13 | 14 | Please report (suspected) security vulnerabilities to our **[bug bounty 15 | program](https://bugcrowd.com/aiven-mbb-og)**. You will receive a response from 16 | us within 2 working days. If the issue is confirmed, we will release a patch as 17 | soon as possible depending on impact and complexity. 18 | 19 | ## Qualifying Vulnerabilities 20 | 21 | Any reproducible vulnerability that has a severe effect on the security or 22 | privacy of our users is likely to be in scope for the program. 23 | 24 | We generally **aren't** interested in the following issues: 25 | * Social engineering (e.g. phishing, vishing, smishing) attacks 26 | * Brute force, DoS, text injection 27 | * Missing best practices such as HTTP security headers (CSP, X-XSS, etc.), 28 | email (SPF/DKIM/DMARC records), SSL/TLS configuration. 29 | * Software version disclosure / Banner identification issues / Descriptive 30 | error messages or headers (e.g. stack traces, application or server errors). 31 | * Clickjacking on pages with no sensitive actions 32 | * Theoretical vulnerabilities where you can't demonstrate a significant 33 | security impact with a proof of concept. 34 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/HashKeyTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import org.apache.kafka.connect.data.Schema; 20 | import org.apache.kafka.connect.sink.SinkRecord; 21 | 22 | public class HashKeyTest extends HashTest { 23 | 24 | @Override 25 | protected String dataPlace() { 26 | return "key"; 27 | } 28 | 29 | @Override 30 | protected Hash createTransformationObject() { 31 | return new Hash.Key<>(); 32 | } 33 | 34 | @Override 35 | protected SinkRecord record(final Schema schema, final Object data) { 36 | return record(schema, data, null, null); 37 | } 38 | 39 | @Override 40 | protected SinkRecord setNewValue(final SinkRecord record, final Object newValue) { 41 | return record.newRecord(record.topic(), 42 | record.kafkaPartition(), 43 | record.keySchema(), 44 | newValue, 45 | record.valueSchema(), 46 | record.value(), 47 | record.timestamp(), 48 | record.headers() 49 | ); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/transforms/MakeTombstone.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Map; 20 | 21 | import org.apache.kafka.common.config.ConfigDef; 22 | import org.apache.kafka.connect.connector.ConnectRecord; 23 | import org.apache.kafka.connect.transforms.Transformation; 24 | 25 | public class MakeTombstone> implements Transformation { 26 | 27 | @Override 28 | public R apply(final R record) { 29 | return record.newRecord( 30 | record.topic(), 31 | record.kafkaPartition(), 32 | record.keySchema(), 33 | record.key(), 34 | null, 35 | null, 36 | record.timestamp(), 37 | record.headers() 38 | ); 39 | } 40 | 41 | @Override 42 | public ConfigDef config() { 43 | return new ConfigDef(); 44 | } 45 | 46 | @Override 47 | public void close() { 48 | 49 | } 50 | 51 | @Override 52 | public void configure(final Map configs) { 53 | 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/HashValueTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import org.apache.kafka.connect.data.Schema; 20 | import org.apache.kafka.connect.sink.SinkRecord; 21 | 22 | public class HashValueTest extends HashTest { 23 | 24 | @Override 25 | protected String dataPlace() { 26 | return "value"; 27 | } 28 | 29 | @Override 30 | protected Hash createTransformationObject() { 31 | return new Hash.Value<>(); 32 | } 33 | 34 | @Override 35 | protected SinkRecord record(final Schema schema, final Object data) { 36 | return record(null, null, schema, data); 37 | } 38 | 39 | @Override 40 | protected SinkRecord setNewValue(final SinkRecord record, final Object newValue) { 41 | return record.newRecord(record.topic(), 42 | record.kafkaPartition(), 43 | record.keySchema(), 44 | record.key(), 45 | record.valueSchema(), 46 | newValue, 47 | record.timestamp(), 48 | record.headers() 49 | ); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /demo/compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | services: 3 | zookeeper: 4 | image: "confluentinc/cp-zookeeper:7.3.3" 5 | ports: 6 | - "2181:2181" 7 | environment: 8 | ZOOKEEPER_CLIENT_PORT: 2181 9 | 10 | kafka: 11 | image: "confluentinc/cp-kafka:7.3.3" 12 | ports: 13 | - "9092:9092" 14 | environment: 15 | KAFKA_BROKER_ID: 1 16 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 17 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT 18 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:19092,PLAINTEXT_HOST://localhost:9092 19 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 20 | KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 21 | KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 22 | 23 | connect: 24 | build: 25 | context: ../. 26 | dockerfile: demo/Dockerfile 27 | ports: 28 | - "8083:8083" 29 | environment: 30 | CONNECT_BOOTSTRAP_SERVERS: kafka:19092 31 | CONNECT_REST_ADVERTISED_HOST_NAME: localhost 32 | CONNECT_REST_PORT: 8083 33 | CONNECT_GROUP_ID: connect-demo 34 | CONNECT_CONFIG_STORAGE_TOPIC: connect-demo-configs 35 | CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 36 | CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 37 | CONNECT_OFFSET_STORAGE_TOPIC: connect-demo-offsets 38 | CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 39 | CONNECT_STATUS_STORAGE_TOPIC: connect-demo-status 40 | CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 41 | 42 | CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter 43 | CONNECT_KEY_CONVERTER_SCHEMA_ENABLE: "false" 44 | CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter 45 | CONNECT_PLUGIN_PATH: /usr/share/java,/transforms # /transforms added on Dockerfile build 46 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/ConcatFieldsKeyTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import org.apache.kafka.connect.data.Schema; 20 | import org.apache.kafka.connect.sink.SinkRecord; 21 | 22 | public class ConcatFieldsKeyTest extends ConcatFieldsTest { 23 | 24 | @Override 25 | protected String dataPlace() { 26 | return "key"; 27 | } 28 | 29 | @Override 30 | protected ConcatFields createTransformationObject() { 31 | return new ConcatFields.Key<>(); 32 | } 33 | 34 | @Override 35 | protected SinkRecord record(final Schema schema, final Object data) { 36 | return record(schema, data, null, null); 37 | } 38 | 39 | @Override 40 | protected SinkRecord setNewValue(final SinkRecord record, final Schema newValueSchema, final Object newValue) { 41 | return record.newRecord(record.topic(), 42 | record.kafkaPartition(), 43 | newValueSchema, 44 | newValue, 45 | record.valueSchema(), 46 | record.value(), 47 | record.timestamp(), 48 | record.headers() 49 | ); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/ConcatFieldsValueTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import org.apache.kafka.connect.data.Schema; 20 | import org.apache.kafka.connect.sink.SinkRecord; 21 | 22 | public class ConcatFieldsValueTest extends ConcatFieldsTest { 23 | 24 | @Override 25 | protected String dataPlace() { 26 | return "value"; 27 | } 28 | 29 | @Override 30 | protected ConcatFields createTransformationObject() { 31 | return new ConcatFields.Value<>(); 32 | } 33 | 34 | @Override 35 | protected SinkRecord record(final Schema schema, final Object data) { 36 | return record(null, null, schema, data); 37 | } 38 | 39 | @Override 40 | protected SinkRecord setNewValue(final SinkRecord record, final Schema newValueSchema, final Object newValue) { 41 | return record.newRecord(record.topic(), 42 | record.kafkaPartition(), 43 | record.keySchema(), 44 | record.key(), 45 | newValueSchema, 46 | newValue, 47 | record.timestamp(), 48 | record.headers() 49 | ); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/integration-test/java/io/aiven/kafka/connect/transforms/AbstractTestSourceConnector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Collections; 20 | import java.util.List; 21 | import java.util.Map; 22 | 23 | import org.apache.kafka.common.config.ConfigDef; 24 | import org.apache.kafka.connect.source.SourceConnector; 25 | 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | public abstract class AbstractTestSourceConnector extends SourceConnector { 30 | 31 | private static final Logger LOGGER = LoggerFactory.getLogger(AbstractTestSourceConnector.class); 32 | 33 | @Override 34 | public void start(final Map props) { 35 | // no-op 36 | } 37 | 38 | @Override 39 | public void stop() { 40 | // no-op 41 | } 42 | 43 | @Override 44 | public List> taskConfigs(final int maxTasks) { 45 | return Collections.singletonList(Collections.emptyMap()); 46 | } 47 | 48 | @Override 49 | public ConfigDef config() { 50 | return new ConfigDef(); 51 | } 52 | 53 | @Override 54 | public String version() { 55 | return null; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/debezium/converters/utils/DummyRelationalColumn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.debezium.converters.utils; 18 | 19 | import java.util.OptionalInt; 20 | 21 | import io.debezium.spi.converter.RelationalColumn; 22 | 23 | 24 | public class DummyRelationalColumn implements RelationalColumn { 25 | @Override 26 | public int jdbcType() { 27 | return 0; 28 | } 29 | 30 | @Override 31 | public int nativeType() { 32 | return 0; 33 | } 34 | 35 | @Override 36 | public String typeName() { 37 | return "dummy"; 38 | } 39 | 40 | @Override 41 | public String typeExpression() { 42 | return null; 43 | } 44 | 45 | @Override 46 | public OptionalInt length() { 47 | return null; 48 | } 49 | 50 | @Override 51 | public OptionalInt scale() { 52 | return null; 53 | } 54 | 55 | @Override 56 | public boolean isOptional() { 57 | return false; 58 | } 59 | 60 | @Override 61 | public Object defaultValue() { 62 | return null; 63 | } 64 | 65 | @Override 66 | public boolean hasDefaultValue() { 67 | return false; 68 | } 69 | 70 | @Override 71 | public String name() { 72 | return null; 73 | } 74 | 75 | @Override 76 | public String dataCollection() { 77 | return null; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/debezium/converters/utils/MoneyTestRelationalColumn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.debezium.converters.utils; 18 | 19 | import java.util.OptionalInt; 20 | 21 | import io.debezium.spi.converter.RelationalColumn; 22 | 23 | 24 | public class MoneyTestRelationalColumn implements RelationalColumn { 25 | 26 | public boolean isOptional = false; 27 | 28 | @Override 29 | public int jdbcType() { 30 | return 0; 31 | } 32 | 33 | @Override 34 | public int nativeType() { 35 | return 0; 36 | } 37 | 38 | @Override 39 | public String typeName() { 40 | return "money"; 41 | } 42 | 43 | @Override 44 | public String typeExpression() { 45 | return null; 46 | } 47 | 48 | @Override 49 | public OptionalInt length() { 50 | return null; 51 | } 52 | 53 | @Override 54 | public OptionalInt scale() { 55 | return null; 56 | } 57 | 58 | @Override 59 | public boolean isOptional() { 60 | return isOptional; 61 | } 62 | 63 | @Override 64 | public Object defaultValue() { 65 | return null; 66 | } 67 | 68 | @Override 69 | public boolean hasDefaultValue() { 70 | return false; 71 | } 72 | 73 | @Override 74 | public String name() { 75 | return null; 76 | } 77 | 78 | @Override 79 | public String dataCollection() { 80 | return null; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /.github/workflows/release_pr_workflow.yml: -------------------------------------------------------------------------------- 1 | # The workflow to create PRs with release commits. 2 | name: Create release PR 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | release_version: 7 | description: "Release version '0.1.2' (without 'v')" 8 | required: true 9 | snapshot_version: 10 | description: "Snapshot version '0.2.0-SNAPSHOT' (without 'v')" 11 | required: true 12 | 13 | jobs: 14 | create_release_pr: 15 | name: Create release PR (job) 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: Check versions 19 | run: | 20 | echo "Checking release version..." 21 | if echo ${{ github.event.inputs.release_version }} | grep --invert-match '^[0-9]\+\.[0-9]\+\.[0-9]\+$' > /dev/null; then 22 | echo "Release version is invalid" 23 | exit 1 24 | fi 25 | 26 | echo "Checking snapshot version..." 27 | if echo ${{ github.event.inputs.snapshot_version }} | grep --invert-match '^[0-9]\+\.[0-9]\+\.[0-9]\+-SNAPSHOT$' > /dev/null; then 28 | echo "Snapshot version is invalid" 29 | exit 1 30 | fi 31 | 32 | - name: Checkout master 33 | uses: actions/checkout@v4 34 | with: 35 | ref: master 36 | fetch-depth: 0 37 | 38 | - name: Create release commits 39 | run: | 40 | git config --local user.name "GitHub Action" 41 | git config --local user.email "action@github.com" 42 | sed -i -e "s/^version=.\+$/version=${{ github.event.inputs.release_version }}/g" gradle.properties 43 | git add gradle.properties 44 | git commit -m "Release version ${{ github.event.inputs.release_version }}" 45 | sed -i -e "s/^version=.\+$/version=${{ github.event.inputs.snapshot_version }}/g" gradle.properties 46 | git add gradle.properties 47 | git commit -m "Bump version to ${{ github.event.inputs.snapshot_version }}" 48 | 49 | - name: Create Pull Request 50 | uses: peter-evans/create-pull-request@v4 51 | with: 52 | branch: release-${{ github.event.inputs.release_version }} 53 | delete-branch: true 54 | draft: true 55 | title: Release version ${{ github.event.inputs.release_version }} 56 | body: | 57 | Proposed changelog: 58 | - *fill in* 59 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/ExtractTopicConfigTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | import org.junit.jupiter.api.Test; 23 | import org.junit.jupiter.params.ParameterizedTest; 24 | import org.junit.jupiter.params.provider.ValueSource; 25 | 26 | import static org.assertj.core.api.Assertions.assertThat; 27 | 28 | class ExtractTopicConfigTest { 29 | @Test 30 | void defaults() { 31 | final Map props = new HashMap<>(); 32 | final ExtractTopicConfig config = new ExtractTopicConfig(props); 33 | assertThat(config.fieldName()).isNotPresent(); 34 | assertThat(config.skipMissingOrNull()).isFalse(); 35 | } 36 | 37 | @ParameterizedTest 38 | @ValueSource(booleans = {true, false}) 39 | void skipMissingOrNull(final boolean skipMissingOrNull) { 40 | final Map props = new HashMap<>(); 41 | props.put("skip.missing.or.null", Boolean.toString(skipMissingOrNull)); 42 | final ExtractTopicConfig config = new ExtractTopicConfig(props); 43 | assertThat(config.skipMissingOrNull()).isEqualTo(skipMissingOrNull); 44 | } 45 | 46 | @Test 47 | void emptyFieldName() { 48 | final Map props = new HashMap<>(); 49 | props.put("field.name", ""); 50 | final ExtractTopicConfig config = new ExtractTopicConfig(props); 51 | assertThat(config.fieldName()).isNotPresent(); 52 | } 53 | 54 | @Test 55 | void definedFieldName() { 56 | final Map props = new HashMap<>(); 57 | props.put("field.name", "test"); 58 | final ExtractTopicConfig config = new ExtractTopicConfig(props); 59 | assertThat(config.fieldName()).hasValue("test"); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/TombstoneHandlerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Map; 20 | 21 | import org.apache.kafka.common.record.TimestampType; 22 | import org.apache.kafka.connect.errors.DataException; 23 | import org.apache.kafka.connect.sink.SinkRecord; 24 | 25 | import io.aiven.kafka.connect.transforms.TombstoneHandlerConfig.Behavior; 26 | 27 | import org.junit.jupiter.api.Test; 28 | import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; 29 | 30 | import static org.assertj.core.api.Assertions.assertThat; 31 | import static org.assertj.core.api.Assertions.assertThatThrownBy; 32 | 33 | final class TombstoneHandlerTest { 34 | 35 | @Test 36 | final void shouldDropTombstoneRecord() { 37 | assertThat(tombstoneHandler(Behavior.DROP_SILENT).apply(record(null))) 38 | .isNull(); 39 | } 40 | 41 | @Test 42 | final void shouldThrowDataAccessExceptionOnTombstoneRecords() { 43 | assertThatThrownBy(() -> tombstoneHandler(Behavior.FAIL).apply(record(null))) 44 | .isInstanceOf(DataException.class) 45 | .hasMessage("Tombstone record encountered, failing due to configured 'fail' behavior"); 46 | } 47 | 48 | private SinkRecord record(final Object value) { 49 | return new SinkRecord("some_topic", 0, 50 | null, null, 51 | null, value, 52 | 123L, 53 | 456L, TimestampType.CREATE_TIME); 54 | } 55 | 56 | private TombstoneHandler tombstoneHandler(final Behavior b) { 57 | final Map props = 58 | ImmutableMap.of( 59 | TombstoneHandlerConfig.TOMBSTONE_HANDLER_BEHAVIOR, 60 | b.name() 61 | ); 62 | final TombstoneHandler tombstoneHandler = new TombstoneHandler<>(); 63 | tombstoneHandler.configure(props); 64 | return tombstoneHandler; 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/debezium/converters/MoneyConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.debezium.converters; 18 | 19 | import java.math.BigDecimal; 20 | import java.util.Properties; 21 | 22 | import org.apache.kafka.connect.data.SchemaBuilder; 23 | 24 | import io.debezium.spi.converter.CustomConverter; 25 | import io.debezium.spi.converter.RelationalColumn; 26 | 27 | public class MoneyConverter implements CustomConverter { 28 | /** 29 | * Convert money type to a correct format. 30 | * Source: https://github.com/moyphilip/MoneyConverter 31 | */ 32 | private SchemaBuilder moneySchema; 33 | 34 | @Override 35 | public void configure(final Properties props) { 36 | moneySchema = SchemaBuilder.string().name(props.getProperty("schema.name")); 37 | } 38 | 39 | @Override 40 | public void converterFor(final RelationalColumn column, 41 | final CustomConverter.ConverterRegistration registration) { 42 | 43 | if ("money".equals(column.typeName())) { 44 | registration.register(moneySchema, data -> { 45 | if (data == null) { 46 | if (column.isOptional()) { 47 | return null; 48 | } else { 49 | throw new IllegalArgumentException("Money column is not optional, but data is null"); 50 | } 51 | } 52 | if (data instanceof BigDecimal) { 53 | // Expected type 54 | return String.format("%.2f", data); 55 | } else if (data instanceof Number) { 56 | return String.format("%.2f", ((Number) data).floatValue()); 57 | } else { 58 | throw new IllegalArgumentException("Money type should have BigDecimal type"); 59 | } 60 | 61 | }); 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/transforms/ExtractTopicConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Map; 20 | import java.util.Optional; 21 | 22 | import org.apache.kafka.common.config.AbstractConfig; 23 | import org.apache.kafka.common.config.ConfigDef; 24 | 25 | class ExtractTopicConfig extends AbstractConfig { 26 | public static final String FIELD_NAME_CONFIG = "field.name"; 27 | private static final String FIELD_NAME_DOC = 28 | "The name of the field which should be used as the topic name. " 29 | + "If null or empty, the entire key or value is used (and assumed to be a string)."; 30 | 31 | public static final String SKIP_MISSING_OR_NULL_CONFIG = "skip.missing.or.null"; 32 | private static final String SKIP_MISSING_OR_NULL_DOC = 33 | "In case the source of the new topic name is null or missing, " 34 | + "should a record be silently passed without transformation."; 35 | 36 | ExtractTopicConfig(final Map originals) { 37 | super(config(), originals); 38 | } 39 | 40 | static ConfigDef config() { 41 | return new ConfigDef() 42 | .define( 43 | FIELD_NAME_CONFIG, 44 | ConfigDef.Type.STRING, 45 | null, 46 | ConfigDef.Importance.HIGH, 47 | FIELD_NAME_DOC) 48 | .define( 49 | SKIP_MISSING_OR_NULL_CONFIG, 50 | ConfigDef.Type.BOOLEAN, 51 | false, 52 | ConfigDef.Importance.LOW, 53 | SKIP_MISSING_OR_NULL_DOC); 54 | } 55 | 56 | Optional fieldName() { 57 | final String rawFieldName = getString(FIELD_NAME_CONFIG); 58 | if (null == rawFieldName || "".equals(rawFieldName)) { 59 | return Optional.empty(); 60 | } 61 | return Optional.of(rawFieldName); 62 | } 63 | 64 | boolean skipMissingOrNull() { 65 | return getBoolean(SKIP_MISSING_OR_NULL_CONFIG); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/utils/HexTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms.utils; 18 | 19 | import java.io.IOException; 20 | import java.net.URISyntaxException; 21 | import java.net.URL; 22 | import java.nio.charset.Charset; 23 | import java.nio.file.Files; 24 | import java.nio.file.Paths; 25 | import java.util.List; 26 | 27 | import org.junit.jupiter.api.Test; 28 | 29 | import static org.assertj.core.api.Assertions.assertThat; 30 | 31 | public class HexTest { 32 | @Test 33 | void testEncodeEmpty() { 34 | final byte[] bytes = new byte[0]; 35 | assertThat(Hex.encode(bytes)).isEmpty(); 36 | } 37 | 38 | @Test 39 | void testEncodeSingleByte() { 40 | final byte[] bytes = new byte[1]; 41 | for (int i = 0; i < 256; i++) { 42 | final byte b = (byte) i; 43 | bytes[0] = b; 44 | assertThat(Hex.encode(bytes)).isEqualTo(String.format("%02x", b)); 45 | } 46 | } 47 | 48 | @Test 49 | void testEncodeFromStrings() throws IOException, URISyntaxException { 50 | final URL resource = getClass().getClassLoader().getResource("blns.txt"); 51 | final List strings = Files.readAllLines(Paths.get(resource.toURI())); 52 | for (final String s : strings) { 53 | // Use the string as a byte array and hex-encode it. 54 | final byte[] bytes = s.getBytes(Charset.defaultCharset()); 55 | final String encoded = Hex.encode(bytes); 56 | assertThat(encoded).hasSize(bytes.length * 2); 57 | 58 | // Decode the string back and compare to the original. 59 | final char[] encodedChars = encoded.toCharArray(); 60 | final byte[] decodedBytes = new byte[bytes.length]; 61 | for (int i = 0; i < encoded.length(); i += 2) { 62 | final String s1 = new String(encodedChars, i, 2); 63 | decodedBytes[i / 2] = (byte) Integer.parseInt(s1, 16); 64 | } 65 | assertThat(s).isEqualTo(new String(decodedBytes, Charset.defaultCharset())); 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/transforms/KeyToValueConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.List; 20 | import java.util.Map; 21 | 22 | import org.apache.kafka.common.config.AbstractConfig; 23 | import org.apache.kafka.common.config.ConfigDef; 24 | 25 | /** 26 | *

Configure the SMT to copy fields from the key into the value.

27 | * 28 | * @link org.apache.kafka.connect.transforms.ValueToKey transform for a similar approach 29 | */ 30 | public class KeyToValueConfig extends AbstractConfig { 31 | 32 | 33 | public static final String KEY_FIELDS_CONFIG = "key.fields"; 34 | public static final String VALUE_FIELDS_CONFIG = "value.fields"; 35 | /** 36 | * If the value (destination) column isn't specified and the wildcard '*' is used, this will be the name of the 37 | * column. 38 | */ 39 | public static final String DEFAULT_WHOLE_KEY_FIELD = "_key"; 40 | private static final String KEY_FIELDS_DOC = "Comma-separated list of field names on the record key to copy into " 41 | + "the record value (or * to copy the entire key)."; 42 | private static final String VALUE_FIELDS_DOC = "Corresponding destination field names to be added or replaced on " 43 | + "the record value, or empty if the same field name should be used as the key."; 44 | 45 | KeyToValueConfig(final Map originals) { 46 | super(config(), originals); 47 | } 48 | 49 | static ConfigDef config() { 50 | return new ConfigDef().define(KEY_FIELDS_CONFIG, 51 | ConfigDef.Type.LIST, 52 | ConfigDef.NO_DEFAULT_VALUE, 53 | ConfigDef.Importance.HIGH, 54 | KEY_FIELDS_DOC) 55 | .define(VALUE_FIELDS_CONFIG, 56 | ConfigDef.Type.LIST, 57 | "", 58 | ConfigDef.Importance.HIGH, 59 | VALUE_FIELDS_DOC); 60 | } 61 | 62 | final List keyFields() { 63 | return getList(KEY_FIELDS_CONFIG); 64 | } 65 | 66 | final List valueFields() { 67 | return getList(VALUE_FIELDS_CONFIG); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "master" ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ "master" ] 20 | schedule: 21 | - cron: '33 5 * * 6' 22 | 23 | permissions: 24 | actions: read 25 | contents: read 26 | security-events: write 27 | 28 | jobs: 29 | analyze: 30 | name: Analyze 31 | runs-on: ubuntu-latest 32 | 33 | strategy: 34 | fail-fast: false 35 | matrix: 36 | language: [ 'java' ] 37 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 38 | # Learn more: 39 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 40 | 41 | steps: 42 | - name: Checkout repository 43 | uses: actions/checkout@v4 44 | 45 | # Initializes the CodeQL tools for scanning. 46 | - name: Initialize CodeQL 47 | uses: github/codeql-action/init@v3 48 | with: 49 | languages: ${{ matrix.language }} 50 | # If you wish to specify custom queries, you can do so here or in a config file. 51 | # By default, queries listed here will override any specified in a config file. 52 | # Prefix the list here with "+" to use these queries and those in the config file. 53 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 54 | 55 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 56 | # If this step fails, then you should remove it and run the build manually (see below) 57 | #- name: Autobuild 58 | # uses: github/codeql-action/autobuild@v3 59 | 60 | # ℹ️ Command-line programs to run using the OS shell. 61 | # 📚 https://git.io/JvXDl 62 | 63 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 64 | # and modify them (or add more) to build your code if your project 65 | # uses a compiled language 66 | 67 | #- run: | 68 | # make bootstrap 69 | # make release 70 | - name: Set up JDK 11 71 | uses: actions/setup-java@v4 72 | with: 73 | distribution: 'temurin' 74 | java-version: 11 75 | 76 | - name: Build with Gradle 77 | run: ./gradlew build 78 | 79 | - name: Perform CodeQL Analysis 80 | uses: github/codeql-action/analyze@v3 81 | 82 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/ConcatFieldsConfigTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Arrays; 20 | import java.util.HashMap; 21 | import java.util.Map; 22 | 23 | import org.apache.kafka.common.config.ConfigException; 24 | 25 | import org.junit.jupiter.api.Test; 26 | 27 | import static io.aiven.kafka.connect.transforms.ConcatFieldsConfig.DELIMITER_CONFIG; 28 | import static io.aiven.kafka.connect.transforms.ConcatFieldsConfig.FIELD_NAMES_CONFIG; 29 | import static io.aiven.kafka.connect.transforms.ConcatFieldsConfig.FIELD_REPLACE_MISSING_CONFIG; 30 | import static io.aiven.kafka.connect.transforms.ConcatFieldsConfig.OUTPUT_FIELD_NAME_CONFIG; 31 | import static org.assertj.core.api.Assertions.assertThat; 32 | import static org.assertj.core.api.Assertions.assertThatThrownBy; 33 | 34 | class ConcatFieldsConfigTest { 35 | @Test 36 | void emptyConfig() { 37 | final Map props = new HashMap<>(); 38 | assertThatThrownBy(() -> new ConcatFieldsConfig(props)) 39 | .isInstanceOf(ConfigException.class) 40 | .hasMessage("Missing required configuration \"field.names\" which has no default value."); 41 | } 42 | 43 | @Test 44 | void emptyFieldName() { 45 | final Map props = new HashMap<>(); 46 | props.put(FIELD_NAMES_CONFIG, ""); 47 | assertThatThrownBy(() -> new ConcatFieldsConfig(props)) 48 | .isInstanceOf(ConfigException.class) 49 | .hasMessage("Missing required configuration \"output.field.name\" which has no default value."); 50 | } 51 | 52 | @Test 53 | void definedFieldName() { 54 | final Map props = new HashMap<>(); 55 | props.put(FIELD_NAMES_CONFIG, Arrays.asList("test", "foo", "bar")); 56 | props.put(OUTPUT_FIELD_NAME_CONFIG, "combined"); 57 | props.put(DELIMITER_CONFIG, "-"); 58 | props.put(FIELD_REPLACE_MISSING_CONFIG, "*"); 59 | final ConcatFieldsConfig config = new ConcatFieldsConfig(props); 60 | assertThat(config.fieldNames()).isEqualTo(Arrays.asList("test", "foo", "bar")); 61 | assertThat(config.outputFieldName()).isEqualTo("combined"); 62 | assertThat(config.delimiter()).isEqualTo("-"); 63 | assertThat(config.fieldReplaceMissing()).isEqualTo("*"); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /.github/workflows/create_release.yml: -------------------------------------------------------------------------------- 1 | name: Create release 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | commit_hash: 7 | description: "Hash of 'Release version x.y.z' commit" 8 | required: true 9 | 10 | jobs: 11 | build: 12 | name: Create Release 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Set up JDK 11 16 | uses: actions/setup-java@v4 17 | with: 18 | distribution: 'temurin' 19 | java-version: 11 20 | 21 | - name: Checkout code 22 | uses: actions/checkout@v4 23 | with: 24 | ref: ${{ github.event.inputs.commit_hash }} 25 | 26 | - name: Check commit title and extract version 27 | run: | 28 | export commit_title=$(git log --pretty=format:%s -1 ${{ github.event.inputs.commit_hash }}) 29 | echo "Commit title: $commit_title" 30 | if [[ $commit_title =~ ^Release\ version\ [0-9]*\.[0-9]*\.[0-9]*$ ]]; then 31 | echo "Valid commit title" 32 | else 33 | echo "Invalid commit title" 34 | exit 1 35 | fi 36 | export version=$(echo ${commit_title} | sed s/^Release\ version\ //g) 37 | echo "Will use version ${version}" 38 | echo "version=${version}" >> $GITHUB_ENV 39 | 40 | - name: Build 41 | run: | 42 | ./gradlew distTar distZip 43 | 44 | export tar_file=$(ls ./build/distributions/ | grep tar) 45 | export zip_file=$(ls ./build/distributions/ | grep zip) 46 | echo tar_file=${tar_file} >> $GITHUB_ENV 47 | echo zip_file=${zip_file} >> $GITHUB_ENV 48 | 49 | echo tar_path=`realpath ./build/distributions/${tar_file}` >> $GITHUB_ENV 50 | echo zip_path=`realpath ./build/distributions/${zip_file}` >> $GITHUB_ENV 51 | 52 | - name: Create release draft 53 | id: create_release 54 | uses: actions/create-release@v1 55 | env: 56 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 57 | with: 58 | tag_name: "v${{ env.version }}" 59 | release_name: "v${{ env.version }}" 60 | commitish: ${{ github.event.inputs.commit_hash }} 61 | body: | 62 | *Fill in* 63 | draft: true 64 | prerelease: false 65 | 66 | - name: Upload tar 67 | uses: actions/upload-release-asset@v1 68 | env: 69 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 70 | with: 71 | upload_url: ${{ steps.create_release.outputs.upload_url }} 72 | asset_path: ${{ env.tar_path }} 73 | asset_name: ${{ env.tar_file }} 74 | asset_content_type: application/tar 75 | 76 | - name: Upload zip 77 | uses: actions/upload-release-asset@v1 78 | env: 79 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 80 | with: 81 | upload_url: ${{ steps.create_release.outputs.upload_url }} 82 | asset_path: ${{ env.zip_path }} 83 | asset_name: ${{ env.zip_file }} 84 | asset_content_type: application/zip 85 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/TombstoneHandlerConfigTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Map; 20 | 21 | import org.apache.kafka.common.config.ConfigException; 22 | 23 | import org.junit.jupiter.api.Test; 24 | import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; 25 | 26 | import static org.assertj.core.api.Assertions.assertThat; 27 | import static org.assertj.core.api.Assertions.assertThatThrownBy; 28 | 29 | final class TombstoneHandlerConfigTest { 30 | 31 | @Test 32 | final void failOnUnknownBehaviorName() { 33 | assertThatThrownBy(() -> new TombstoneHandlerConfig(newBehaviorProps("asdasdsadas"))) 34 | .isInstanceOf(ConfigException.class) 35 | .hasMessage("Invalid value asdasdsadas for configuration behavior: " 36 | + "Unsupported behavior name: asdasdsadas. Supported are: drop_silent,drop_warn,fail"); 37 | } 38 | 39 | @Test 40 | final void acceptCorrectBehaviorNames() { 41 | 42 | TombstoneHandlerConfig c = 43 | new TombstoneHandlerConfig( 44 | newBehaviorProps( 45 | TombstoneHandlerConfig.Behavior.DROP_SILENT.name() 46 | ) 47 | ); 48 | assertThat(c.getBehavior()).isEqualTo(TombstoneHandlerConfig.Behavior.DROP_SILENT); 49 | 50 | c = 51 | new TombstoneHandlerConfig( 52 | newBehaviorProps( 53 | TombstoneHandlerConfig.Behavior.FAIL.name().toLowerCase() 54 | ) 55 | ); 56 | assertThat(c.getBehavior()).isEqualTo(TombstoneHandlerConfig.Behavior.FAIL); 57 | 58 | c = 59 | new TombstoneHandlerConfig( 60 | newBehaviorProps( 61 | "Drop_WArn" 62 | ) 63 | ); 64 | assertThat(c.getBehavior()).isEqualTo(TombstoneHandlerConfig.Behavior.DROP_WARN); 65 | } 66 | 67 | @Test 68 | final void failOnEmptyBehaviorName() { 69 | assertThatThrownBy(() -> new TombstoneHandlerConfig(newBehaviorProps(""))) 70 | .isInstanceOf(ConfigException.class) 71 | .hasMessage("Invalid value for configuration behavior: String must be non-empty"); 72 | } 73 | 74 | private Map newBehaviorProps(final String bv) { 75 | return ImmutableMap.of(TombstoneHandlerConfig.TOMBSTONE_HANDLER_BEHAVIOR, bv); 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /src/integration-test/java/io/aiven/kafka/connect/transforms/TestSourceConnector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Collections; 20 | import java.util.HashMap; 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | import org.apache.kafka.connect.connector.Task; 25 | import org.apache.kafka.connect.data.Schema; 26 | import org.apache.kafka.connect.data.SchemaBuilder; 27 | import org.apache.kafka.connect.data.Struct; 28 | import org.apache.kafka.connect.source.SourceRecord; 29 | import org.apache.kafka.connect.source.SourceTask; 30 | 31 | /** 32 | * A connector needed for testing of ExtractTopic. 33 | * 34 | *

It just produces a fixed number of struct records. 35 | */ 36 | public final class TestSourceConnector extends AbstractTestSourceConnector { 37 | static final long MESSAGES_TO_PRODUCE = 10L; 38 | 39 | static final String ORIGINAL_TOPIC = "original-topic"; 40 | static final String NEW_TOPIC = "new-topic"; 41 | static final String ROUTING_FIELD = "field-0"; 42 | 43 | @Override 44 | public Class taskClass() { 45 | return TestSourceConnectorTask.class; 46 | } 47 | 48 | public static class TestSourceConnectorTask extends SourceTask { 49 | private int counter = 0; 50 | 51 | private final Schema valueSchema = SchemaBuilder.struct() 52 | .field(ROUTING_FIELD, SchemaBuilder.STRING_SCHEMA) 53 | .schema(); 54 | private final Struct value = new Struct(valueSchema).put(ROUTING_FIELD, NEW_TOPIC); 55 | 56 | @Override 57 | public void start(final Map props) { 58 | } 59 | 60 | @Override 61 | public List poll() { 62 | if (counter >= MESSAGES_TO_PRODUCE) { 63 | return null; // indicate pause 64 | } 65 | 66 | final Map sourcePartition = new HashMap<>(); 67 | sourcePartition.put("partition", "0"); 68 | final Map sourceOffset = new HashMap<>(); 69 | sourceOffset.put("offset", Integer.toString(counter)); 70 | 71 | counter += 1; 72 | 73 | return Collections.singletonList( 74 | new SourceRecord(sourcePartition, sourceOffset, 75 | ORIGINAL_TOPIC, 76 | valueSchema, value) 77 | ); 78 | } 79 | 80 | @Override 81 | public void stop() { 82 | } 83 | 84 | @Override 85 | public String version() { 86 | return null; 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/integration-test/java/io/aiven/kafka/connect/transforms/TestCaseTransformConnector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Collections; 20 | import java.util.HashMap; 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | import org.apache.kafka.connect.connector.Task; 25 | import org.apache.kafka.connect.data.Schema; 26 | import org.apache.kafka.connect.data.SchemaBuilder; 27 | import org.apache.kafka.connect.data.Struct; 28 | import org.apache.kafka.connect.source.SourceRecord; 29 | import org.apache.kafka.connect.source.SourceTask; 30 | 31 | public class TestCaseTransformConnector extends AbstractTestSourceConnector { 32 | static final long MESSAGES_TO_PRODUCE = 10L; 33 | 34 | static final String SOURCE_TOPIC = "case-transform-source-topic"; 35 | static final String TARGET_TOPIC = "case-transform-target-topic"; 36 | static final String TRANSFORM_FIELD = "transform"; 37 | 38 | @Override 39 | public Class taskClass() { 40 | return TestCaseTransformConnector.TestSourceConnectorTask.class; 41 | } 42 | 43 | public static class TestSourceConnectorTask extends SourceTask { 44 | private int counter = 0; 45 | 46 | private final Schema valueSchema = SchemaBuilder.struct() 47 | .field(TRANSFORM_FIELD, SchemaBuilder.STRING_SCHEMA) 48 | .schema(); 49 | private final Struct value = 50 | new Struct(valueSchema).put(TRANSFORM_FIELD, "lower-case-data-transforms-to-uppercase"); 51 | 52 | @Override 53 | public void start(final Map props) { 54 | } 55 | 56 | @Override 57 | public List poll() { 58 | if (counter >= MESSAGES_TO_PRODUCE) { 59 | return null; // indicate pause 60 | } 61 | 62 | final Map sourcePartition = new HashMap<>(); 63 | sourcePartition.put("partition", "0"); 64 | final Map sourceOffset = new HashMap<>(); 65 | sourceOffset.put("offset", Integer.toString(counter)); 66 | 67 | counter += 1; 68 | 69 | return Collections.singletonList( 70 | new SourceRecord(sourcePartition, sourceOffset, 71 | SOURCE_TOPIC, 72 | valueSchema, value) 73 | ); 74 | } 75 | 76 | @Override 77 | public void stop() { 78 | } 79 | 80 | @Override 81 | public String version() { 82 | return null; 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/transforms/TombstoneHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Map; 20 | 21 | import org.apache.kafka.common.config.ConfigDef; 22 | import org.apache.kafka.connect.connector.ConnectRecord; 23 | import org.apache.kafka.connect.errors.DataException; 24 | import org.apache.kafka.connect.transforms.Transformation; 25 | 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | public class TombstoneHandler> implements Transformation { 30 | 31 | private static final Logger LOGGER = LoggerFactory.getLogger(TombstoneHandler.class); 32 | 33 | private TombstoneHandlerConfig tombstoneHandlerConfig; 34 | 35 | @Override 36 | public ConfigDef config() { 37 | return TombstoneHandlerConfig.config(); 38 | } 39 | 40 | @Override 41 | public void configure(final Map configs) { 42 | this.tombstoneHandlerConfig = new TombstoneHandlerConfig(configs); 43 | } 44 | 45 | @Override 46 | public R apply(final R record) { 47 | if (record.value() == null) { 48 | behaveOnNull(record); 49 | return null; 50 | } else { 51 | return record; 52 | } 53 | } 54 | 55 | private void behaveOnNull(final ConnectRecord r) { 56 | 57 | switch (tombstoneHandlerConfig.getBehavior()) { 58 | case FAIL: 59 | throw new DataException( 60 | String.format( 61 | "Tombstone record encountered, failing due to configured '%s' behavior", 62 | TombstoneHandlerConfig.Behavior.FAIL.name().toLowerCase() 63 | ) 64 | ); 65 | case DROP_SILENT: 66 | LOGGER.debug( 67 | "Tombstone record encountered, dropping due to configured '{}' behavior", 68 | TombstoneHandlerConfig.Behavior.DROP_SILENT.name().toLowerCase() 69 | ); 70 | break; 71 | case DROP_WARN: 72 | LOGGER.warn( 73 | "Tombstone record encountered, dropping due to configured '{}' behavior", 74 | TombstoneHandlerConfig.Behavior.DROP_WARN.name().toLowerCase() 75 | ); 76 | break; 77 | default: 78 | throw new DataException( 79 | String.format("Unknown behavior: %s", tombstoneHandlerConfig.getBehavior()) 80 | ); 81 | } 82 | } 83 | 84 | @Override 85 | public void close() { 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /demo/README.md: -------------------------------------------------------------------------------- 1 | # Demo 2 | 3 | How to load plugins to Kafka Connect installations. 4 | 5 | ## Requirements 6 | 7 | To run the demo, you need: 8 | - Docker compose 9 | - `curl` 10 | - `jq` 11 | 12 | ## Demo 13 | 14 | To install transforms into a Kafka Connect installation, it needs: 15 | 16 | - Build `transforms-for-apache-kafka-connect` libraries 17 | - Add libraries to Kafka Connect nodes 18 | - Configure Kafka Connect `plugin.path` 19 | 20 | ### Build `transforms-for-apache-kafka-connect` libraries 21 | 22 | To build libraries, use `gradlew installDist` command. 23 | e.g. Dockerfile build: 24 | 25 | ```dockerfile 26 | FROM eclipse-temurin:11-jdk AS base 27 | 28 | ADD ./ transforms 29 | WORKDIR transforms 30 | 31 | RUN ./gradlew installDist 32 | ``` 33 | 34 | This generates the set of libraries to be installed in Kafka Connect workers. 35 | 36 | ### Add libraries to Kafka Connect nodes 37 | 38 | Copy the directory with libraries into your Kafka Connect nodes. 39 | e.g. add directory to Docker images: 40 | 41 | ```dockerfile 42 | FROM confluentinc/cp-kafka-connect:7.3.3 43 | 44 | COPY --from=base /transforms/build/install/transforms-for-apache-kafka-connect /transforms 45 | ``` 46 | 47 | ### Configure Kafka Connect `plugin.path` 48 | 49 | On Kafka Connect configuration file, set `plugin.path` to indicate where to load plugins from, 50 | e.g. with Docker compose: 51 | 52 | ```yaml 53 | connect: 54 | # ... 55 | environment: 56 | # ... 57 | CONNECT_PLUGIN_PATH: /usr/share/java,/transforms # /transforms added on Dockerfile build 58 | ``` 59 | 60 | ## Running 61 | 62 | 1. Build docker images: `make build` or `docker compose build` 63 | 2. Start environment: `make up` or `docker compose up -d` 64 | 3. Test connect plugins are loaded: `make test` 65 | 66 | Sample response: 67 | ```json lines 68 | { 69 | "class": "io.aiven.kafka.connect.transforms.ConcatFields$Key", 70 | "type": "transformation" 71 | } 72 | { 73 | "class": "io.aiven.kafka.connect.transforms.ConcatFields$Value", 74 | "type": "transformation" 75 | } 76 | { 77 | "class": "io.aiven.kafka.connect.transforms.ExtractTimestamp$Key", 78 | "type": "transformation" 79 | } 80 | { 81 | "class": "io.aiven.kafka.connect.transforms.ExtractTimestamp$Value", 82 | "type": "transformation" 83 | } 84 | { 85 | "class": "io.aiven.kafka.connect.transforms.ExtractTopic$Key", 86 | "type": "transformation" 87 | } 88 | { 89 | "class": "io.aiven.kafka.connect.transforms.ExtractTopic$Value", 90 | "type": "transformation" 91 | } 92 | { 93 | "class": "io.aiven.kafka.connect.transforms.FilterByFieldValue$Key", 94 | "type": "transformation" 95 | } 96 | { 97 | "class": "io.aiven.kafka.connect.transforms.FilterByFieldValue$Value", 98 | "type": "transformation" 99 | } 100 | { 101 | "class": "io.aiven.kafka.connect.transforms.Hash$Key", 102 | "type": "transformation" 103 | } 104 | { 105 | "class": "io.aiven.kafka.connect.transforms.Hash$Value", 106 | "type": "transformation" 107 | } 108 | { 109 | "class": "io.aiven.kafka.connect.transforms.MakeTombstone", 110 | "type": "transformation" 111 | } 112 | { 113 | "class": "io.aiven.kafka.connect.transforms.TombstoneHandler", 114 | "type": "transformation" 115 | } 116 | ``` 117 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/transforms/ConcatFieldsConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.List; 20 | import java.util.Map; 21 | 22 | import org.apache.kafka.common.config.AbstractConfig; 23 | import org.apache.kafka.common.config.ConfigDef; 24 | 25 | final class ConcatFieldsConfig extends AbstractConfig { 26 | public static final String FIELD_NAMES_CONFIG = "field.names"; 27 | private static final String FIELD_NAMES_DOC = 28 | "A comma-separated list of fields to concatenate."; 29 | public static final String OUTPUT_FIELD_NAME_CONFIG = "output.field.name"; 30 | private static final String OUTPUT_FIELD_NAME_DOC = 31 | "The name of field the concatenated value should be placed into."; 32 | public static final String DELIMITER_CONFIG = "delimiter"; 33 | private static final String DELIMITER_DOC = 34 | "The string which should be used to join the extracted fields."; 35 | public static final String FIELD_REPLACE_MISSING_CONFIG = "field.replace.missing"; 36 | private static final String FIELD_REPLACE_MISSING_DOC = 37 | "The string which should be used when a field is not found or its value is null."; 38 | 39 | ConcatFieldsConfig(final Map originals) { 40 | super(config(), originals); 41 | } 42 | 43 | static ConfigDef config() { 44 | return new ConfigDef() 45 | .define( 46 | FIELD_NAMES_CONFIG, 47 | ConfigDef.Type.LIST, 48 | ConfigDef.NO_DEFAULT_VALUE, 49 | ConfigDef.Importance.HIGH, 50 | FIELD_NAMES_DOC) 51 | .define( 52 | OUTPUT_FIELD_NAME_CONFIG, 53 | ConfigDef.Type.STRING, 54 | ConfigDef.NO_DEFAULT_VALUE, 55 | new ConfigDef.NonEmptyString(), 56 | ConfigDef.Importance.HIGH, 57 | OUTPUT_FIELD_NAME_DOC) 58 | .define( 59 | FIELD_REPLACE_MISSING_CONFIG, 60 | ConfigDef.Type.STRING, 61 | "", 62 | ConfigDef.Importance.HIGH, 63 | FIELD_REPLACE_MISSING_DOC) 64 | .define( 65 | DELIMITER_CONFIG, 66 | ConfigDef.Type.STRING, 67 | "", 68 | ConfigDef.Importance.HIGH, 69 | DELIMITER_DOC); 70 | } 71 | 72 | final List fieldNames() { 73 | return getList(FIELD_NAMES_CONFIG); 74 | } 75 | 76 | final String outputFieldName() { 77 | return getString(OUTPUT_FIELD_NAME_CONFIG); 78 | } 79 | 80 | final String fieldReplaceMissing() { 81 | return getString(FIELD_REPLACE_MISSING_CONFIG); 82 | } 83 | 84 | final String delimiter() { 85 | return getString(DELIMITER_CONFIG); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/integration-test/java/io/aiven/kafka/connect/transforms/TestKeyToValueConnector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Collections; 20 | import java.util.HashMap; 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | import org.apache.kafka.connect.connector.Task; 25 | import org.apache.kafka.connect.data.Schema; 26 | import org.apache.kafka.connect.data.SchemaBuilder; 27 | import org.apache.kafka.connect.data.Struct; 28 | import org.apache.kafka.connect.source.SourceRecord; 29 | import org.apache.kafka.connect.source.SourceTask; 30 | 31 | public class TestKeyToValueConnector extends AbstractTestSourceConnector { 32 | 33 | static final long MESSAGES_TO_PRODUCE = 10L; 34 | 35 | static final String TARGET_TOPIC = "key-to-value-target-topic"; 36 | 37 | @Override 38 | public Class taskClass() { 39 | return TestKeyToValueConnector.TestSourceConnectorTask.class; 40 | } 41 | 42 | public static class TestSourceConnectorTask extends SourceTask { 43 | private int counter = 0; 44 | 45 | private final Schema keySchema = SchemaBuilder.struct().field("a1", SchemaBuilder.STRING_SCHEMA) 46 | .field("a2", SchemaBuilder.STRING_SCHEMA) 47 | .field("a3", SchemaBuilder.STRING_SCHEMA).schema(); 48 | private final Struct key = new Struct(keySchema).put("a1", "a1").put("a2", "a2").put("a3", "a3"); 49 | private final Schema valueSchema = SchemaBuilder.struct().field("b1", SchemaBuilder.STRING_SCHEMA) 50 | .field("b2", SchemaBuilder.STRING_SCHEMA) 51 | .field("b3", SchemaBuilder.STRING_SCHEMA).schema(); 52 | private final Struct value = new Struct(valueSchema).put("b1", "b1").put("b2", "b2").put("b3", "b3"); 53 | 54 | @Override 55 | public void start(final Map props) { 56 | } 57 | 58 | @Override 59 | public List poll() { 60 | if (counter >= MESSAGES_TO_PRODUCE) { 61 | return null; // indicate pause 62 | } 63 | 64 | final Map sourcePartition = new HashMap<>(); 65 | sourcePartition.put("partition", "0"); 66 | final Map sourceOffset = new HashMap<>(); 67 | sourceOffset.put("offset", Integer.toString(counter)); 68 | 69 | counter += 1; 70 | 71 | return Collections.singletonList( 72 | new SourceRecord(sourcePartition, sourceOffset, 73 | TARGET_TOPIC, 74 | keySchema, key, 75 | valueSchema, value) 76 | ); 77 | } 78 | 79 | @Override 80 | public void stop() { 81 | } 82 | 83 | @Override 84 | public String version() { 85 | return null; 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/integration-test/java/io/aiven/kafka/connect/transforms/TopicFromValueSchemaConnector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Collections; 20 | import java.util.HashMap; 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | import org.apache.kafka.connect.connector.Task; 25 | import org.apache.kafka.connect.data.Schema; 26 | import org.apache.kafka.connect.data.SchemaBuilder; 27 | import org.apache.kafka.connect.data.Struct; 28 | import org.apache.kafka.connect.source.SourceRecord; 29 | import org.apache.kafka.connect.source.SourceTask; 30 | 31 | import org.slf4j.Logger; 32 | import org.slf4j.LoggerFactory; 33 | 34 | /** 35 | * A connector needed for testing of ExtractTopicFromValueSchema. 36 | * 37 | *

It just produces a fixed number of struct records with value schema name set. 38 | */ 39 | public class TopicFromValueSchemaConnector extends AbstractTestSourceConnector { 40 | static final int MESSAGES_TO_PRODUCE = 10; 41 | 42 | private static final Logger log = LoggerFactory.getLogger(TopicFromValueSchemaConnector.class); 43 | static final String TOPIC = "topic-for-value-schema-connector-test"; 44 | static final String FIELD = "field-0"; 45 | 46 | static final String NAME = "com.acme.schema.SchemaNameToTopic"; 47 | 48 | @Override 49 | public Class taskClass() { 50 | return TopicFromValueSchemaConnectorTask.class; 51 | } 52 | 53 | public static class TopicFromValueSchemaConnectorTask extends SourceTask { 54 | private int counter = 0; 55 | 56 | private final Schema valueSchema = SchemaBuilder.struct() 57 | .field(FIELD, SchemaBuilder.STRING_SCHEMA) 58 | .name(NAME) 59 | .schema(); 60 | private final Struct value = new Struct(valueSchema).put(FIELD, "Data"); 61 | 62 | @Override 63 | public void start(final Map props) { 64 | log.info("Started TopicFromValueSchemaConnector!!!"); 65 | } 66 | 67 | @Override 68 | public List poll() { 69 | if (counter >= MESSAGES_TO_PRODUCE) { 70 | return null; // indicate pause 71 | } 72 | 73 | final Map sourcePartition = new HashMap<>(); 74 | sourcePartition.put("partition", "0"); 75 | final Map sourceOffset = new HashMap<>(); 76 | sourceOffset.put("offset", Integer.toString(counter)); 77 | 78 | counter += 1; 79 | 80 | return Collections.singletonList( 81 | new SourceRecord(sourcePartition, sourceOffset, 82 | TOPIC, 83 | valueSchema, value) 84 | ); 85 | } 86 | 87 | @Override 88 | public void stop() { 89 | } 90 | 91 | @Override 92 | public String version() { 93 | return null; 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/HashConfigTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | import org.apache.kafka.common.config.ConfigException; 23 | 24 | import org.junit.jupiter.api.Test; 25 | import org.junit.jupiter.params.ParameterizedTest; 26 | import org.junit.jupiter.params.provider.ValueSource; 27 | 28 | import static org.assertj.core.api.Assertions.assertThat; 29 | import static org.assertj.core.api.Assertions.assertThatThrownBy; 30 | 31 | class HashConfigTest { 32 | @Test 33 | void defaults() { 34 | final Map props = new HashMap<>(); 35 | assertThatThrownBy(() -> new HashConfig(props)) 36 | .isInstanceOf(ConfigException.class) 37 | .hasMessage("Missing required configuration \"function\" which has no default value."); 38 | } 39 | 40 | @ParameterizedTest 41 | @ValueSource(booleans = {true, false}) 42 | void skipMissingOrNull(final boolean skipMissingOrNull) { 43 | final Map props = new HashMap<>(); 44 | props.put("skip.missing.or.null", Boolean.toString(skipMissingOrNull)); 45 | props.put("function", "sha256"); 46 | final HashConfig config = new HashConfig(props); 47 | assertThat(config.skipMissingOrNull()).isEqualTo(skipMissingOrNull); 48 | } 49 | 50 | @Test 51 | void hashFunctionMd5() { 52 | final Map props = new HashMap<>(); 53 | props.put("function", "md5"); 54 | final HashConfig config = new HashConfig(props); 55 | assertThat(config.hashFunction()).isEqualTo(HashConfig.HashFunction.MD5); 56 | } 57 | 58 | @Test 59 | void hashFunctionSha1() { 60 | final Map props = new HashMap<>(); 61 | props.put("function", "sha1"); 62 | final HashConfig config = new HashConfig(props); 63 | assertThat(config.hashFunction()).isEqualTo(HashConfig.HashFunction.SHA1); 64 | } 65 | 66 | @Test 67 | void hashFunctionSha256() { 68 | final Map props = new HashMap<>(); 69 | props.put("function", "sha256"); 70 | final HashConfig config = new HashConfig(props); 71 | assertThat(config.hashFunction()).isEqualTo(HashConfig.HashFunction.SHA256); 72 | } 73 | 74 | @Test 75 | void emptyFieldName() { 76 | final Map props = new HashMap<>(); 77 | props.put("field.name", ""); 78 | props.put("function", "sha256"); 79 | final HashConfig config = new HashConfig(props); 80 | assertThat(config.fieldName()).isNotPresent(); 81 | } 82 | 83 | @Test 84 | void definedFieldName() { 85 | final Map props = new HashMap<>(); 86 | props.put("field.name", "test"); 87 | props.put("function", "sha256"); 88 | final HashConfig config = new HashConfig(props); 89 | assertThat(config.fieldName()).hasValue("test"); 90 | assertThat(config.hashFunction()).isEqualTo(HashConfig.HashFunction.SHA256); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/transforms/CaseTransformConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.List; 20 | import java.util.Map; 21 | import java.util.Objects; 22 | 23 | import org.apache.kafka.common.config.AbstractConfig; 24 | import org.apache.kafka.common.config.ConfigDef; 25 | 26 | /** 27 | * Case transform configuration. 28 | * 29 | *

Configure the SMT to do case transform on configured fields. 30 | * Supported case transformations are transform to upper case and transform to lowercase.

31 | */ 32 | public class CaseTransformConfig extends AbstractConfig { 33 | 34 | /** 35 | * A comma-separated list of fields to concatenate. 36 | */ 37 | public static final String FIELD_NAMES_CONFIG = "field.names"; 38 | private static final String FIELD_NAMES_DOC = 39 | "A comma-separated list of fields to concatenate."; 40 | /** 41 | * Set the case configuration, 'upper' or 'lower' are supported. 42 | */ 43 | public static final String CASE_CONFIG = "case"; 44 | private static final String CASE_DOC = 45 | "Set the case configuration, 'upper' or 'lower'."; 46 | 47 | CaseTransformConfig(final Map originals) { 48 | super(config(), originals); 49 | } 50 | 51 | static ConfigDef config() { 52 | return new ConfigDef() 53 | .define( 54 | FIELD_NAMES_CONFIG, 55 | ConfigDef.Type.LIST, 56 | ConfigDef.NO_DEFAULT_VALUE, 57 | ConfigDef.Importance.HIGH, 58 | FIELD_NAMES_DOC) 59 | .define( 60 | CASE_CONFIG, 61 | ConfigDef.Type.STRING, 62 | ConfigDef.NO_DEFAULT_VALUE, 63 | new ConfigDef.NonEmptyString(), 64 | ConfigDef.Importance.HIGH, 65 | CASE_DOC); 66 | } 67 | 68 | final List fieldNames() { 69 | return getList(FIELD_NAMES_CONFIG); 70 | } 71 | 72 | final Case transformCase() { 73 | return Objects.requireNonNull(Case.fromString(getString(CASE_CONFIG))); 74 | } 75 | 76 | /** 77 | * Case enumeration for supported transforms. 78 | */ 79 | public enum Case { 80 | LOWER("lower"), 81 | UPPER("upper"); 82 | 83 | private final String transformCase; 84 | 85 | Case(final String transformCase) { 86 | this.transformCase = transformCase; 87 | } 88 | 89 | /** 90 | * Return the case enumeration object resolved from the given parameter. 91 | * @param string The case enumeration to fetch. 92 | * @return 93 | */ 94 | public static Case fromString(final String string) { 95 | for (final Case caseValue : values()) { 96 | if (caseValue.transformCase.equals(string)) { 97 | return caseValue; 98 | } 99 | } 100 | throw new IllegalArgumentException(String.format("Unknown enum value %s", string)); 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/transforms/TombstoneHandlerConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Arrays; 20 | import java.util.List; 21 | import java.util.Map; 22 | import java.util.Objects; 23 | import java.util.stream.Collectors; 24 | 25 | import org.apache.kafka.common.config.AbstractConfig; 26 | import org.apache.kafka.common.config.ConfigDef; 27 | import org.apache.kafka.common.config.ConfigException; 28 | 29 | public final class TombstoneHandlerConfig extends AbstractConfig { 30 | 31 | public static final String TOMBSTONE_HANDLER_BEHAVIOR = "behavior"; 32 | 33 | public TombstoneHandlerConfig(final Map originals) { 34 | super(config(), originals); 35 | } 36 | 37 | static ConfigDef config() { 38 | return new ConfigDef() 39 | .define( 40 | TOMBSTONE_HANDLER_BEHAVIOR, 41 | ConfigDef.Type.STRING, 42 | ConfigDef.NO_DEFAULT_VALUE, 43 | new ConfigDef.Validator() { 44 | @Override 45 | public void ensureValid(final String name, final Object value) { 46 | assert value instanceof String; 47 | 48 | final String strValue = (String) value; 49 | 50 | if (Objects.isNull(strValue) || strValue.isEmpty()) { 51 | throw new ConfigException( 52 | TOMBSTONE_HANDLER_BEHAVIOR, 53 | value, 54 | "String must be non-empty"); 55 | } 56 | 57 | try { 58 | Behavior.of(strValue); 59 | } catch (final IllegalArgumentException e) { 60 | throw new ConfigException( 61 | TOMBSTONE_HANDLER_BEHAVIOR, 62 | value, 63 | e.getMessage()); 64 | } 65 | } 66 | }, 67 | ConfigDef.Importance.MEDIUM, 68 | String.format( 69 | "Specifies the behavior on encountering tombstone messages. Possible values are: %s", 70 | Behavior.BEHAVIOR_NAMES 71 | ) 72 | ); 73 | } 74 | 75 | public Behavior getBehavior() { 76 | return Behavior.of(getString(TOMBSTONE_HANDLER_BEHAVIOR)); 77 | } 78 | 79 | public enum Behavior { 80 | 81 | DROP_SILENT, 82 | DROP_WARN, 83 | FAIL; 84 | 85 | private static final List BEHAVIOR_NAMES = 86 | Arrays.stream(Behavior.values()) 87 | .map(b -> b.name().toLowerCase()) 88 | .collect(Collectors.toList()); 89 | 90 | public static Behavior of(final String v) { 91 | 92 | for (final Behavior b : Behavior.values()) { 93 | if (b.name().equalsIgnoreCase(v)) { 94 | return b; 95 | } 96 | } 97 | throw new IllegalArgumentException( 98 | String.format( 99 | "Unsupported behavior name: %s. Supported are: %s", v, 100 | String.join(",", BEHAVIOR_NAMES)) 101 | ); 102 | 103 | } 104 | 105 | } 106 | 107 | } 108 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/ExtractTopicFromSchemaNameConfigTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | import org.apache.kafka.common.config.ConfigException; 23 | 24 | import org.junit.jupiter.api.Test; 25 | 26 | import static org.junit.jupiter.api.Assertions.assertEquals; 27 | import static org.junit.jupiter.api.Assertions.assertNotNull; 28 | import static org.junit.jupiter.api.Assertions.assertThrows; 29 | 30 | public class ExtractTopicFromSchemaNameConfigTest { 31 | 32 | @Test 33 | void defaults() { 34 | final Map configs = new HashMap<>(); 35 | assertNotNull(new ExtractTopicFromSchemaNameConfig(configs)); 36 | } 37 | 38 | @Test 39 | void testAllConfigsSet() { 40 | final Map configs = new HashMap<>(); 41 | configs.put(ExtractTopicFromSchemaNameConfig.REGEX_SCHEMA_NAME_TO_TOPIC, "regex"); 42 | configs.put(ExtractTopicFromSchemaNameConfig.SCHEMA_NAME_TO_TOPIC_MAP, "map:value"); 43 | final Throwable e = assertThrows(ConfigException.class, 44 | () -> new ExtractTopicFromSchemaNameConfig(configs)); 45 | assertEquals("schema.name.topic-map and schema.name.regex should not be defined together.", 46 | e.getMessage()); 47 | } 48 | 49 | @Test 50 | void testRegExConfigSetWithNameToTopicMap() { 51 | final Map configs = new HashMap<>(); 52 | configs.put(ExtractTopicFromSchemaNameConfig.SCHEMA_NAME_TO_TOPIC_MAP, 53 | "com.acme.schema.SchemaNameToTopic1:Name1,com.acme.schema.SchemaNameToTopic2:Name2"); 54 | final ExtractTopicFromSchemaNameConfig extractTopicFromSchemaNameConfig 55 | = new ExtractTopicFromSchemaNameConfig(configs); 56 | assertEquals(2, 57 | extractTopicFromSchemaNameConfig.schemaNameToTopicMap().size()); 58 | } 59 | 60 | @Test 61 | void testRegExConfigSetWithInvalidNameToTopicMap() { 62 | final Map configs = new HashMap<>(); 63 | configs.put(ExtractTopicFromSchemaNameConfig.SCHEMA_NAME_TO_TOPIC_MAP, 64 | "com.acme.schema.SchemaNameToTopic1TheNameToReplace1"); 65 | final Throwable e = assertThrows(ConfigException.class, 66 | () -> new ExtractTopicFromSchemaNameConfig(configs)); 67 | assertEquals("schema.name.topic-map is not valid. Format should be: " 68 | + "\"SchemaValue1:NewValue1,SchemaValue2:NewValue2\"", e.getMessage()); 69 | } 70 | 71 | @Test 72 | void testRegExConfigSetWithInvalidRegEx() { 73 | final Map configs = new HashMap<>(); 74 | configs.put(ExtractTopicFromSchemaNameConfig.REGEX_SCHEMA_NAME_TO_TOPIC, "***"); 75 | final Throwable e = assertThrows(ConfigException.class, 76 | () -> new ExtractTopicFromSchemaNameConfig(configs)); 77 | assertEquals("*** set as schema.name.regex is not valid regex.", e.getMessage()); 78 | } 79 | 80 | @Test 81 | void testRegExConfigSetWithValidRegEx() { 82 | final Map configs = new HashMap<>(); 83 | configs.put(ExtractTopicFromSchemaNameConfig.REGEX_SCHEMA_NAME_TO_TOPIC, "(?:[.]|^)([^.]*)$"); 84 | final ExtractTopicFromSchemaNameConfig extractTopicFromSchemaNameConfig 85 | = new ExtractTopicFromSchemaNameConfig(configs); 86 | assertEquals("(?:[.]|^)([^.]*)$", extractTopicFromSchemaNameConfig.regEx().get()); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/transforms/ExtractTopicFromSchemaName.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Map; 20 | import java.util.Optional; 21 | import java.util.regex.Matcher; 22 | import java.util.regex.Pattern; 23 | 24 | import org.apache.kafka.common.config.ConfigDef; 25 | import org.apache.kafka.connect.connector.ConnectRecord; 26 | import org.apache.kafka.connect.errors.DataException; 27 | import org.apache.kafka.connect.transforms.Transformation; 28 | 29 | import org.slf4j.Logger; 30 | import org.slf4j.LoggerFactory; 31 | 32 | public abstract class ExtractTopicFromSchemaName> implements Transformation { 33 | 34 | private static final Logger log = LoggerFactory.getLogger(ExtractTopicFromSchemaName.class); 35 | 36 | private Map schemaNameToTopicMap; 37 | private Pattern pattern; 38 | 39 | @Override 40 | public ConfigDef config() { 41 | return ExtractTopicFromSchemaNameConfig.config(); 42 | } 43 | 44 | @Override 45 | public void configure(final Map configs) { 46 | final ExtractTopicFromSchemaNameConfig config = new ExtractTopicFromSchemaNameConfig(configs); 47 | schemaNameToTopicMap = config.schemaNameToTopicMap(); 48 | final Optional regex = config.regEx(); 49 | regex.ifPresent(s -> pattern = Pattern.compile(s)); 50 | } 51 | 52 | public abstract String schemaName(R record); 53 | 54 | @Override 55 | public R apply(final R record) { 56 | 57 | final String schemaName = schemaName(record); 58 | // First check schema value name -> desired topic name mapping and use that if it is set. 59 | if (schemaNameToTopicMap.containsKey(schemaName)) { 60 | return createConnectRecord(record, schemaNameToTopicMap.get(schemaName)); 61 | } 62 | // Secondly check if regex parsing from schema value name is set and use that. 63 | if (pattern != null) { 64 | final Matcher matcher = pattern.matcher(schemaName); 65 | if (matcher.find() && matcher.groupCount() == 1) { 66 | return createConnectRecord(record, matcher.group(1)); 67 | } 68 | log.trace("No match with pattern {} from schema name {}", pattern.pattern(), schemaName); 69 | } 70 | // If no other configurations are set use value schema name as new topic name. 71 | return createConnectRecord(record, schemaName); 72 | } 73 | 74 | private R createConnectRecord(final R record, final String newTopicName) { 75 | return record.newRecord( 76 | newTopicName, 77 | record.kafkaPartition(), 78 | record.keySchema(), 79 | record.key(), 80 | record.valueSchema(), 81 | record.value(), 82 | record.timestamp(), 83 | record.headers() 84 | ); 85 | } 86 | 87 | @Override 88 | public void close() { 89 | } 90 | 91 | public static class Value> extends ExtractTopicFromSchemaName { 92 | @Override 93 | public String schemaName(final R record) { 94 | if (null == record.valueSchema() || null == record.valueSchema().name()) { 95 | throw new DataException(" value schema name can't be null: " + record); 96 | } 97 | return record.valueSchema().name(); 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 7. Before merging, clean up the commit history for the PR. Each commit should be self-contained with an informative message, since each commit will be added to the history for this project. 39 | 40 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 41 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 42 | 43 | ## Developer Certificate of Origin 44 | 45 | Aiven Transformations for Apache Kafka® Connect is an open source product released under the Apache 2.0 license (see either [the Apache site](https://www.apache.org/licenses/LICENSE-2.0) or the [LICENSE.txt file](LICENSE.txt)). The Apache 2.0 license allows you to freely use, modify, distribute, and sell your own products that include Apache 2.0 licensed software. 46 | 47 | We respect intellectual property rights of others and we want to make sure all incoming contributions are correctly attributed and licensed. A Developer Certificate of Origin (DCO) is a lightweight mechanism to do that. 48 | 49 | So we require by making a contribution every contributor certifies that: 50 | ``` 51 | The contribution was created in whole or in part by me and I have the right to submit it under the open source license 52 | indicated in the file 53 | ``` 54 | 55 | ## Finding contributions to work on 56 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 57 | 58 | 59 | ## Code of Conduct 60 | This project has adopted the [Contributor Covenant Code of Conduct](CODE_OF_CONDUCT.md). 61 | For more information see the [Code of Conduct FAQ](https://www.contributor-covenant.org/faq/). 62 | 63 | 64 | ## Security issue notifications 65 | If you discover a potential security issue in this project we ask that you report it according to [Security Policy](SECURITY.md). Please do **not** create a public github issue. 66 | 67 | ## Licensing 68 | 69 | See the [LICENSE](LICENSE.txt) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 70 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/transforms/HashConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Map; 20 | import java.util.Optional; 21 | 22 | import org.apache.kafka.common.config.AbstractConfig; 23 | import org.apache.kafka.common.config.ConfigDef; 24 | 25 | class HashConfig extends AbstractConfig { 26 | private static final String FIELD_NAME_CONFIG = "field.name"; 27 | private static final String FIELD_NAME_DOC = 28 | "The name of the field which value should be hashed. If null or empty, " 29 | + "the entire key or value is used (and assumed to be a string). By default is null."; 30 | private static final String SKIP_MISSING_OR_NULL_CONFIG = "skip.missing.or.null"; 31 | private static final String SKIP_MISSING_OR_NULL_DOC = 32 | "In case the value to be hashed is null or missing, " 33 | + "should a record be silently passed without transformation."; 34 | private static final String FUNCTION_CONFIG = "function"; 35 | private static final String FUNCTION_DOC = 36 | "The name of the hash function to use. The supported values are: md5, sha1, sha256."; 37 | 38 | HashConfig(final Map originals) { 39 | super(config(), originals); 40 | } 41 | 42 | static ConfigDef config() { 43 | return new ConfigDef() 44 | .define( 45 | FIELD_NAME_CONFIG, 46 | ConfigDef.Type.STRING, 47 | null, 48 | ConfigDef.Importance.HIGH, 49 | FIELD_NAME_DOC) 50 | .define( 51 | SKIP_MISSING_OR_NULL_CONFIG, 52 | ConfigDef.Type.BOOLEAN, 53 | false, 54 | ConfigDef.Importance.LOW, 55 | SKIP_MISSING_OR_NULL_DOC) 56 | .define( 57 | FUNCTION_CONFIG, 58 | ConfigDef.Type.STRING, 59 | ConfigDef.NO_DEFAULT_VALUE, 60 | ConfigDef.ValidString.in( 61 | HashFunction.MD5.toString(), 62 | HashFunction.SHA1.toString(), 63 | HashFunction.SHA256.toString()), 64 | ConfigDef.Importance.HIGH, 65 | FUNCTION_DOC); 66 | } 67 | 68 | public enum HashFunction { 69 | MD5 { 70 | public String toString() { 71 | return "md5"; 72 | } 73 | }, 74 | SHA1 { 75 | public String toString() { 76 | return "sha1"; 77 | } 78 | }, 79 | SHA256 { 80 | public String toString() { 81 | return "sha256"; 82 | } 83 | }; 84 | 85 | public static HashFunction fromString(final String value) { 86 | return valueOf(value.toUpperCase()); 87 | } 88 | } 89 | 90 | Optional fieldName() { 91 | final String rawFieldName = getString(FIELD_NAME_CONFIG); 92 | if (null == rawFieldName || "".equals(rawFieldName)) { 93 | return Optional.empty(); 94 | } 95 | return Optional.of(rawFieldName); 96 | } 97 | 98 | boolean skipMissingOrNull() { 99 | return getBoolean(SKIP_MISSING_OR_NULL_CONFIG); 100 | } 101 | 102 | HashFunction hashFunction() { 103 | return HashFunction.fromString(getString(FUNCTION_CONFIG)); 104 | } 105 | } 106 | 107 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/ExtractTimestampConfigTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | import org.apache.kafka.common.config.ConfigException; 23 | 24 | import org.junit.jupiter.api.Test; 25 | 26 | import static org.assertj.core.api.Assertions.assertThat; 27 | import static org.assertj.core.api.Assertions.assertThatThrownBy; 28 | 29 | class ExtractTimestampConfigTest { 30 | @Test 31 | void emptyConfig() { 32 | final Map props = new HashMap<>(); 33 | assertThatThrownBy(() -> new ExtractTimestampConfig(props)) 34 | .isInstanceOf(ConfigException.class) 35 | .hasMessage("Missing required configuration \"field.name\" which has no default value."); 36 | } 37 | 38 | @Test 39 | void emptyFieldName() { 40 | final Map props = new HashMap<>(); 41 | props.put("field.name", ""); 42 | assertThatThrownBy(() -> new ExtractTimestampConfig(props)) 43 | .isInstanceOf(ConfigException.class) 44 | .hasMessage("Invalid value for configuration field.name: String must be non-empty"); 45 | } 46 | 47 | @Test 48 | void definedFieldName() { 49 | final Map props = new HashMap<>(); 50 | props.put("field.name", "test"); 51 | final ExtractTimestampConfig config = new ExtractTimestampConfig(props); 52 | assertThat(config.fieldName()).isEqualTo("test"); 53 | } 54 | 55 | @Test 56 | void emptyTimestampResolution() { 57 | final var props = new HashMap<>(); 58 | props.put("field.name", "test"); 59 | final var config = new ExtractTimestampConfig(props); 60 | assertThat(config.timestampResolution()).isEqualTo(ExtractTimestampConfig.TimestampResolution.MILLISECONDS); 61 | } 62 | 63 | @Test 64 | void definedTimestampResolutionInSeconds() { 65 | final var props = new HashMap<>(); 66 | props.put("field.name", "test"); 67 | props.put( 68 | ExtractTimestampConfig.EPOCH_RESOLUTION_CONFIG, 69 | ExtractTimestampConfig.TimestampResolution.SECONDS.resolution 70 | ); 71 | final var config = new ExtractTimestampConfig(props); 72 | assertThat(config.timestampResolution()).isEqualTo(ExtractTimestampConfig.TimestampResolution.SECONDS); 73 | } 74 | 75 | @Test 76 | void definedTimestampResolutionInMillis() { 77 | final var props = new HashMap<>(); 78 | props.put("field.name", "test"); 79 | props.put( 80 | ExtractTimestampConfig.EPOCH_RESOLUTION_CONFIG, 81 | ExtractTimestampConfig.TimestampResolution.MILLISECONDS.resolution 82 | ); 83 | final var config = new ExtractTimestampConfig(props); 84 | assertThat(config.timestampResolution()).isEqualTo(ExtractTimestampConfig.TimestampResolution.MILLISECONDS); 85 | } 86 | 87 | @Test 88 | void wrongTimestampResolution() { 89 | final var props = new HashMap<>(); 90 | props.put("field.name", "test"); 91 | props.put( 92 | ExtractTimestampConfig.EPOCH_RESOLUTION_CONFIG, 93 | "foo" 94 | ); 95 | assertThatThrownBy(() -> new ExtractTimestampConfig(props)) 96 | .isInstanceOf(ConfigException.class) 97 | .hasMessage("Invalid value foo for configuration timestamp.resolution: " 98 | + "Unsupported resolution type 'foo'. Supported are: milliseconds, seconds"); 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/transforms/ExtractTopicFromSchemaNameConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Arrays; 20 | import java.util.Collections; 21 | import java.util.Map; 22 | import java.util.Optional; 23 | import java.util.regex.Pattern; 24 | import java.util.regex.PatternSyntaxException; 25 | import java.util.stream.Collectors; 26 | 27 | import org.apache.kafka.common.config.AbstractConfig; 28 | import org.apache.kafka.common.config.ConfigDef; 29 | import org.apache.kafka.common.config.ConfigException; 30 | 31 | public class ExtractTopicFromSchemaNameConfig extends AbstractConfig { 32 | public static final String SCHEMA_NAME_TO_TOPIC_MAP = "schema.name.topic-map"; 33 | public static final String REGEX_SCHEMA_NAME_TO_TOPIC = "schema.name.regex"; 34 | 35 | public static final String SCHEMA_NAME_TO_TOPIC_DOC = "Map of schema name (key), " 36 | + "new topic name (value) in String format \"key1:value1,key2:value2\""; 37 | 38 | public static final String REGEX_SCHEMA_NAME_TO_TOPIC_DOC = "Regular expression that is used to find the " 39 | + "first desired new topic value from value schema name " 40 | + "(for example (?:[.\\t]|^)([^.\\t]*)$ which parses the name after last ."; 41 | 42 | public ExtractTopicFromSchemaNameConfig(final Map originals) { 43 | super(config(), originals); 44 | 45 | if (originals.containsKey(SCHEMA_NAME_TO_TOPIC_MAP) && originals.containsKey(REGEX_SCHEMA_NAME_TO_TOPIC)) { 46 | throw new ConfigException(SCHEMA_NAME_TO_TOPIC_MAP + " and " 47 | + REGEX_SCHEMA_NAME_TO_TOPIC + " should not be defined together."); 48 | } 49 | 50 | if (originals.containsKey(REGEX_SCHEMA_NAME_TO_TOPIC)) { 51 | final String regex = (String) originals.get(REGEX_SCHEMA_NAME_TO_TOPIC); 52 | try { 53 | Pattern.compile(regex); 54 | } catch (final PatternSyntaxException e) { 55 | throw new ConfigException(regex + " set as " + REGEX_SCHEMA_NAME_TO_TOPIC + " is not valid regex."); 56 | } 57 | } 58 | 59 | if (originals.containsKey(SCHEMA_NAME_TO_TOPIC_MAP)) { 60 | final String mapString = (String) originals.get(SCHEMA_NAME_TO_TOPIC_MAP); 61 | if (!mapString.contains(":")) { 62 | throw new ConfigException(SCHEMA_NAME_TO_TOPIC_MAP + " is not valid. Format should be: " 63 | + "\"SchemaValue1:NewValue1,SchemaValue2:NewValue2\""); 64 | } 65 | } 66 | } 67 | 68 | static ConfigDef config() { 69 | return new ConfigDef().define( 70 | SCHEMA_NAME_TO_TOPIC_MAP, 71 | ConfigDef.Type.STRING, 72 | null, 73 | ConfigDef.Importance.LOW, 74 | SCHEMA_NAME_TO_TOPIC_DOC) 75 | .define(REGEX_SCHEMA_NAME_TO_TOPIC, 76 | ConfigDef.Type.STRING, 77 | null, 78 | ConfigDef.Importance.LOW, 79 | REGEX_SCHEMA_NAME_TO_TOPIC_DOC 80 | ); 81 | } 82 | 83 | Map schemaNameToTopicMap() { 84 | final String schemaNameToTopicValue = getString(SCHEMA_NAME_TO_TOPIC_MAP); 85 | if (null == schemaNameToTopicValue) { 86 | return Collections.emptyMap(); 87 | } 88 | return Arrays.stream(schemaNameToTopicValue.split(",")).map(entry -> entry.split(":")) 89 | .collect(Collectors.toMap(key -> key[0], value -> value[1])); 90 | } 91 | 92 | Optional regEx() { 93 | final String rawFieldName = getString(REGEX_SCHEMA_NAME_TO_TOPIC); 94 | if (null == rawFieldName || rawFieldName.isEmpty()) { 95 | return Optional.empty(); 96 | } 97 | return Optional.of(rawFieldName); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/transforms/ExtractTimestampConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Arrays; 20 | import java.util.Map; 21 | import java.util.stream.Collectors; 22 | 23 | import org.apache.kafka.common.config.AbstractConfig; 24 | import org.apache.kafka.common.config.ConfigDef; 25 | import org.apache.kafka.common.config.ConfigException; 26 | 27 | final class ExtractTimestampConfig extends AbstractConfig { 28 | 29 | public static final String FIELD_NAME_CONFIG = "field.name"; 30 | private static final String FIELD_NAME_DOC = "The name of the field is to be used as the source of timestamp. " 31 | + "The field must have INT64 or org.apache.kafka.connect.data.Timestamp type " 32 | + "and must mot be null."; 33 | 34 | public static final String EPOCH_RESOLUTION_CONFIG = "timestamp.resolution"; 35 | private static final String EPOCH_RESOLUTION_DOC = "Time resolution used for INT64 type field. " 36 | + "Valid values are \"seconds\" for seconds since epoch and \"milliseconds\" for " 37 | + "milliseconds since epoch. Default is \"milliseconds\" and ignored for " 38 | + "org.apache.kafka.connect.data.Timestamp type."; 39 | 40 | 41 | public enum TimestampResolution { 42 | 43 | MILLISECONDS("milliseconds"), 44 | SECONDS("seconds"); 45 | 46 | final String resolution; 47 | 48 | private static final String RESOLUTIONS = 49 | Arrays.stream(values()).map(TimestampResolution::resolution).collect(Collectors.joining(", ")); 50 | 51 | private TimestampResolution(final String resolution) { 52 | this.resolution = resolution; 53 | } 54 | 55 | public String resolution() { 56 | return resolution; 57 | } 58 | 59 | public static TimestampResolution fromString(final String value) { 60 | for (final var r : values()) { 61 | if (r.resolution.equals(value)) { 62 | return r; 63 | } 64 | } 65 | throw new IllegalArgumentException( 66 | "Unsupported resolution type '" + value + "'. Supported are: " + RESOLUTIONS); 67 | } 68 | 69 | } 70 | 71 | ExtractTimestampConfig(final Map originals) { 72 | super(config(), originals); 73 | } 74 | 75 | static ConfigDef config() { 76 | return new ConfigDef() 77 | .define( 78 | FIELD_NAME_CONFIG, 79 | ConfigDef.Type.STRING, 80 | ConfigDef.NO_DEFAULT_VALUE, 81 | new ConfigDef.NonEmptyString(), 82 | ConfigDef.Importance.HIGH, 83 | FIELD_NAME_DOC) 84 | .define( 85 | EPOCH_RESOLUTION_CONFIG, 86 | ConfigDef.Type.STRING, 87 | TimestampResolution.MILLISECONDS.resolution, 88 | new ConfigDef.Validator() { 89 | @Override 90 | public void ensureValid(final String name, final Object value) { 91 | assert value instanceof String; 92 | try { 93 | TimestampResolution.fromString((String) value); 94 | } catch (final IllegalArgumentException e) { 95 | throw new ConfigException(EPOCH_RESOLUTION_CONFIG, value, e.getMessage()); 96 | } 97 | } 98 | }, 99 | ConfigDef.Importance.LOW, 100 | EPOCH_RESOLUTION_DOC); 101 | } 102 | 103 | final String fieldName() { 104 | return getString(FIELD_NAME_CONFIG); 105 | } 106 | 107 | final TimestampResolution timestampResolution() { 108 | return TimestampResolution.fromString(getString(EPOCH_RESOLUTION_CONFIG)); 109 | } 110 | 111 | } 112 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/transforms/ExtractTimestamp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Date; 20 | import java.util.Map; 21 | import java.util.concurrent.TimeUnit; 22 | 23 | import org.apache.kafka.common.config.ConfigDef; 24 | import org.apache.kafka.connect.connector.ConnectRecord; 25 | import org.apache.kafka.connect.data.SchemaAndValue; 26 | import org.apache.kafka.connect.data.Struct; 27 | import org.apache.kafka.connect.errors.DataException; 28 | import org.apache.kafka.connect.transforms.Transformation; 29 | 30 | public abstract class ExtractTimestamp> implements Transformation { 31 | 32 | private ExtractTimestampConfig config; 33 | 34 | @Override 35 | public ConfigDef config() { 36 | return ExtractTimestampConfig.config(); 37 | } 38 | 39 | @Override 40 | public void configure(final Map configs) { 41 | this.config = new ExtractTimestampConfig(configs); 42 | } 43 | 44 | @Override 45 | public R apply(final R record) { 46 | final SchemaAndValue schemaAndValue = getSchemaAndValue(record); 47 | 48 | if (schemaAndValue.value() == null) { 49 | throw new DataException(keyOrValue() + " can't be null: " + record); 50 | } 51 | 52 | final Object fieldValue; 53 | if (schemaAndValue.value() instanceof Struct) { 54 | final Struct struct = (Struct) schemaAndValue.value(); 55 | if (struct.schema().field(config.fieldName()) == null) { 56 | throw new DataException(config.fieldName() + " field must be present and its value can't be null: " 57 | + record); 58 | } 59 | fieldValue = struct.get(config.fieldName()); 60 | } else if (schemaAndValue.value() instanceof Map) { 61 | final Map map = (Map) schemaAndValue.value(); 62 | fieldValue = map.get(config.fieldName()); 63 | } else { 64 | throw new DataException(keyOrValue() + " type must be STRUCT or MAP: " + record); 65 | } 66 | 67 | if (fieldValue == null) { 68 | throw new DataException(config.fieldName() + " field must be present and its value can't be null: " 69 | + record); 70 | } 71 | 72 | final long newTimestamp; 73 | if (fieldValue instanceof Long) { 74 | final var longFieldValue = (long) fieldValue; 75 | if (config.timestampResolution() == ExtractTimestampConfig.TimestampResolution.SECONDS) { 76 | newTimestamp = TimeUnit.SECONDS.toMillis(longFieldValue); 77 | } else { 78 | newTimestamp = longFieldValue; 79 | } 80 | } else if (fieldValue instanceof Date) { 81 | final var dateFieldValue = (Date) fieldValue; 82 | newTimestamp = dateFieldValue.getTime(); 83 | } else { 84 | throw new DataException(config.fieldName() 85 | + " field must be INT64 or org.apache.kafka.connect.data.Timestamp: " 86 | + record); 87 | } 88 | 89 | return record.newRecord( 90 | record.topic(), 91 | record.kafkaPartition(), 92 | record.keySchema(), 93 | record.key(), 94 | record.valueSchema(), 95 | record.value(), 96 | newTimestamp 97 | ); 98 | } 99 | 100 | @Override 101 | public void close() { 102 | } 103 | 104 | protected abstract String keyOrValue(); 105 | 106 | protected abstract SchemaAndValue getSchemaAndValue(final R record); 107 | 108 | public static final class Key> extends ExtractTimestamp { 109 | @Override 110 | protected SchemaAndValue getSchemaAndValue(final R record) { 111 | return new SchemaAndValue(record.keySchema(), record.key()); 112 | } 113 | 114 | @Override 115 | protected String keyOrValue() { 116 | return "key"; 117 | } 118 | } 119 | 120 | public static final class Value> extends ExtractTimestamp { 121 | @Override 122 | protected SchemaAndValue getSchemaAndValue(final R record) { 123 | return new SchemaAndValue(record.valueSchema(), record.value()); 124 | } 125 | 126 | @Override 127 | protected String keyOrValue() { 128 | return "value"; 129 | } 130 | } 131 | 132 | } 133 | -------------------------------------------------------------------------------- /src/integration-test/java/io/aiven/kafka/connect/transforms/ConnectRunner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.io.File; 20 | import java.util.HashMap; 21 | import java.util.Map; 22 | import java.util.concurrent.ExecutionException; 23 | 24 | import org.apache.kafka.common.utils.Time; 25 | import org.apache.kafka.connect.runtime.Connect; 26 | import org.apache.kafka.connect.runtime.ConnectorConfig; 27 | import org.apache.kafka.connect.runtime.Herder; 28 | import org.apache.kafka.connect.runtime.Worker; 29 | import org.apache.kafka.connect.runtime.isolation.Plugins; 30 | import org.apache.kafka.connect.runtime.rest.RestServer; 31 | import org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo; 32 | import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; 33 | import org.apache.kafka.connect.runtime.standalone.StandaloneHerder; 34 | import org.apache.kafka.connect.storage.MemoryOffsetBackingStore; 35 | import org.apache.kafka.connect.util.Callback; 36 | import org.apache.kafka.connect.util.FutureCallback; 37 | 38 | import org.slf4j.Logger; 39 | import org.slf4j.LoggerFactory; 40 | 41 | final class ConnectRunner { 42 | private static final Logger log = LoggerFactory.getLogger(ConnectRunner.class); 43 | 44 | private final File pluginDir; 45 | private final String bootstrapServers; 46 | 47 | private Herder herder; 48 | private Connect connect; 49 | 50 | public ConnectRunner(final File pluginDir, 51 | final String bootstrapServers) { 52 | this.pluginDir = pluginDir; 53 | this.bootstrapServers = bootstrapServers; 54 | } 55 | 56 | void start() { 57 | final Map workerProps = new HashMap<>(); 58 | workerProps.put("bootstrap.servers", bootstrapServers); 59 | 60 | workerProps.put("offset.flush.interval.ms", "5000"); 61 | 62 | // These don't matter much (each connector sets its own converters), but need to be filled with valid classes. 63 | workerProps.put("key.converter", "org.apache.kafka.connect.converters.ByteArrayConverter"); 64 | workerProps.put("value.converter", "org.apache.kafka.connect.converters.ByteArrayConverter"); 65 | workerProps.put("internal.key.converter", "org.apache.kafka.connect.json.JsonConverter"); 66 | workerProps.put("internal.key.converter.schemas.enable", "false"); 67 | workerProps.put("internal.value.converter", "org.apache.kafka.connect.json.JsonConverter"); 68 | workerProps.put("internal.value.converter.schemas.enable", "false"); 69 | 70 | // Don't need it since we'll memory MemoryOffsetBackingStore. 71 | workerProps.put("offset.storage.file.filename", ""); 72 | 73 | workerProps.put("plugin.path", pluginDir.getPath()); 74 | 75 | final Time time = Time.SYSTEM; 76 | final String workerId = "test-worker"; 77 | 78 | final Plugins plugins = new Plugins(workerProps); 79 | final StandaloneConfig config = new StandaloneConfig(workerProps); 80 | 81 | final Worker worker = new Worker( 82 | workerId, time, plugins, config, new MemoryOffsetBackingStore()); 83 | herder = new StandaloneHerder(worker, "cluster-id"); 84 | 85 | final RestServer rest = new RestServer(config); 86 | 87 | connect = new Connect(herder, rest); 88 | 89 | connect.start(); 90 | } 91 | 92 | void createConnector(final Map config) throws ExecutionException, InterruptedException { 93 | assert herder != null; 94 | 95 | final FutureCallback> cb = new FutureCallback<>( 96 | new Callback>() { 97 | @Override 98 | public void onCompletion(final Throwable error, final Herder.Created info) { 99 | if (error != null) { 100 | log.error("Failed to create job"); 101 | } else { 102 | log.info("Created connector {}", info.result().name()); 103 | } 104 | } 105 | }); 106 | herder.putConnectorConfig( 107 | config.get(ConnectorConfig.NAME_CONFIG), 108 | config, false, cb 109 | ); 110 | 111 | final Herder.Created connectorInfoCreated = cb.get(); 112 | assert connectorInfoCreated.created(); 113 | } 114 | 115 | void stop() { 116 | connect.stop(); 117 | } 118 | 119 | void awaitStop() { 120 | connect.awaitStop(); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/debezium/converters/MoneyConverterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.debezium.converters; 18 | 19 | import java.math.BigDecimal; 20 | import java.util.Properties; 21 | 22 | import org.apache.kafka.connect.data.Schema; 23 | import org.apache.kafka.connect.data.SchemaBuilder; 24 | 25 | import io.aiven.kafka.connect.debezium.converters.utils.DummyRelationalColumn; 26 | import io.aiven.kafka.connect.debezium.converters.utils.MoneyTestRelationalColumn; 27 | 28 | import io.debezium.spi.converter.CustomConverter; 29 | import org.junit.jupiter.api.AfterEach; 30 | import org.junit.jupiter.api.BeforeEach; 31 | import org.junit.jupiter.api.Test; 32 | 33 | import static org.assertj.core.api.Assertions.assertThat; 34 | import static org.assertj.core.api.Assertions.assertThatThrownBy; 35 | 36 | public class MoneyConverterTest { 37 | 38 | private MoneyConverter transform; 39 | private StubConverterRegistration registration; 40 | private Properties prop; 41 | 42 | 43 | @BeforeEach 44 | void init() { 45 | transform = new MoneyConverter(); 46 | registration = new StubConverterRegistration(); 47 | prop = new Properties(); 48 | prop.setProperty("schema.name", "price"); 49 | } 50 | 51 | @AfterEach 52 | void teardown() { 53 | transform = null; 54 | registration = null; 55 | prop = null; 56 | } 57 | 58 | @Test 59 | void shouldRegisterCorrectSchema() { 60 | transform.configure(prop); 61 | assertThat(registration.currFieldSchema).isNull(); 62 | transform.converterFor(new MoneyTestRelationalColumn(), registration); 63 | 64 | assertThat(registration.currFieldSchema.schema().name()).isEqualTo("price"); 65 | assertThat(registration.currFieldSchema.schema().type()).isEqualTo(Schema.Type.STRING); 66 | } 67 | 68 | @Test 69 | void shouldDoNothingIfColumnIsNotMoney() { 70 | transform.configure(prop); 71 | 72 | transform.converterFor(new DummyRelationalColumn(), registration); 73 | 74 | assertThat(registration.currFieldSchema).isNull(); 75 | assertThat(registration.currConverter).isNull(); 76 | } 77 | 78 | @Test 79 | void shouldFormatDataToMoneyFormat() { 80 | assertThat(registration.currConverter).isNull(); 81 | transform.converterFor(new MoneyTestRelationalColumn(), registration); 82 | 83 | final String result = (String) registration.currConverter.convert(BigDecimal.valueOf(103.6999)); 84 | assertThat(result).isEqualTo("103.70"); 85 | 86 | final String result2 = (String) registration.currConverter.convert((long) 103); 87 | assertThat(result2).isEqualTo("103.00"); 88 | } 89 | 90 | @Test 91 | void shouldFailIfDataIsNotBigDecimal() { 92 | assertThat(registration.currConverter).isNull(); 93 | transform.converterFor(new MoneyTestRelationalColumn(), registration); 94 | 95 | assertThatThrownBy(() -> registration.currConverter.convert("103.6999")) 96 | .isInstanceOf(IllegalArgumentException.class) 97 | .hasMessage("Money type should have BigDecimal type"); 98 | } 99 | 100 | @Test 101 | void shouldFailIfDataIsMissing() { 102 | assertThat(registration.currConverter).isNull(); 103 | transform.converterFor(new MoneyTestRelationalColumn(), registration); 104 | 105 | assertThatThrownBy(() -> registration.currConverter.convert(null)) 106 | .isInstanceOf(IllegalArgumentException.class) 107 | .hasMessage("Money column is not optional, but data is null"); 108 | } 109 | 110 | @Test 111 | void shouldDoNothingIfColumnIsOptional() { 112 | transform.configure(prop); 113 | final MoneyTestRelationalColumn moneyColumn = new MoneyTestRelationalColumn(); 114 | moneyColumn.isOptional = true; 115 | 116 | transform.converterFor(moneyColumn, registration); 117 | 118 | final String result = (String) registration.currConverter.convert(null); 119 | assertThat(result).isNull(); 120 | } 121 | 122 | class StubConverterRegistration implements CustomConverter.ConverterRegistration { 123 | SchemaBuilder currFieldSchema; 124 | CustomConverter.Converter currConverter; 125 | 126 | @Override 127 | public void register(final SchemaBuilder fieldSchema, 128 | final CustomConverter.Converter converter) { 129 | currFieldSchema = fieldSchema; 130 | currConverter = converter; 131 | } 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/ExtractTopicFromSchemaNameTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | import org.apache.kafka.common.record.TimestampType; 23 | import org.apache.kafka.connect.data.Schema; 24 | import org.apache.kafka.connect.data.SchemaBuilder; 25 | import org.apache.kafka.connect.sink.SinkRecord; 26 | 27 | import org.junit.jupiter.api.Test; 28 | 29 | import static org.junit.jupiter.api.Assertions.assertEquals; 30 | 31 | public class ExtractTopicFromSchemaNameTest { 32 | @Test 33 | void emptyConfigsValueSchemaNameToTopicTest() { 34 | 35 | final Schema keySchema = SchemaBuilder.struct().keySchema(); 36 | final Schema valueSchema = SchemaBuilder.struct().name("com.acme.schema.SchemaNameToTopic").schema(); 37 | final SinkRecord originalRecord = record(keySchema, "key", valueSchema, "{}"); 38 | final SinkRecord transformedRecord = transformation(new HashMap<>()).apply(originalRecord); 39 | assertEquals("com.acme.schema.SchemaNameToTopic", transformedRecord.topic()); 40 | 41 | } 42 | 43 | @Test 44 | void configMapValueSchemaNameToTopicTest() { 45 | final Map configs = new HashMap<>(); 46 | configs.put(ExtractTopicFromSchemaNameConfig.SCHEMA_NAME_TO_TOPIC_MAP, 47 | "com.acme.schema.SchemaNameToTopic1:TheNameToReplace1," 48 | + "com.acme.schema.SchemaNameToTopic2:TheNameToReplace2," 49 | + "com.acme.schema.SchemaNameToTopic3:TheNameToReplace3" 50 | ); 51 | final Schema keySchema = SchemaBuilder.struct().keySchema(); 52 | final Schema valueSchema = SchemaBuilder.struct().name("com.acme.schema.SchemaNameToTopic1").schema(); 53 | final SinkRecord originalRecord = record(keySchema, "key", valueSchema, "{}"); 54 | final SinkRecord transformedRecord = transformation(configs).apply(originalRecord); 55 | assertEquals("TheNameToReplace1", transformedRecord.topic()); 56 | 57 | final Schema valueSchema2 = SchemaBuilder.struct().name("com.acme.schema.SchemaNameToTopic3").schema(); 58 | final SinkRecord originalRecord2 = record(keySchema, "key", valueSchema2, "{}"); 59 | final SinkRecord transformedRecord2 = transformation(configs).apply(originalRecord2); 60 | assertEquals("TheNameToReplace3", transformedRecord2.topic()); 61 | 62 | } 63 | 64 | @Test 65 | void regexConfigValueAfterLastDotToTopicTest() { 66 | final Map configs = new HashMap<>(); 67 | // pass regegx that will parse the class name after last dot 68 | configs.put(ExtractTopicFromSchemaNameConfig.REGEX_SCHEMA_NAME_TO_TOPIC, 69 | "(?:[.]|^)([^.]*)$"); 70 | final Schema keySchema = SchemaBuilder.struct().keySchema(); 71 | final Schema valueSchema = SchemaBuilder.struct().name("com.acme.schema.SchemaNameToTopic").schema(); 72 | final SinkRecord originalRecord = record(keySchema, "key", valueSchema, "{}"); 73 | final SinkRecord transformedRecord = transformation(configs).apply(originalRecord); 74 | assertEquals("SchemaNameToTopic", transformedRecord.topic()); 75 | 76 | } 77 | 78 | @Test 79 | void regexNoMatchToTopicTest() { 80 | final Map configs = new HashMap<>(); 81 | // pass regegx that will parse the class name after last dot 82 | configs.put(ExtractTopicFromSchemaNameConfig.REGEX_SCHEMA_NAME_TO_TOPIC, 83 | "/[^;]*/"); 84 | final Schema keySchema = SchemaBuilder.struct().keySchema(); 85 | final Schema valueSchema = SchemaBuilder.struct().name("com.acme.schema.SchemaNameToTopic").schema(); 86 | final SinkRecord originalRecord = record(keySchema, "key", valueSchema, "{}"); 87 | final SinkRecord transformedRecord = transformation(configs).apply(originalRecord); 88 | assertEquals("com.acme.schema.SchemaNameToTopic", transformedRecord.topic()); 89 | 90 | } 91 | 92 | private ExtractTopicFromSchemaName transformation(final Map configs) { 93 | final ExtractTopicFromSchemaName transform = new ExtractTopicFromSchemaName.Value<>(); 94 | transform.configure(configs); 95 | return transform; 96 | } 97 | 98 | protected SinkRecord record(final Schema keySchema, 99 | final Object key, 100 | final Schema valueSchema, 101 | final Object value) { 102 | return new SinkRecord("original_topic", 0, 103 | keySchema, key, 104 | valueSchema, value, 105 | 123L, 106 | 456L, TimestampType.CREATE_TIME); 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | opensource@aiven.io. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/transforms/ConcatFields.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | import java.util.StringJoiner; 22 | 23 | import org.apache.kafka.common.config.ConfigDef; 24 | import org.apache.kafka.connect.connector.ConnectRecord; 25 | import org.apache.kafka.connect.data.Schema; 26 | import org.apache.kafka.connect.data.SchemaAndValue; 27 | import org.apache.kafka.connect.data.SchemaBuilder; 28 | import org.apache.kafka.connect.data.Struct; 29 | import org.apache.kafka.connect.errors.DataException; 30 | import org.apache.kafka.connect.transforms.Transformation; 31 | 32 | import org.slf4j.Logger; 33 | import org.slf4j.LoggerFactory; 34 | 35 | public abstract class ConcatFields> implements Transformation { 36 | private ConcatFieldsConfig config; 37 | private static final Logger log = LoggerFactory.getLogger(ConcatFields.class); 38 | 39 | protected abstract String dataPlace(); 40 | 41 | protected abstract SchemaAndValue getSchemaAndValue(final R record); 42 | 43 | protected abstract R createNewRecord(final R record, final Schema newSchema, final Object newValue); 44 | 45 | @Override 46 | public ConfigDef config() { 47 | return ConcatFieldsConfig.config(); 48 | } 49 | 50 | @Override 51 | public void configure(final Map configs) { 52 | this.config = new ConcatFieldsConfig(configs); 53 | } 54 | 55 | @Override 56 | public R apply(final R record) { 57 | final SchemaAndValue schemaAndValue = getSchemaAndValue(record); 58 | final SchemaBuilder newSchema = SchemaBuilder.struct(); 59 | 60 | if (schemaAndValue.value() == null) { 61 | throw new DataException(dataPlace() + " Value can't be null: " + record); 62 | } 63 | 64 | final R newRecord; 65 | 66 | if (schemaAndValue.value() instanceof Struct) { 67 | final Struct struct = (Struct) schemaAndValue.value(); 68 | final StringJoiner outputValue = new StringJoiner(config.delimiter()); 69 | 70 | if (schemaAndValue.schema() != null) { 71 | schemaAndValue.schema().fields().forEach(field -> newSchema.field(field.name(), field.schema())); 72 | } else { 73 | struct.schema().fields().forEach(field -> newSchema.field(field.name(), field.schema())); 74 | } 75 | newSchema.field(config.outputFieldName(), Schema.OPTIONAL_STRING_SCHEMA); 76 | final Struct newStruct = new Struct(newSchema.build()); 77 | struct.schema().fields().forEach(field -> { 78 | newStruct.put(field.name(), struct.get(field)); 79 | }); 80 | config.fieldNames().forEach(field -> { 81 | try { 82 | if (struct.get(field) == null) { 83 | outputValue.add(config.fieldReplaceMissing()); 84 | } else { 85 | outputValue.add(struct.get(field).toString()); 86 | } 87 | } catch (final DataException e) { 88 | log.debug("{} is missing, concat will use {}", field, config.fieldReplaceMissing()); 89 | outputValue.add(config.fieldReplaceMissing()); 90 | } 91 | }); 92 | newStruct.put(config.outputFieldName(), outputValue.toString()); 93 | newRecord = createNewRecord(record, newSchema.build(), newStruct); 94 | } else if (schemaAndValue.value() instanceof Map) { 95 | final Map newValue = new HashMap<>((Map) schemaAndValue.value()); 96 | final StringJoiner outputValue = new StringJoiner(config.delimiter()); 97 | config.fieldNames().forEach(field -> { 98 | if (newValue.get(field) == null) { 99 | outputValue.add(config.fieldReplaceMissing()); 100 | } else { 101 | outputValue.add(newValue.get(field).toString()); 102 | } 103 | }); 104 | newValue.put(config.outputFieldName(), outputValue.toString()); 105 | 106 | //if we have a schema, we can add the new field to it, otherwise just keep the schema null 107 | if (schemaAndValue.schema() != null) { 108 | schemaAndValue.schema().fields().forEach(field -> newSchema.field(field.name(), field.schema())); 109 | newSchema.field(config.outputFieldName(), Schema.OPTIONAL_STRING_SCHEMA); 110 | newRecord = createNewRecord(record, newSchema.build(), newValue); 111 | } else { 112 | newRecord = createNewRecord(record, null, newValue); 113 | } 114 | } else { 115 | throw new DataException("Value type must be STRUCT or MAP: " + record); 116 | } 117 | 118 | return newRecord; 119 | } 120 | 121 | public static class Key> extends ConcatFields { 122 | @Override 123 | protected SchemaAndValue getSchemaAndValue(final R record) { 124 | return new SchemaAndValue(record.keySchema(), record.key()); 125 | } 126 | 127 | @Override 128 | protected R createNewRecord(final R record, final Schema newSchema, final Object newValue) { 129 | return record.newRecord( 130 | record.topic(), 131 | record.kafkaPartition(), 132 | newSchema, 133 | newValue, 134 | record.valueSchema(), 135 | record.value(), 136 | record.timestamp(), 137 | record.headers() 138 | ); 139 | } 140 | 141 | @Override 142 | protected String dataPlace() { 143 | return "key"; 144 | } 145 | } 146 | 147 | public static class Value> extends ConcatFields { 148 | @Override 149 | protected SchemaAndValue getSchemaAndValue(final R record) { 150 | return new SchemaAndValue(record.valueSchema(), record.value()); 151 | } 152 | 153 | @Override 154 | protected R createNewRecord(final R record, final Schema newSchema, final Object newValue) { 155 | return record.newRecord( 156 | record.topic(), 157 | record.kafkaPartition(), 158 | record.keySchema(), 159 | record.key(), 160 | newSchema, 161 | newValue, 162 | record.timestamp(), 163 | record.headers() 164 | ); 165 | } 166 | 167 | @Override 168 | protected String dataPlace() { 169 | return "value"; 170 | } 171 | } 172 | 173 | @Override 174 | public void close() { 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/transforms/FilterByFieldValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Map; 20 | import java.util.Optional; 21 | import java.util.function.Predicate; 22 | import java.util.regex.Pattern; 23 | 24 | import org.apache.kafka.common.config.AbstractConfig; 25 | import org.apache.kafka.common.config.ConfigDef; 26 | import org.apache.kafka.common.config.ConfigException; 27 | import org.apache.kafka.connect.connector.ConnectRecord; 28 | import org.apache.kafka.connect.data.Field; 29 | import org.apache.kafka.connect.data.Schema; 30 | import org.apache.kafka.connect.data.SchemaAndValue; 31 | import org.apache.kafka.connect.data.Struct; 32 | import org.apache.kafka.connect.data.Values; 33 | import org.apache.kafka.connect.transforms.Transformation; 34 | 35 | public abstract class FilterByFieldValue> implements Transformation { 36 | 37 | private String fieldName; 38 | private Optional fieldExpectedValue; 39 | private Optional fieldValuePattern; 40 | 41 | @Override 42 | public ConfigDef config() { 43 | return new ConfigDef() 44 | .define("field.name", 45 | ConfigDef.Type.STRING, 46 | null, 47 | ConfigDef.Importance.HIGH, 48 | "The field name to filter by." 49 | + "Schema-based records (Avro), schemaless (e.g. JSON), and raw values are supported." 50 | + "If empty, the whole key/value record will be filtered.") 51 | .define("field.value", 52 | ConfigDef.Type.STRING, 53 | null, 54 | ConfigDef.Importance.HIGH, 55 | "Expected value to match. Either define this, or a regex pattern") 56 | .define("field.value.pattern", 57 | ConfigDef.Type.STRING, 58 | null, 59 | ConfigDef.Importance.HIGH, 60 | "The pattern to match. Either define this, or an expected value") 61 | .define("field.value.matches", 62 | ConfigDef.Type.BOOLEAN, 63 | true, 64 | ConfigDef.Importance.MEDIUM, 65 | "The filter mode, 'true' for matching or 'false' for non-matching"); 66 | } 67 | 68 | @Override 69 | public void configure(final Map configs) { 70 | final AbstractConfig config = new AbstractConfig(config(), configs); 71 | this.fieldName = config.getString("field.name"); 72 | this.fieldExpectedValue = Optional.ofNullable(config.getString("field.value")); 73 | this.fieldValuePattern = Optional.ofNullable(config.getString("field.value.pattern")); 74 | final boolean expectedValuePresent = fieldExpectedValue.isPresent(); 75 | final boolean regexPatternPresent = fieldValuePattern.map(s -> !s.isEmpty()).orElse(false); 76 | if (expectedValuePresent == regexPatternPresent) { 77 | throw new ConfigException( 78 | "Either field.value or field.value.pattern have to be set to apply filter transform"); 79 | } 80 | final Predicate matchCondition; 81 | 82 | if (expectedValuePresent) { 83 | final SchemaAndValue expectedSchemaAndValue = Values.parseString(fieldExpectedValue.get()); 84 | matchCondition = schemaAndValue -> expectedSchemaAndValue.value().equals(schemaAndValue.value()); 85 | } else { 86 | final String pattern = fieldValuePattern.get(); 87 | final Predicate regexPredicate = Pattern.compile(pattern).asPredicate(); 88 | matchCondition = schemaAndValue -> 89 | schemaAndValue != null 90 | && regexPredicate.test(Values.convertToString(schemaAndValue.schema(), schemaAndValue.value())); 91 | } 92 | 93 | this.filterCondition = config.getBoolean("field.value.matches") 94 | ? matchCondition 95 | : (result -> !matchCondition.test(result)); 96 | } 97 | 98 | private Predicate filterCondition; 99 | 100 | protected abstract Schema operatingSchema(R record); 101 | 102 | protected abstract Object operatingValue(R record); 103 | 104 | @Override 105 | public R apply(final R record) { 106 | if (operatingValue(record) == null) { 107 | return record; 108 | } 109 | 110 | if (operatingSchema(record) == null) { 111 | return applySchemaless(record); 112 | } else { 113 | return applyWithSchema(record); 114 | } 115 | } 116 | 117 | private R applyWithSchema(final R record) { 118 | final Struct struct = (Struct) operatingValue(record); 119 | final SchemaAndValue schemaAndValue = getStructFieldValue(struct, fieldName).orElse(null); 120 | return filterCondition.test(schemaAndValue) ? record : null; 121 | } 122 | 123 | private Optional getStructFieldValue(final Struct struct, final String fieldName) { 124 | final Schema schema = struct.schema(); 125 | final Field field = schema.field(fieldName); 126 | final Object fieldValue = struct.get(field); 127 | if (fieldValue == null) { 128 | return Optional.empty(); 129 | } else { 130 | return Optional.of(new SchemaAndValue(field.schema(), struct.get(field))); 131 | } 132 | } 133 | 134 | @SuppressWarnings("unchecked") 135 | private R applySchemaless(final R record) { 136 | if (fieldName == null || fieldName.isEmpty()) { 137 | final SchemaAndValue schemaAndValue = getSchemalessFieldValue(operatingValue(record)).orElse(null); 138 | return filterCondition.test(schemaAndValue) ? record : null; 139 | } else { 140 | final Map map = (Map) operatingValue(record); 141 | final SchemaAndValue schemaAndValue = getSchemalessFieldValue(map.get(fieldName)).orElse(null); 142 | return filterCondition.test(schemaAndValue) ? record : null; 143 | } 144 | } 145 | 146 | private Optional getSchemalessFieldValue(final Object fieldValue) { 147 | if (fieldValue == null) { 148 | return Optional.empty(); 149 | } 150 | return Optional.of(new SchemaAndValue(Values.inferSchema(fieldValue), fieldValue)); 151 | } 152 | 153 | @Override 154 | public void close() { 155 | } 156 | 157 | public static final class Key> extends FilterByFieldValue { 158 | 159 | @Override 160 | protected Schema operatingSchema(final R record) { 161 | return record.keySchema(); 162 | } 163 | 164 | @Override 165 | protected Object operatingValue(final R record) { 166 | return record.key(); 167 | } 168 | } 169 | 170 | public static final class Value> extends FilterByFieldValue { 171 | 172 | @Override 173 | protected Schema operatingSchema(final R record) { 174 | return record.valueSchema(); 175 | } 176 | 177 | @Override 178 | protected Object operatingValue(final R record) { 179 | return record.value(); 180 | } 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/transforms/CaseTransform.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.HashMap; 20 | import java.util.Locale; 21 | import java.util.Map; 22 | import java.util.function.UnaryOperator; 23 | 24 | import org.apache.kafka.common.config.ConfigDef; 25 | import org.apache.kafka.connect.connector.ConnectRecord; 26 | import org.apache.kafka.connect.data.Schema; 27 | import org.apache.kafka.connect.data.SchemaAndValue; 28 | import org.apache.kafka.connect.data.Struct; 29 | import org.apache.kafka.connect.errors.ConnectException; 30 | import org.apache.kafka.connect.errors.DataException; 31 | import org.apache.kafka.connect.transforms.Transformation; 32 | 33 | import org.slf4j.Logger; 34 | import org.slf4j.LoggerFactory; 35 | 36 | /** 37 | * Transform field value case based on the configuration. 38 | * Supports maps and structs. 39 | * @param ConnectRecord 40 | */ 41 | public abstract class CaseTransform> implements Transformation { 42 | 43 | private static final Logger LOGGER = LoggerFactory.getLogger(CaseTransform.class); 44 | 45 | /** 46 | * The configuration for case transform. 47 | */ 48 | private CaseTransformConfig config; 49 | 50 | /** 51 | * Configured transform operation. 52 | */ 53 | private UnaryOperator caseTransformFunc; 54 | 55 | protected abstract String dataPlace(); 56 | 57 | protected abstract SchemaAndValue getSchemaAndValue(final R record); 58 | 59 | protected abstract R createNewRecord(final R record, final Schema newSchema, final Object newValue); 60 | 61 | /** 62 | * Apply the case transformation to given new struct from the original struct field. 63 | * @param newStruct New struct 64 | * @param fieldName The field name to case transform 65 | */ 66 | private void applyStruct(final Struct newStruct, final String fieldName) { 67 | try { 68 | final Object value = newStruct.get(fieldName); 69 | if (value == null) { 70 | newStruct.put(fieldName, null); 71 | return; 72 | } 73 | newStruct.put(fieldName, caseTransformFunc.apply(value.toString())); 74 | } catch (final DataException e) { 75 | LOGGER.debug("{} is missing, cannot transform the case", fieldName); 76 | } 77 | } 78 | 79 | /** 80 | * Apply the case transformation to given map from the map field. 81 | * @param newValue The mutable map 82 | * @param fieldName The field name to case transform 83 | */ 84 | private void applyMap(final Map newValue, final String fieldName) { 85 | final Object value = newValue.get(fieldName); 86 | if (value == null) { 87 | newValue.put(fieldName, null); 88 | return; 89 | } 90 | newValue.put(fieldName, caseTransformFunc.apply(newValue.get(fieldName).toString())); 91 | } 92 | 93 | @Override 94 | public R apply(final R record) { 95 | final SchemaAndValue schemaAndValue = getSchemaAndValue(record); 96 | 97 | if (schemaAndValue.value() == null) { 98 | throw new DataException(dataPlace() + " Value can't be null: " + record); 99 | } 100 | 101 | final R newRecord; 102 | 103 | if (schemaAndValue.value() instanceof Struct) { 104 | final Struct struct = (Struct) schemaAndValue.value(); 105 | final Struct newStruct = new Struct(struct.schema()); 106 | struct.schema().fields().forEach(field -> { 107 | newStruct.put(field.name(), struct.get(field)); 108 | }); 109 | config.fieldNames().forEach(field -> { 110 | applyStruct(newStruct, field); 111 | }); 112 | newRecord = createNewRecord(record, struct.schema(), newStruct); 113 | } else if (schemaAndValue.value() instanceof Map) { 114 | final Map newValue = new HashMap<>((Map) schemaAndValue.value()); 115 | config.fieldNames().forEach(field -> { 116 | applyMap(newValue, field); 117 | }); 118 | //if we have a schema, use it, otherwise leave null. 119 | if (schemaAndValue.schema() != null) { 120 | newRecord = createNewRecord(record, schemaAndValue.schema(), newValue); 121 | } else { 122 | newRecord = createNewRecord(record, null, newValue); 123 | } 124 | } else { 125 | throw new DataException("Value type must be STRUCT or MAP: " + record); 126 | } 127 | return newRecord; 128 | } 129 | 130 | 131 | @Override 132 | public void close() { 133 | // no-op 134 | } 135 | 136 | @Override 137 | public ConfigDef config() { 138 | return CaseTransformConfig.config(); 139 | } 140 | 141 | @Override 142 | public void configure(final Map settings) { 143 | this.config = new CaseTransformConfig(settings); 144 | 145 | switch (config.transformCase()) { 146 | case LOWER: 147 | caseTransformFunc = value -> value.toLowerCase(Locale.ROOT); 148 | break; 149 | case UPPER: 150 | caseTransformFunc = value -> value.toUpperCase(Locale.ROOT); 151 | break; 152 | default: 153 | throw new ConnectException("Unknown case transform function " + config.transformCase()); 154 | } 155 | } 156 | 157 | /** 158 | * Record key transform implementation. 159 | * @param ConnectRecord 160 | */ 161 | public static class Key> extends CaseTransform { 162 | @Override 163 | protected SchemaAndValue getSchemaAndValue(final R record) { 164 | return new SchemaAndValue(record.keySchema(), record.key()); 165 | } 166 | 167 | @Override 168 | protected R createNewRecord(final R record, final Schema newSchema, final Object newValue) { 169 | return record.newRecord( 170 | record.topic(), 171 | record.kafkaPartition(), 172 | newSchema, 173 | newValue, 174 | record.valueSchema(), 175 | record.value(), 176 | record.timestamp(), 177 | record.headers() 178 | ); 179 | } 180 | 181 | @Override 182 | protected String dataPlace() { 183 | return "key"; 184 | } 185 | } 186 | 187 | /** 188 | * Record value transform implementation. 189 | * @param ConnectRecord 190 | */ 191 | public static class Value> extends CaseTransform { 192 | @Override 193 | protected SchemaAndValue getSchemaAndValue(final R record) { 194 | return new SchemaAndValue(record.valueSchema(), record.value()); 195 | } 196 | 197 | @Override 198 | protected R createNewRecord(final R record, final Schema newSchema, final Object newValue) { 199 | return record.newRecord( 200 | record.topic(), 201 | record.kafkaPartition(), 202 | record.keySchema(), 203 | record.key(), 204 | newSchema, 205 | newValue, 206 | record.timestamp(), 207 | record.headers() 208 | ); 209 | } 210 | 211 | @Override 212 | protected String dataPlace() { 213 | return "value"; 214 | } 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | # 21 | # Gradle start up script for POSIX generated by Gradle. 22 | # 23 | # Important for running: 24 | # 25 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 26 | # noncompliant, but you have some other compliant shell such as ksh or 27 | # bash, then to run this script, type that shell name before the whole 28 | # command line, like: 29 | # 30 | # ksh Gradle 31 | # 32 | # Busybox and similar reduced shells will NOT work, because this script 33 | # requires all of these POSIX shell features: 34 | # * functions; 35 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 36 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 37 | # * compound commands having a testable exit status, especially «case»; 38 | # * various built-in commands including «command», «set», and «ulimit». 39 | # 40 | # Important for patching: 41 | # 42 | # (2) This script targets any POSIX shell, so it avoids extensions provided 43 | # by Bash, Ksh, etc; in particular arrays are avoided. 44 | # 45 | # The "traditional" practice of packing multiple parameters into a 46 | # space-separated string is a well documented source of bugs and security 47 | # problems, so this is (mostly) avoided, by progressively accumulating 48 | # options in "$@", and eventually passing that to Java. 49 | # 50 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 51 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 52 | # see the in-line comments for details. 53 | # 54 | # There are tweaks for specific operating systems such as AIX, CygWin, 55 | # Darwin, MinGW, and NonStop. 56 | # 57 | # (3) This script is generated from the Groovy template 58 | # https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 59 | # within the Gradle project. 60 | # 61 | # You can find Gradle at https://github.com/gradle/gradle/. 62 | # 63 | ############################################################################## 64 | 65 | # Attempt to set APP_HOME 66 | 67 | # Resolve links: $0 may be a link 68 | app_path=$0 69 | 70 | # Need this for daisy-chained symlinks. 71 | while 72 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 73 | [ -h "$app_path" ] 74 | do 75 | ls=$( ls -ld "$app_path" ) 76 | link=${ls#*' -> '} 77 | case $link in #( 78 | /*) app_path=$link ;; #( 79 | *) app_path=$APP_HOME$link ;; 80 | esac 81 | done 82 | 83 | APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit 84 | 85 | APP_NAME="Gradle" 86 | APP_BASE_NAME=${0##*/} 87 | 88 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 89 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 90 | 91 | # Use the maximum available, or set MAX_FD != -1 to use that value. 92 | MAX_FD=maximum 93 | 94 | warn () { 95 | echo "$*" 96 | } >&2 97 | 98 | die () { 99 | echo 100 | echo "$*" 101 | echo 102 | exit 1 103 | } >&2 104 | 105 | # OS specific support (must be 'true' or 'false'). 106 | cygwin=false 107 | msys=false 108 | darwin=false 109 | nonstop=false 110 | case "$( uname )" in #( 111 | CYGWIN* ) cygwin=true ;; #( 112 | Darwin* ) darwin=true ;; #( 113 | MSYS* | MINGW* ) msys=true ;; #( 114 | NONSTOP* ) nonstop=true ;; 115 | esac 116 | 117 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 118 | 119 | 120 | # Determine the Java command to use to start the JVM. 121 | if [ -n "$JAVA_HOME" ] ; then 122 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 123 | # IBM's JDK on AIX uses strange locations for the executables 124 | JAVACMD=$JAVA_HOME/jre/sh/java 125 | else 126 | JAVACMD=$JAVA_HOME/bin/java 127 | fi 128 | if [ ! -x "$JAVACMD" ] ; then 129 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 130 | 131 | Please set the JAVA_HOME variable in your environment to match the 132 | location of your Java installation." 133 | fi 134 | else 135 | JAVACMD=java 136 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 137 | 138 | Please set the JAVA_HOME variable in your environment to match the 139 | location of your Java installation." 140 | fi 141 | 142 | # Increase the maximum file descriptors if we can. 143 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 144 | case $MAX_FD in #( 145 | max*) 146 | MAX_FD=$( ulimit -H -n ) || 147 | warn "Could not query maximum file descriptor limit" 148 | esac 149 | case $MAX_FD in #( 150 | '' | soft) :;; #( 151 | *) 152 | ulimit -n "$MAX_FD" || 153 | warn "Could not set maximum file descriptor limit to $MAX_FD" 154 | esac 155 | fi 156 | 157 | # Collect all arguments for the java command, stacking in reverse order: 158 | # * args from the command line 159 | # * the main class name 160 | # * -classpath 161 | # * -D...appname settings 162 | # * --module-path (only if needed) 163 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 164 | 165 | # For Cygwin or MSYS, switch paths to Windows format before running java 166 | if "$cygwin" || "$msys" ; then 167 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 168 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 169 | 170 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 171 | 172 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 173 | for arg do 174 | if 175 | case $arg in #( 176 | -*) false ;; # don't mess with options #( 177 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 178 | [ -e "$t" ] ;; #( 179 | *) false ;; 180 | esac 181 | then 182 | arg=$( cygpath --path --ignore --mixed "$arg" ) 183 | fi 184 | # Roll the args list around exactly as many times as the number of 185 | # args, so each arg winds up back in the position where it started, but 186 | # possibly modified. 187 | # 188 | # NB: a `for` loop captures its iteration list before it begins, so 189 | # changing the positional parameters here affects neither the number of 190 | # iterations, nor the values presented in `arg`. 191 | shift # remove old arg 192 | set -- "$@" "$arg" # push replacement arg 193 | done 194 | fi 195 | 196 | # Collect all arguments for the java command; 197 | # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of 198 | # shell script including quotes and variable substitutions, so put them in 199 | # double quotes to make sure that they get re-expanded; and 200 | # * put everything else in single quotes, so that it's not re-expanded. 201 | 202 | set -- \ 203 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 204 | -classpath "$CLASSPATH" \ 205 | org.gradle.wrapper.GradleWrapperMain \ 206 | "$@" 207 | 208 | # Use "xargs" to parse quoted args. 209 | # 210 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 211 | # 212 | # In Bash we could simply go: 213 | # 214 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 215 | # set -- "${ARGS[@]}" "$@" 216 | # 217 | # but POSIX shell has neither arrays nor command substitution, so instead we 218 | # post-process each arg (as a line of input to sed) to backslash-escape any 219 | # character that might be a shell metacharacter, then use eval to reverse 220 | # that process (while maintaining the separation between arguments), and wrap 221 | # the whole thing up as a single "set" statement. 222 | # 223 | # This will of course break if any of these variables contains a newline or 224 | # an unmatched quote. 225 | # 226 | 227 | eval "set -- $( 228 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 229 | xargs -n1 | 230 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 231 | tr '\n' ' ' 232 | )" '"$@"' 233 | 234 | exec "$JAVACMD" "$@" 235 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/transforms/Hash.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.security.MessageDigest; 20 | import java.security.NoSuchAlgorithmException; 21 | import java.util.Map; 22 | import java.util.Optional; 23 | 24 | import org.apache.kafka.common.config.ConfigDef; 25 | import org.apache.kafka.connect.connector.ConnectRecord; 26 | import org.apache.kafka.connect.data.Field; 27 | import org.apache.kafka.connect.data.Schema; 28 | import org.apache.kafka.connect.data.SchemaAndValue; 29 | import org.apache.kafka.connect.data.Struct; 30 | import org.apache.kafka.connect.errors.ConnectException; 31 | import org.apache.kafka.connect.errors.DataException; 32 | import org.apache.kafka.connect.transforms.Transformation; 33 | 34 | import io.aiven.kafka.connect.transforms.utils.Hex; 35 | 36 | import org.slf4j.Logger; 37 | import org.slf4j.LoggerFactory; 38 | 39 | 40 | public abstract class Hash> implements Transformation { 41 | private static final Logger log = LoggerFactory.getLogger(Hash.class); 42 | 43 | private HashConfig config; 44 | private MessageDigest messageDigest; 45 | 46 | @Override 47 | public ConfigDef config() { 48 | return HashConfig.config(); 49 | } 50 | 51 | @Override 52 | public void configure(final Map configs) { 53 | this.config = new HashConfig(configs); 54 | 55 | try { 56 | switch (config.hashFunction()) { 57 | case MD5: 58 | messageDigest = MessageDigest.getInstance("MD5"); 59 | break; 60 | case SHA1: 61 | messageDigest = MessageDigest.getInstance("SHA1"); 62 | break; 63 | case SHA256: 64 | messageDigest = MessageDigest.getInstance("SHA-256"); 65 | break; 66 | default: 67 | throw new ConnectException("Unknown hash function " + config.hashFunction()); 68 | } 69 | } catch (final NoSuchAlgorithmException e) { 70 | throw new ConnectException(e); 71 | } 72 | } 73 | 74 | public final Optional getNewValue(final R record, final SchemaAndValue schemaAndValue) { 75 | final Optional newValue; 76 | if (config.fieldName().isPresent()) { 77 | newValue = getNewValueForNamedField( 78 | record.toString(), schemaAndValue.schema(), schemaAndValue.value(), config.fieldName().get()); 79 | } else { 80 | newValue = getNewValueWithoutFieldName( 81 | record.toString(), schemaAndValue.schema(), schemaAndValue.value()); 82 | } 83 | return newValue; 84 | } 85 | 86 | @Override 87 | public void close() { 88 | } 89 | 90 | protected abstract String dataPlace(); 91 | 92 | protected abstract SchemaAndValue getSchemaAndValue(final R record); 93 | 94 | private Optional getNewValueForNamedField(final String recordStr, 95 | final Schema schema, 96 | final Object value, 97 | final String fieldName) { 98 | if (Schema.Type.STRUCT != schema.type()) { 99 | throw new DataException(dataPlace() + " schema type must be STRUCT if field name is specified: " 100 | + recordStr); 101 | } 102 | 103 | if (value == null) { 104 | throw new DataException(dataPlace() + " can't be null if field name is specified: " + recordStr); 105 | } 106 | 107 | final Field field = schema.field(fieldName); 108 | if (field == null) { 109 | if (config.skipMissingOrNull()) { 110 | log.debug(fieldName + " in " + dataPlace() + " schema is missing, skipping transformation"); 111 | return Optional.empty(); 112 | } else { 113 | throw new DataException(fieldName + " in " + dataPlace() + " schema can't be missing: " + recordStr); 114 | } 115 | } 116 | 117 | if (field.schema().type() != Schema.Type.STRING) { 118 | throw new DataException(fieldName + " schema type in " + dataPlace() 119 | + " must be " + Schema.Type.STRING 120 | + ": " + recordStr); 121 | } 122 | 123 | final Struct struct = (Struct) value; 124 | final String stringValue = struct.getString(fieldName); 125 | if (stringValue == null) { 126 | if (config.skipMissingOrNull()) { 127 | log.debug(fieldName + " in " + dataPlace() + " is null, skipping transformation"); 128 | return Optional.empty(); 129 | } else { 130 | throw new DataException(fieldName + " in " + dataPlace() + " can't be null: " + recordStr); 131 | } 132 | } else { 133 | final String updatedValue = hashString(stringValue); 134 | final Struct updatedRecord = struct.put(fieldName, updatedValue); 135 | return Optional.ofNullable(updatedRecord); 136 | } 137 | } 138 | 139 | private Optional getNewValueWithoutFieldName(final String recordStr, 140 | final Schema schema, 141 | final Object value) { 142 | if (schema.type() != Schema.Type.STRING) { 143 | throw new DataException(dataPlace() + " schema type must be " 144 | + Schema.Type.STRING 145 | + " if field name is not specified: " 146 | + recordStr); 147 | } 148 | 149 | if (value == null) { 150 | if (config.skipMissingOrNull()) { 151 | log.debug(dataPlace() + " is null, skipping transformation"); 152 | return Optional.empty(); 153 | } else { 154 | throw new DataException(dataPlace() + " can't be null: " + recordStr); 155 | } 156 | } 157 | 158 | return Optional.of(hashString(value.toString())); 159 | } 160 | 161 | private String hashString(final String string) { 162 | // We don't call reset() here because digest() does resetting afterwards. 163 | final byte[] digest = messageDigest.digest(string.getBytes()); 164 | return Hex.encode(digest); 165 | } 166 | 167 | public static class Key> extends Hash { 168 | @Override 169 | protected SchemaAndValue getSchemaAndValue(final R record) { 170 | return new SchemaAndValue(record.keySchema(), record.key()); 171 | } 172 | 173 | @Override 174 | public R apply(final R record) { 175 | final SchemaAndValue schemaAndValue = getSchemaAndValue(record); 176 | if (schemaAndValue.schema() == null) { 177 | throw new DataException(dataPlace() + " schema can't be null: " + record); 178 | } 179 | 180 | final Optional newValue = getNewValue(record, schemaAndValue); 181 | 182 | if (newValue.isPresent()) { 183 | return record.newRecord( 184 | record.topic(), 185 | record.kafkaPartition(), 186 | record.keySchema(), 187 | newValue.get(), 188 | record.valueSchema(), 189 | record.value(), 190 | record.timestamp(), 191 | record.headers() 192 | ); 193 | } else { 194 | return record; 195 | } 196 | } 197 | 198 | @Override 199 | protected String dataPlace() { 200 | return "key"; 201 | } 202 | } 203 | 204 | public static class Value> extends Hash { 205 | @Override 206 | protected SchemaAndValue getSchemaAndValue(final R record) { 207 | return new SchemaAndValue(record.valueSchema(), record.value()); 208 | } 209 | 210 | @Override 211 | public R apply(final R record) { 212 | final SchemaAndValue schemaAndValue = getSchemaAndValue(record); 213 | if (schemaAndValue.schema() == null) { 214 | throw new DataException(dataPlace() + " schema can't be null: " + record); 215 | } 216 | 217 | final Optional newValue = getNewValue(record, schemaAndValue); 218 | 219 | if (newValue.isPresent()) { 220 | return record.newRecord( 221 | record.topic(), 222 | record.kafkaPartition(), 223 | record.keySchema(), 224 | record.key(), 225 | record.valueSchema(), 226 | newValue.get(), 227 | record.timestamp(), 228 | record.headers() 229 | ); 230 | } else { 231 | return record; 232 | } 233 | } 234 | 235 | @Override 236 | protected String dataPlace() { 237 | return "value"; 238 | } 239 | } 240 | 241 | protected HashConfig getConfig() { 242 | return this.config; 243 | } 244 | } 245 | -------------------------------------------------------------------------------- /src/main/java/io/aiven/kafka/connect/transforms/ExtractTopic.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.Arrays; 20 | import java.util.List; 21 | import java.util.Map; 22 | import java.util.Optional; 23 | 24 | import org.apache.kafka.common.config.ConfigDef; 25 | import org.apache.kafka.connect.connector.ConnectRecord; 26 | import org.apache.kafka.connect.data.Field; 27 | import org.apache.kafka.connect.data.Schema; 28 | import org.apache.kafka.connect.data.SchemaAndValue; 29 | import org.apache.kafka.connect.data.Struct; 30 | import org.apache.kafka.connect.errors.DataException; 31 | import org.apache.kafka.connect.transforms.Transformation; 32 | 33 | public abstract class ExtractTopic> implements Transformation { 34 | 35 | private static final List SUPPORTED_SCHEMA_TYPES_TO_CONVERT_FROM = Arrays.asList( 36 | Schema.Type.INT8, 37 | Schema.Type.INT16, 38 | Schema.Type.INT32, 39 | Schema.Type.INT64, 40 | Schema.Type.FLOAT32, 41 | Schema.Type.FLOAT64, 42 | Schema.Type.BOOLEAN, 43 | Schema.Type.STRING 44 | ); 45 | 46 | private static final List> SUPPORTED_VALUE_CLASS_TO_CONVERT_FROM = Arrays.asList( 47 | Byte.class, 48 | Short.class, 49 | Integer.class, 50 | Long.class, 51 | Double.class, 52 | Float.class, 53 | Boolean.class, 54 | String.class 55 | ); 56 | 57 | private ExtractTopicConfig config; 58 | 59 | @Override 60 | public ConfigDef config() { 61 | return ExtractTopicConfig.config(); 62 | } 63 | 64 | @Override 65 | public void configure(final Map settings) { 66 | this.config = new ExtractTopicConfig(settings); 67 | } 68 | 69 | @Override 70 | public R apply(final R record) { 71 | final SchemaAndValue schemaAndValue = getSchemaAndValue(record); 72 | 73 | final Optional newTopic; 74 | 75 | if (schemaAndValue.schema() == null) { // schemaless values (Map) 76 | if (config.fieldName().isPresent()) { 77 | newTopic = topicNameFromNamedFieldSchemaless( 78 | record.toString(), schemaAndValue.value(), config.fieldName().get()); 79 | } else { 80 | newTopic = topicNameWithoutFieldNameSchemaless( 81 | record.toString(), schemaAndValue.value()); 82 | } 83 | } else { // schema-based values (Struct) 84 | if (config.fieldName().isPresent()) { 85 | newTopic = topicNameFromNamedFieldWithSchema( 86 | record.toString(), schemaAndValue.schema(), schemaAndValue.value(), config.fieldName().get()); 87 | } else { 88 | newTopic = topicNameWithoutFieldNameWithSchema( 89 | record.toString(), schemaAndValue.schema(), schemaAndValue.value()); 90 | } 91 | } 92 | 93 | if (newTopic.isPresent()) { 94 | return record.newRecord( 95 | newTopic.get(), 96 | record.kafkaPartition(), 97 | record.keySchema(), 98 | record.key(), 99 | record.valueSchema(), 100 | record.value(), 101 | record.timestamp(), 102 | record.headers() 103 | ); 104 | } else { 105 | return record; 106 | } 107 | } 108 | 109 | protected abstract String dataPlace(); 110 | 111 | protected abstract SchemaAndValue getSchemaAndValue(final R record); 112 | 113 | private Optional topicNameFromNamedFieldSchemaless(final String recordStr, 114 | final Object value, 115 | final String fieldName) { 116 | if (value == null) { 117 | throw new DataException(dataPlace() + " can't be null if field name is specified: " + recordStr); 118 | } 119 | 120 | if (!(value instanceof Map)) { 121 | throw new DataException(dataPlace() + " type must be Map if field name is specified: " + recordStr); 122 | } 123 | 124 | @SuppressWarnings("unchecked") final Map valueMap = (Map) value; 125 | 126 | final Optional result = Optional.ofNullable(valueMap.get(fieldName)) 127 | .map(field -> { 128 | if (!SUPPORTED_VALUE_CLASS_TO_CONVERT_FROM.contains(field.getClass())) { 129 | throw new DataException(fieldName + " type in " + dataPlace() 130 | + " " + value 131 | + " must be " + SUPPORTED_VALUE_CLASS_TO_CONVERT_FROM 132 | + ": " + recordStr); 133 | } 134 | return field; 135 | }) 136 | .map(Object::toString); 137 | 138 | if (result.isPresent() && !result.get().isBlank()) { 139 | return result; 140 | } else { 141 | if (config.skipMissingOrNull()) { 142 | return Optional.empty(); 143 | } else { 144 | throw new DataException(fieldName + " in " + dataPlace() + " can't be null or empty: " + recordStr); 145 | } 146 | } 147 | } 148 | 149 | private Optional topicNameWithoutFieldNameSchemaless(final String recordStr, 150 | final Object value) { 151 | if (value == null || value.toString().isBlank()) { 152 | if (config.skipMissingOrNull()) { 153 | return Optional.empty(); 154 | } else { 155 | throw new DataException(dataPlace() + " can't be null or empty: " + recordStr); 156 | } 157 | } 158 | 159 | if (!SUPPORTED_VALUE_CLASS_TO_CONVERT_FROM.contains(value.getClass())) { 160 | throw new DataException("type in " + dataPlace() 161 | + " " + value 162 | + " must be " + SUPPORTED_VALUE_CLASS_TO_CONVERT_FROM 163 | + ": " + recordStr); 164 | } 165 | 166 | return Optional.of(value.toString()); 167 | } 168 | 169 | private Optional topicNameFromNamedFieldWithSchema(final String recordStr, 170 | final Schema schema, 171 | final Object value, 172 | final String fieldName) { 173 | if (Schema.Type.STRUCT != schema.type()) { 174 | throw new DataException(dataPlace() + " schema type must be STRUCT if field name is specified: " 175 | + recordStr); 176 | } 177 | 178 | if (value == null) { 179 | throw new DataException(dataPlace() + " can't be null if field name is specified: " + recordStr); 180 | } 181 | 182 | final Field field = schema.field(fieldName); 183 | if (field == null) { 184 | if (config.skipMissingOrNull()) { 185 | return Optional.empty(); 186 | } else { 187 | throw new DataException(fieldName + " in " + dataPlace() + " schema can't be missing: " + recordStr); 188 | } 189 | } 190 | 191 | if (!SUPPORTED_SCHEMA_TYPES_TO_CONVERT_FROM.contains(field.schema().type())) { 192 | throw new DataException(fieldName + " schema type in " + dataPlace() 193 | + " must be " + SUPPORTED_SCHEMA_TYPES_TO_CONVERT_FROM 194 | + ": " + recordStr); 195 | } 196 | 197 | final Struct struct = (Struct) value; 198 | 199 | final Optional result = Optional.ofNullable(struct.get(fieldName)) 200 | .map(Object::toString); 201 | if (result.isPresent() && !result.get().equals("")) { 202 | return result; 203 | } else { 204 | if (config.skipMissingOrNull()) { 205 | return Optional.empty(); 206 | } else { 207 | throw new DataException(fieldName + " in " + dataPlace() + " can't be null or empty: " + recordStr); 208 | } 209 | } 210 | } 211 | 212 | private Optional topicNameWithoutFieldNameWithSchema(final String recordStr, 213 | final Schema schema, 214 | final Object value) { 215 | if (!SUPPORTED_SCHEMA_TYPES_TO_CONVERT_FROM.contains(schema.type())) { 216 | throw new DataException(dataPlace() + " schema type must be " 217 | + SUPPORTED_SCHEMA_TYPES_TO_CONVERT_FROM 218 | + " if field name is not specified: " 219 | + recordStr); 220 | } 221 | 222 | if (value == null || "".equals(value)) { 223 | if (config.skipMissingOrNull()) { 224 | return Optional.empty(); 225 | } else { 226 | throw new DataException(dataPlace() + " can't be null or empty: " + recordStr); 227 | } 228 | } 229 | 230 | return Optional.of(value.toString()); 231 | } 232 | 233 | public static class Key> extends ExtractTopic { 234 | @Override 235 | protected SchemaAndValue getSchemaAndValue(final R record) { 236 | return new SchemaAndValue(record.keySchema(), record.key()); 237 | } 238 | 239 | @Override 240 | protected String dataPlace() { 241 | return "key"; 242 | } 243 | } 244 | 245 | public static class Value> extends ExtractTopic { 246 | @Override 247 | protected SchemaAndValue getSchemaAndValue(final R record) { 248 | return new SchemaAndValue(record.valueSchema(), record.value()); 249 | } 250 | 251 | @Override 252 | protected String dataPlace() { 253 | return "value"; 254 | } 255 | } 256 | 257 | @Override 258 | public void close() { 259 | } 260 | } 261 | -------------------------------------------------------------------------------- /src/test/java/io/aiven/kafka/connect/transforms/FilterByFieldValueTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Aiven Oy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.aiven.kafka.connect.transforms; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | import java.util.function.Consumer; 22 | import java.util.function.Supplier; 23 | 24 | import org.apache.kafka.connect.data.Schema; 25 | import org.apache.kafka.connect.data.SchemaBuilder; 26 | import org.apache.kafka.connect.data.Struct; 27 | import org.apache.kafka.connect.source.SourceRecord; 28 | 29 | import org.junit.jupiter.api.Test; 30 | 31 | import static org.assertj.core.api.Assertions.assertThat; 32 | 33 | class FilterByFieldValueTest { 34 | 35 | 36 | @Test 37 | void shouldFilterOutValueRecordsEqualsToReadEvents() { 38 | final FilterByFieldValue filter = new FilterByFieldValue.Value<>(); 39 | filter.configure(Map.of( 40 | "field.name", "op", 41 | "field.value", "r", 42 | "field.value.matches", "false" 43 | )); 44 | 45 | assertThat(filter.apply( 46 | prepareStructRecord( 47 | struct -> { 48 | }, 49 | struct -> struct.put("op", "r") 50 | ))) 51 | .as("Record with op 'r' should be filtered out") 52 | .isNull(); 53 | final SourceRecord record = prepareStructRecord( 54 | struct -> { 55 | }, 56 | struct -> struct.put("op", "u") 57 | ); 58 | assertThat(filter.apply(record)) 59 | .as("Record with op not 'r' should be not filtered out") 60 | .isEqualTo(record); 61 | } 62 | 63 | @Test 64 | void shouldFilterOutKeyRecordsEqualsToId() { 65 | final FilterByFieldValue filter = new FilterByFieldValue.Key<>(); 66 | filter.configure(Map.of( 67 | "field.name", "id", 68 | "field.value", "A123", 69 | "field.value.matches", "false" 70 | )); 71 | 72 | assertThat(filter.apply(prepareStructRecord( 73 | struct -> struct.put("id", "A123"), 74 | struct -> { 75 | }))) 76 | .as("Record with id 'A132' should be filtered out") 77 | .isNull(); 78 | final SourceRecord record = prepareStructRecord( 79 | struct -> struct.put("id", "A111"), 80 | struct -> { 81 | }); 82 | assertThat(filter.apply(record)) 83 | .as("Record with id not 'A132' should not be filtered out") 84 | .isEqualTo(record); 85 | } 86 | 87 | @Test 88 | void shouldFilterOutValueRecordsNotEqualsReadEvents() { 89 | final FilterByFieldValue filter = new FilterByFieldValue.Value<>(); 90 | filter.configure(Map.of( 91 | "field.name", "op", 92 | "field.value", "r", 93 | "field.value.matches", "true" 94 | )); 95 | 96 | assertThat(filter.apply( 97 | prepareStructRecord( 98 | struct -> { 99 | }, 100 | struct -> struct.put("op", "u") 101 | ))) 102 | .as("Record with op not equal to 'r' should be filtered out") 103 | .isNull(); 104 | final SourceRecord record = prepareStructRecord( 105 | struct -> { 106 | }, 107 | struct -> struct.put("op", "r") 108 | ); 109 | assertThat(filter.apply(record)) 110 | .as("Record with op equal to 'r' should not be filtered out") 111 | .isEqualTo(record); 112 | } 113 | 114 | @Test 115 | void shouldFilterOutKeyRecordsNotEqualsToId() { 116 | final FilterByFieldValue filter = new FilterByFieldValue.Key<>(); 117 | filter.configure(Map.of( 118 | "field.name", "id", 119 | "field.value", "A123", 120 | "field.value.matches", "true" 121 | )); 122 | 123 | assertThat(filter.apply( 124 | prepareStructRecord( 125 | struct -> struct.put("id", "111"), 126 | struct -> { 127 | } 128 | ))) 129 | .as("Record with id not equal to 'A132' should be filtered out") 130 | .isNull(); 131 | final SourceRecord record = prepareStructRecord( 132 | struct -> struct.put("id", "A123"), 133 | struct -> { 134 | } 135 | ); 136 | assertThat(filter.apply(record)) 137 | .as("Record with id equal to 'A132' should not be filtered out") 138 | .isEqualTo(record); 139 | } 140 | 141 | @Test 142 | void shouldFilterOutMapValueRecordsWithRegex() { 143 | final FilterByFieldValue filterByFieldValue = new FilterByFieldValue.Value<>(); 144 | final Map configs = new HashMap<>(); 145 | configs.put("field.name", "language"); 146 | configs.put("field.value.pattern", ".*Java.*"); 147 | configs.put("field.value.matches", "false"); 148 | filterByFieldValue.configure(configs); 149 | 150 | assertThat(filterByFieldValue.apply(prepareRecord(() -> "A42", () -> Map.of("language", "Javascript")))) 151 | .as("The record should be filtered out") 152 | .isNull(); 153 | final SourceRecord record = prepareRecord(() -> "A42", () -> Map.of("language", "Rust")); 154 | assertThat(filterByFieldValue.apply(record)) 155 | .as("The record should not be filtered out") 156 | .isEqualTo(record); 157 | } 158 | 159 | @Test 160 | void shouldFilterOutMapKeyRecordsWithRegex() { 161 | final FilterByFieldValue filterByFieldValue = new FilterByFieldValue.Key<>(); 162 | final Map configs = new HashMap<>(); 163 | configs.put("field.name", "language"); 164 | configs.put("field.value.pattern", ".*Java.*"); 165 | configs.put("field.value.matches", "false"); 166 | filterByFieldValue.configure(configs); 167 | 168 | assertThat(filterByFieldValue.apply(prepareRecord(() -> Map.of("language", "Javascript"), () -> "A42"))) 169 | .as("The record should be filtered out") 170 | .isNull(); 171 | final SourceRecord record = prepareRecord(() -> Map.of("language", "Rust"), () -> "A42"); 172 | assertThat(filterByFieldValue.apply(record)) 173 | .as("The record should not be filtered out") 174 | .isEqualTo(record); 175 | } 176 | 177 | @Test 178 | void shouldFilterOutRawKeyRecords() { 179 | final FilterByFieldValue filterByFieldValue = new FilterByFieldValue.Key<>(); 180 | final Map configs = new HashMap<>(); 181 | configs.put("field.value", "A42"); 182 | configs.put("field.value.matches", "false"); 183 | filterByFieldValue.configure(configs); 184 | 185 | assertThat(filterByFieldValue.apply(prepareRecord(() -> "A42", () -> Map.of("language", "Javascript")))) 186 | .as("The record should be filtered out") 187 | .isNull(); 188 | final SourceRecord record = prepareRecord(() -> "43", () -> Map.of("language", "Rust")); 189 | assertThat(filterByFieldValue.apply(record)) 190 | .as("The record should be filtered out") 191 | .isEqualTo(record); 192 | } 193 | 194 | @Test 195 | void shouldFilterOutRawValueRecords() { 196 | final FilterByFieldValue filterByFieldValue = new FilterByFieldValue.Value<>(); 197 | final Map configs = new HashMap<>(); 198 | configs.put("field.value", "A42"); 199 | configs.put("field.value.matches", "false"); 200 | filterByFieldValue.configure(configs); 201 | 202 | assertThat(filterByFieldValue.apply(prepareRecord(() -> Map.of("language", "Javascript"), () -> "A42"))) 203 | .as("The record should be filtered out") 204 | .isNull(); 205 | final SourceRecord record = prepareRecord(() -> Map.of("language", "Rust"), () -> "43"); 206 | assertThat(filterByFieldValue.apply(record)) 207 | .as("The record should be filtered out") 208 | .isEqualTo(record); 209 | } 210 | 211 | @Test 212 | void shouldFilterOutRawNumericValueRecords() { 213 | final FilterByFieldValue filterByFieldValue = new FilterByFieldValue.Value<>(); 214 | final Map configs = new HashMap<>(); 215 | configs.put("field.value", "42"); 216 | configs.put("field.value.matches", "false"); 217 | filterByFieldValue.configure(configs); 218 | 219 | assertThat(filterByFieldValue.apply(prepareRecord(() -> Map.of("language", "Javascript"), () -> (byte) 42))) 220 | .as("The record should be filtered out") 221 | .isNull(); 222 | final SourceRecord record = prepareRecord(() -> Map.of("language", "Rust"), () -> (byte) 43); 223 | assertThat(filterByFieldValue.apply(record)) 224 | .as("The record should be filtered out") 225 | .isEqualTo(record); 226 | } 227 | 228 | private SourceRecord prepareRecord( 229 | final Supplier keySupplier, 230 | final Supplier valueSupplier 231 | ) { 232 | return new SourceRecord(null, null, "some_topic", 233 | null, keySupplier.get(), 234 | null, valueSupplier.get()); 235 | } 236 | 237 | private SourceRecord prepareStructRecord( 238 | final Consumer keyChanges, 239 | final Consumer valueChanges 240 | ) { 241 | final Schema keySchema = SchemaBuilder.struct() 242 | .field("id", Schema.STRING_SCHEMA) 243 | .build(); 244 | final Schema valueSchema = SchemaBuilder.struct() 245 | .field("before", Schema.OPTIONAL_STRING_SCHEMA) 246 | .field("after", SchemaBuilder.struct() 247 | .field("pk", Schema.STRING_SCHEMA) 248 | .field("value", Schema.STRING_SCHEMA) 249 | .build()) 250 | .field("source", SchemaBuilder.struct().optional()) 251 | .field("op", Schema.STRING_SCHEMA) 252 | .field("ts_ms", Schema.STRING_SCHEMA) 253 | .field("transaction", Schema.OPTIONAL_STRING_SCHEMA) 254 | .build(); 255 | 256 | final Struct key = new Struct(keySchema) 257 | .put("id", "A123"); 258 | keyChanges.accept(key); 259 | 260 | final Struct after = new Struct(valueSchema.field("after").schema()) 261 | .put("pk", "1") 262 | .put("value", "New data"); 263 | 264 | final Struct value = new Struct(valueSchema) 265 | .put("before", null) 266 | .put("after", after) 267 | .put("source", null) 268 | .put("op", "u") 269 | .put("ts_ms", "1620393591654") 270 | .put("transaction", null); 271 | valueChanges.accept(value); 272 | 273 | return new SourceRecord( 274 | null, null, "some_topic", 275 | keySchema, key, 276 | valueSchema, value 277 | ); 278 | } 279 | } 280 | --------------------------------------------------------------------------------