├── .gitignore ├── gradle ├── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties └── libs.versions.toml ├── gradle.properties ├── error-handling-avro ├── lombok.config ├── src │ ├── test │ │ ├── resources │ │ │ └── log4j2.xml │ │ └── java │ │ │ └── com │ │ │ └── bakdata │ │ │ └── kafka │ │ │ └── AvroDeadLetterConverterTest.java │ └── main │ │ ├── avro │ │ └── DeadLetter.avsc │ │ └── java │ │ └── com │ │ └── bakdata │ │ └── kafka │ │ └── AvroDeadLetterConverter.java └── build.gradle.kts ├── error-handling-core ├── lombok.config ├── src │ ├── test │ │ ├── java │ │ │ └── com │ │ │ │ └── bakdata │ │ │ │ └── kafka │ │ │ │ ├── FilterHelper.java │ │ │ │ ├── DeadLetterDescriptionTest.java │ │ │ │ ├── ProcessingErrorTest.java │ │ │ │ └── ErrorUtilTest.java │ │ ├── resources │ │ │ └── log4j2.xml │ │ └── avro │ │ │ └── TestValue.avsc │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── bakdata │ │ │ └── kafka │ │ │ ├── DecoratorProcessorContext.java │ │ │ ├── DecoratorProcessingContext.java │ │ │ ├── DeadLetterConverter.java │ │ │ ├── ProcessingException.java │ │ │ ├── ProcessingError.java │ │ │ ├── SuccessValue.java │ │ │ ├── DeadLetterDescription.java │ │ │ ├── DecoratorTransformer.java │ │ │ ├── DecoratorValueTransformer.java │ │ │ ├── SuccessKeyValue.java │ │ │ ├── ErrorValue.java │ │ │ ├── DecoratorValueTransformerWithKey.java │ │ │ ├── ErrorCapturingProcessorContext.java │ │ │ ├── DecoratorProcessor.java │ │ │ ├── ErrorKeyValue.java │ │ │ ├── DecoratorValueProcessor.java │ │ │ ├── ProcessedValue.java │ │ │ ├── ErrorDescribingValueMapper.java │ │ │ ├── ErrorCapturingApiProcessorContext.java │ │ │ ├── ErrorCapturingFixedKeyProcessorContext.java │ │ │ ├── ErrorDescribingKeyValueMapper.java │ │ │ ├── ErrorDescribingValueMapperWithKey.java │ │ │ ├── ErrorCapturingValueMapper.java │ │ │ ├── ProcessedKeyValue.java │ │ │ ├── ErrorLoggingFlatValueMapper.java │ │ │ ├── ErrorLoggingValueMapper.java │ │ │ ├── ErrorDescribingProcessor.java │ │ │ ├── ErrorDescribingValueProcessor.java │ │ │ ├── ErrorLoggingFlatKeyValueMapper.java │ │ │ ├── ErrorLoggingKeyValueMapper.java │ │ │ ├── ErrorCapturingFlatValueMapper.java │ │ │ ├── ErrorLoggingFlatValueMapperWithKey.java │ │ │ ├── ErrorCapturingValueMapperWithKey.java │ │ │ ├── ErrorLoggingValueMapperWithKey.java │ │ │ ├── ErrorCapturingFlatValueMapperWithKey.java │ │ │ ├── ErrorCapturingKeyValueMapper.java │ │ │ ├── DeadLetterProcessor.java │ │ │ ├── ErrorCapturingFlatKeyValueMapper.java │ │ │ ├── ErrorLoggingFlatValueTransformer.java │ │ │ ├── ErrorHeaderProcessor.java │ │ │ ├── ErrorLoggingProcessor.java │ │ │ ├── ErrorLoggingValueProcessor.java │ │ │ └── ErrorUtil.java │ └── testFixtures │ │ └── java │ │ └── com │ │ └── bakdata │ │ └── kafka │ │ ├── TestDeadLetterSerde.java │ │ └── ErrorCaptureTopologyTest.java └── build.gradle.kts ├── error-handling-proto ├── lombok.config ├── src │ ├── test │ │ ├── resources │ │ │ └── log4j2.xml │ │ └── java │ │ │ └── com │ │ │ └── bakdata │ │ │ └── kafka │ │ │ └── ProtoDeadLetterConverterTest.java │ └── main │ │ ├── proto │ │ └── bakdata │ │ │ └── kafka │ │ │ └── proto │ │ │ └── v1 │ │ │ └── deadletter.proto │ │ └── java │ │ └── com │ │ └── bakdata │ │ └── kafka │ │ └── ProtoDeadLetterConverter.java └── build.gradle.kts ├── settings.gradle.kts ├── error-handling-bom └── build.gradle.kts ├── .github ├── workflows │ ├── release.yaml │ └── build-and-publish.yaml └── dependabot.yaml ├── LICENSE ├── gradlew.bat └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | !.github 3 | !.gitignore 4 | build/ 5 | out/ 6 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bakdata/kafka-error-handling/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | version=2.1.1-SNAPSHOT 2 | org.gradle.caching=true 3 | org.gradle.parallel=true 4 | org.gradle.jvmargs=-Xmx2048m 5 | -------------------------------------------------------------------------------- /error-handling-avro/lombok.config: -------------------------------------------------------------------------------- 1 | # This file is generated by the 'io.freefair.lombok' Gradle plugin 2 | config.stopBubbling = true 3 | lombok.addLombokGeneratedAnnotation = true 4 | -------------------------------------------------------------------------------- /error-handling-core/lombok.config: -------------------------------------------------------------------------------- 1 | # This file is generated by the 'io.freefair.lombok' Gradle plugin 2 | config.stopBubbling = true 3 | lombok.addLombokGeneratedAnnotation = true 4 | -------------------------------------------------------------------------------- /error-handling-proto/lombok.config: -------------------------------------------------------------------------------- 1 | # This file is generated by the 'io.freefair.lombok' Gradle plugin 2 | config.stopBubbling = true 3 | lombok.addLombokGeneratedAnnotation = true 4 | -------------------------------------------------------------------------------- /settings.gradle.kts: -------------------------------------------------------------------------------- 1 | pluginManagement { 2 | repositories { 3 | gradlePluginPortal() 4 | } 5 | } 6 | 7 | rootProject.name = "error-handling" 8 | include("error-handling-core") 9 | include("error-handling-avro") 10 | include("error-handling-proto") 11 | include("error-handling-bom") 12 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-all.zip 4 | networkTimeout=10000 5 | validateDistributionUrl=true 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /error-handling-core/src/test/java/com/bakdata/kafka/FilterHelper.java: -------------------------------------------------------------------------------- 1 | package com.bakdata.kafka; 2 | 3 | import java.util.function.Predicate; 4 | import lombok.experimental.UtilityClass; 5 | 6 | @UtilityClass 7 | class FilterHelper { 8 | static Predicate filterAll() { 9 | return ignored -> true; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /error-handling-bom/build.gradle.kts: -------------------------------------------------------------------------------- 1 | description = "BOM for error handling in Kafka Streams." 2 | 3 | plugins { 4 | id("java-platform") 5 | } 6 | 7 | dependencies { 8 | constraints { 9 | api(project(":error-handling-core")) 10 | api(project(":error-handling-avro")) 11 | api(project(":error-handling-proto")) 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /error-handling-avro/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /error-handling-core/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /error-handling-proto/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /error-handling-core/src/test/avro/TestValue.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "TestValue", 4 | "namespace": "com.bakdata.kafka", 5 | "fields": [ 6 | { 7 | "name": "optional_field", 8 | "type": [ 9 | "null", 10 | "string" 11 | ], 12 | "default": null 13 | }, 14 | { 15 | "name": "field1", 16 | "type": [ 17 | "null", 18 | "string" 19 | ] 20 | }, 21 | { 22 | "name": "field2", 23 | "type": [ 24 | "null", 25 | "string" 26 | ] 27 | } 28 | ] 29 | } 30 | 31 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | release-type: 7 | description: "The scope of the release (major, minor or patch)." 8 | type: choice 9 | required: true 10 | default: patch 11 | options: 12 | - patch 13 | - minor 14 | - major 15 | 16 | jobs: 17 | java-gradle-release: 18 | name: Java Gradle 19 | uses: bakdata/ci-templates/.github/workflows/java-gradle-release.yaml@1.70.0 20 | with: 21 | java-version: 17 22 | release-type: "${{ inputs.release-type }}" 23 | secrets: 24 | github-email: "${{ secrets.GH_EMAIL }}" 25 | github-username: "${{ secrets.GH_USERNAME }}" 26 | github-token: "${{ secrets.GH_TOKEN }}" 27 | -------------------------------------------------------------------------------- /error-handling-proto/src/main/proto/bakdata/kafka/proto/v1/deadletter.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package bakdata.kafka.proto.v1; 4 | 5 | import "google/protobuf/wrappers.proto"; 6 | import "google/protobuf/timestamp.proto"; 7 | 8 | option java_package = "com.bakdata.kafka.proto.v1"; 9 | option java_multiple_files = true; 10 | 11 | message ProtoDeadLetter { 12 | message Cause { 13 | google.protobuf.StringValue message = 1; 14 | google.protobuf.StringValue stack_trace = 2; 15 | google.protobuf.StringValue error_class = 3; 16 | } 17 | string description = 1; 18 | Cause cause = 2; 19 | google.protobuf.StringValue input_value = 3; 20 | google.protobuf.StringValue topic = 4; 21 | google.protobuf.Int32Value partition = 5; 22 | google.protobuf.Int64Value offset = 6; 23 | google.protobuf.Timestamp input_timestamp = 7; 24 | } 25 | -------------------------------------------------------------------------------- /.github/workflows/build-and-publish.yaml: -------------------------------------------------------------------------------- 1 | name: Build and Publish 2 | 3 | on: 4 | push: 5 | tags: ["**"] 6 | branches: 7 | - master 8 | pull_request: 9 | 10 | jobs: 11 | build-and-publish: 12 | name: Java Gradle 13 | uses: bakdata/ci-templates/.github/workflows/java-gradle-library.yaml@1.70.0 14 | with: 15 | java-version: 17 16 | secrets: 17 | sonar-token: ${{ secrets.SONARCLOUD_TOKEN }} 18 | sonar-organization: ${{ secrets.SONARCLOUD_ORGANIZATION }} 19 | signing-secret-key-ring: ${{ secrets.SONATYPE_SIGNING_SECRET_KEY_RING }} 20 | signing-key-id: ${{ secrets.SONATYPE_SIGNING_KEY_ID }} 21 | signing-password: ${{ secrets.SONATYPE_SIGNING_PASSWORD }} 22 | ossrh-username: ${{ secrets.SONATYPE_OSSRH_USERNAME }} 23 | ossrh-password: ${{ secrets.SONATYPE_OSSRH_PASSWORD }} 24 | github-token: ${{ secrets.GH_TOKEN }} 25 | -------------------------------------------------------------------------------- /error-handling-proto/build.gradle.kts: -------------------------------------------------------------------------------- 1 | description = "Transform dead letters in Kafka Streams applications to protobuf." 2 | 3 | plugins { 4 | id("java-library") 5 | alias(libs.plugins.protobuf) 6 | } 7 | 8 | val protobufVersion = libs.protobuf.get().version 9 | dependencies { 10 | compileOnly(platform(libs.kafka.bom)) 11 | compileOnly(libs.kafka.streams) 12 | api(project(":error-handling-core")) 13 | api(libs.protobuf) 14 | testRuntimeOnly(libs.junit.platform.launcher) 15 | testImplementation(libs.junit.jupiter) 16 | testImplementation(testFixtures(project(":error-handling-core"))) 17 | testImplementation(libs.mockito.junit) 18 | testImplementation(libs.assertj) 19 | testImplementation(libs.log4j.slf4j2) 20 | testImplementation(libs.kafka.streams.protobuf.serde) { 21 | exclude(group = "org.apache.kafka") // force usage of OSS kafka-clients 22 | } 23 | } 24 | 25 | protobuf { 26 | protoc { 27 | artifact = "com.google.protobuf:protoc:$protobufVersion" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /error-handling-core/build.gradle.kts: -------------------------------------------------------------------------------- 1 | description = "A library for error handling in Kafka Streams." 2 | 3 | plugins { 4 | id("java-library") 5 | alias(libs.plugins.avro) 6 | } 7 | 8 | dependencies { 9 | compileOnly(platform(libs.kafka.bom)) 10 | compileOnly(libs.kafka.streams) 11 | implementation(libs.avro) 12 | implementation(libs.jool) 13 | implementation(libs.commons.lang) 14 | 15 | testRuntimeOnly(libs.junit.platform.launcher) 16 | testImplementation(libs.junit.jupiter) 17 | testImplementation(libs.assertj) 18 | testImplementation(libs.mockito.core) 19 | testImplementation(libs.mockito.junit) 20 | testImplementation(libs.log4j.slf4j2) 21 | testFixturesApi(libs.fluentKafkaStreamsTests) 22 | testFixturesImplementation(libs.junit.jupiter) 23 | testFixturesImplementation(libs.jackson.core) 24 | testFixturesImplementation(libs.jackson.databind) 25 | testFixturesImplementation(libs.jackson.datatype.jsr310) 26 | } 27 | 28 | avro { 29 | setGettersReturnOptional(true) 30 | setOptionalGettersForNullableFieldsOnly(true) 31 | setFieldVisibility("PRIVATE") 32 | } 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 bakdata 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/dependabot.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gradle" 4 | directory: "/" 5 | schedule: 6 | interval: "monthly" 7 | groups: 8 | kafka-dependencies: 9 | patterns: 10 | - "com.bakdata.kafka*" 11 | - "com.bakdata.fluent-kafka-streams-tests*" 12 | - "io.confluent*" 13 | - "org.apache.kafka*" 14 | log-dependencies: 15 | patterns: 16 | - "org.slf4j*" 17 | - "org.apache.logging.log4j*" 18 | test-dependencies: 19 | patterns: 20 | - "org.junit*" 21 | - "org.assertj*" 22 | - "*junit*" 23 | - "org.mockito*" 24 | - "org.testcontainers*" 25 | - "org.awaitility*" 26 | exclude-patterns: 27 | - "com.bakdata.fluent-kafka-streams-tests*" 28 | plugins: 29 | patterns: 30 | - "com.bakdata.release" 31 | - "com.bakdata.sonar" 32 | - "com.bakdata.sonatype" 33 | protobuf: 34 | patterns: 35 | - "com.google.protobuf*" 36 | 37 | - package-ecosystem: "github-actions" 38 | directory: "/" 39 | schedule: 40 | interval: "daily" 41 | -------------------------------------------------------------------------------- /error-handling-avro/build.gradle.kts: -------------------------------------------------------------------------------- 1 | description = "Transform dead letters in Kafka Streams applications to Avro format." 2 | 3 | plugins { 4 | id("java-library") 5 | alias(libs.plugins.avro) 6 | } 7 | 8 | // add .avsc files to jar allowing us to use them in other projects as a schema dependency 9 | sourceSets { 10 | main { 11 | resources { 12 | srcDirs("src/main/avro") 13 | } 14 | } 15 | } 16 | 17 | dependencies { 18 | compileOnly(platform(libs.kafka.bom)) 19 | compileOnly(libs.kafka.streams) 20 | api(libs.avro) 21 | api(project(":error-handling-core")) 22 | 23 | testRuntimeOnly(libs.junit.platform.launcher) 24 | testImplementation(libs.junit.jupiter) 25 | testImplementation(testFixtures(project(":error-handling-core"))) 26 | testImplementation(libs.mockito.junit) 27 | testImplementation(libs.assertj) 28 | testImplementation(libs.log4j.slf4j2) 29 | testImplementation(libs.kafka.streams.avro.serde) { 30 | exclude(group = "org.apache.kafka") // force usage of OSS kafka-clients 31 | } 32 | } 33 | 34 | avro { 35 | setGettersReturnOptional(true) 36 | setOptionalGettersForNullableFieldsOnly(true) 37 | setFieldVisibility("PRIVATE") 38 | } 39 | -------------------------------------------------------------------------------- /error-handling-core/src/test/java/com/bakdata/kafka/DeadLetterDescriptionTest.java: -------------------------------------------------------------------------------- 1 | package com.bakdata.kafka; 2 | 3 | import static org.assertj.core.api.Assertions.assertThatNullPointerException; 4 | import static org.assertj.core.api.AssertionsForClassTypes.assertThatCode; 5 | 6 | import com.bakdata.kafka.DeadLetterDescription.Cause; 7 | import org.junit.jupiter.api.Test; 8 | 9 | class DeadLetterDescriptionTest { 10 | 11 | private static final Cause CAUSE = Cause.builder().build(); 12 | 13 | @Test 14 | void shouldNotAllowNullDescription() { 15 | assertThatNullPointerException() 16 | .isThrownBy(() -> DeadLetterDescription.builder().description(null).cause(CAUSE).build()); 17 | } 18 | 19 | @Test 20 | void shouldNotAllowNullCause() { 21 | assertThatNullPointerException() 22 | .isThrownBy(() -> DeadLetterDescription.builder().description("foo").cause(null).build()); 23 | } 24 | 25 | @Test 26 | void shouldAllowOtherFieldsNull() { 27 | assertThatCode(() -> DeadLetterDescription.builder() 28 | .description("foo") 29 | .cause(CAUSE) 30 | .inputValue(null) 31 | .topic(null) 32 | .partition(null) 33 | .offset(null).build()).doesNotThrowAnyException(); 34 | } 35 | 36 | @Test 37 | void shouldAllowCauseFieldsNull() { 38 | assertThatCode(() -> Cause.builder() 39 | .message(null) 40 | .errorClass(null) 41 | .stackTrace(null) 42 | .build()).doesNotThrowAnyException(); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/DecoratorProcessorContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2020 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.NonNull; 28 | import lombok.RequiredArgsConstructor; 29 | import lombok.experimental.Delegate; 30 | import org.apache.kafka.streams.processor.ProcessorContext; 31 | 32 | /** 33 | * Base class for decorating a {@code ProcessorContext} 34 | */ 35 | @RequiredArgsConstructor 36 | public abstract class DecoratorProcessorContext implements ProcessorContext { 37 | @Delegate 38 | private final @NonNull ProcessorContext wrapped; 39 | } 40 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/DecoratorProcessingContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.NonNull; 28 | import lombok.RequiredArgsConstructor; 29 | import lombok.experimental.Delegate; 30 | import org.apache.kafka.streams.processor.api.ProcessingContext; 31 | 32 | /** 33 | * Base class for decorating a {@code ProcessingContext} 34 | */ 35 | @RequiredArgsConstructor 36 | public abstract class DecoratorProcessingContext implements ProcessingContext { 37 | @Delegate 38 | private final @NonNull ProcessingContext wrapped; 39 | } 40 | -------------------------------------------------------------------------------- /error-handling-avro/src/main/avro/DeadLetter.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "DeadLetter", 4 | "namespace": "com.bakdata.kafka", 5 | "fields": [ 6 | { 7 | "name": "input_value", 8 | "type": [ 9 | "null", 10 | "string" 11 | ] 12 | }, 13 | { 14 | "name": "topic", 15 | "type": [ 16 | "null", 17 | "string" 18 | ], 19 | "default": null 20 | }, 21 | { 22 | "name": "partition", 23 | "type": [ 24 | "null", 25 | "int" 26 | ], 27 | "default": null 28 | }, 29 | { 30 | "name": "offset", 31 | "type": [ 32 | "null", 33 | "long" 34 | ], 35 | "default": null 36 | }, 37 | { 38 | "name": "description", 39 | "type": "string" 40 | }, 41 | { 42 | "name": "cause", 43 | "type": { 44 | "type": "record", 45 | "name": "ErrorDescription", 46 | "fields": [ 47 | { 48 | "name": "error_class", 49 | "type": [ 50 | "null", 51 | "string" 52 | ], 53 | "default": null 54 | }, 55 | { 56 | "name": "message", 57 | "type": [ 58 | "null", 59 | "string" 60 | ] 61 | }, 62 | { 63 | "name": "stack_trace", 64 | "type": [ 65 | "null", 66 | "string" 67 | ] 68 | } 69 | ] 70 | } 71 | }, 72 | { 73 | "name": "input_timestamp", 74 | "type": [ 75 | "null", 76 | { 77 | "type": "long", 78 | "logicalType": "timestamp-millis" 79 | } 80 | ], 81 | "default": null 82 | } 83 | ] 84 | } 85 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/DeadLetterConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | /** 28 | * Converts a {@code DeadLetterDescription} to a specific type for serialization 29 | * 30 | * @param The type after the conversion 31 | */ 32 | public interface DeadLetterConverter { 33 | 34 | /** 35 | * Converts a {@code DeadLetterDescription} to T 36 | * 37 | * @param deadLetterDescription contains all information about one error 38 | * @return the converted description 39 | */ 40 | T convert(final DeadLetterDescription deadLetterDescription); 41 | } 42 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ProcessingException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | /** 28 | * {@link Exception} thrown by error descriptors. The message contains information about input key and value. 29 | */ 30 | public class ProcessingException extends RuntimeException { 31 | 32 | ProcessingException(final Object value, final Throwable cause) { 33 | super("Cannot process " + ErrorUtil.toString(value), cause); 34 | } 35 | 36 | ProcessingException(final Object key, final Object value, final Throwable cause) { 37 | super("Cannot process ('" + ErrorUtil.toString(key) + "', '" + ErrorUtil.toString(value) + "')", cause); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ProcessingError.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.AccessLevel; 28 | import lombok.Builder; 29 | import lombok.Getter; 30 | import lombok.NonNull; 31 | import lombok.RequiredArgsConstructor; 32 | 33 | 34 | /** 35 | * This class represents an error that has been thrown upon processing an input value. Both the input value and the 36 | * thrown exception are available for further error handling. 37 | * 38 | * @param type of input value 39 | */ 40 | @Getter 41 | @Builder 42 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 43 | public final class ProcessingError { 44 | 45 | private final V value; 46 | private final @NonNull Throwable throwable; 47 | } 48 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/SuccessValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static java.util.Collections.emptyList; 28 | 29 | import java.util.Collections; 30 | import lombok.AccessLevel; 31 | import lombok.RequiredArgsConstructor; 32 | 33 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 34 | final class SuccessValue implements ProcessedValue { 35 | 36 | private final VR value; 37 | 38 | static ProcessedValue of(final VR vr) { 39 | return new SuccessValue<>(vr); 40 | } 41 | 42 | @Override 43 | public Iterable> getErrors() { 44 | return emptyList(); 45 | } 46 | 47 | @Override 48 | public Iterable getValues() { 49 | // allow null values 50 | return Collections.singletonList(this.value); 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/DeadLetterDescription.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.time.Instant; 28 | import lombok.Builder; 29 | import lombok.NonNull; 30 | import lombok.Value; 31 | import lombok.extern.jackson.Jacksonized; 32 | 33 | /** 34 | * The representation of an error with contextual information. 35 | */ 36 | @Builder 37 | @Value 38 | @Jacksonized 39 | public class DeadLetterDescription { 40 | 41 | /** 42 | * The information of the error/ exception 43 | */ 44 | @Builder 45 | @Value 46 | @Jacksonized 47 | public static class Cause { 48 | String message; 49 | String stackTrace; 50 | String errorClass; 51 | } 52 | 53 | @NonNull String description; 54 | @NonNull Cause cause; 55 | String inputValue; 56 | String topic; 57 | Integer partition; 58 | Long offset; 59 | Instant inputTimestamp; 60 | } 61 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/DecoratorTransformer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2020 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.NonNull; 28 | import lombok.RequiredArgsConstructor; 29 | import org.apache.kafka.streams.kstream.Transformer; 30 | import org.apache.kafka.streams.processor.ProcessorContext; 31 | 32 | /** 33 | * Base class for decorating a {@code Transformer} 34 | */ 35 | @RequiredArgsConstructor 36 | public abstract class DecoratorTransformer implements Transformer { 37 | private final @NonNull Transformer wrapped; 38 | 39 | @Override 40 | public void close() { 41 | this.wrapped.close(); 42 | } 43 | 44 | @Override 45 | public void init(final ProcessorContext context) { 46 | this.wrapped.init(context); 47 | } 48 | 49 | @Override 50 | public R transform(final K key, final V value) { 51 | return this.wrapped.transform(key, value); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/DecoratorValueTransformer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2020 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.NonNull; 28 | import lombok.RequiredArgsConstructor; 29 | import org.apache.kafka.streams.kstream.ValueTransformer; 30 | import org.apache.kafka.streams.processor.ProcessorContext; 31 | 32 | /** 33 | * Base class for decorating a {@code ValueTransformer} 34 | */ 35 | @RequiredArgsConstructor 36 | public abstract class DecoratorValueTransformer implements ValueTransformer { 37 | private final @NonNull ValueTransformer wrapped; 38 | 39 | @Override 40 | public void close() { 41 | this.wrapped.close(); 42 | } 43 | 44 | @Override 45 | public void init(final ProcessorContext context) { 46 | this.wrapped.init(context); 47 | } 48 | 49 | @Override 50 | public R transform(final V value) { 51 | return this.wrapped.transform(value); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/SuccessKeyValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static java.util.Collections.emptyList; 28 | 29 | import java.util.Collections; 30 | import lombok.AccessLevel; 31 | import lombok.RequiredArgsConstructor; 32 | import org.apache.kafka.streams.KeyValue; 33 | 34 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 35 | final class SuccessKeyValue implements ProcessedKeyValue { 36 | 37 | private final VR value; 38 | 39 | static ProcessedKeyValue of(final VR vr) { 40 | return new SuccessKeyValue<>(vr); 41 | } 42 | 43 | @Override 44 | public Iterable>> getErrors() { 45 | return emptyList(); 46 | } 47 | 48 | @Override 49 | public Iterable getValues() { 50 | // allow null values 51 | return Collections.singletonList(this.value); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2020 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static java.util.Collections.emptyList; 28 | 29 | import java.util.List; 30 | import lombok.AccessLevel; 31 | import lombok.NonNull; 32 | import lombok.RequiredArgsConstructor; 33 | 34 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 35 | final class ErrorValue implements ProcessedValue { 36 | private final @NonNull ProcessingError error; 37 | 38 | static ProcessedValue of(final V value, final Throwable throwable) { 39 | return new ErrorValue<>(ProcessingError.builder() 40 | .throwable(throwable) 41 | .value(value) 42 | .build()); 43 | } 44 | 45 | @Override 46 | public Iterable> getErrors() { 47 | return List.of(this.error); 48 | } 49 | 50 | @Override 51 | public Iterable getValues() { 52 | return emptyList(); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/DecoratorValueTransformerWithKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2020 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.NonNull; 28 | import lombok.RequiredArgsConstructor; 29 | import org.apache.kafka.streams.kstream.ValueTransformerWithKey; 30 | import org.apache.kafka.streams.processor.ProcessorContext; 31 | 32 | /** 33 | * Base class for decorating a {@code ValueTransformerWithKey} 34 | */ 35 | @RequiredArgsConstructor 36 | public abstract class DecoratorValueTransformerWithKey implements ValueTransformerWithKey { 37 | private final @NonNull ValueTransformerWithKey wrapped; 38 | 39 | @Override 40 | public void close() { 41 | this.wrapped.close(); 42 | } 43 | 44 | @Override 45 | public void init(final ProcessorContext context) { 46 | this.wrapped.init(context); 47 | } 48 | 49 | @Override 50 | public R transform(final K key, final V value) { 51 | return this.wrapped.transform(key, value); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /gradle/libs.versions.toml: -------------------------------------------------------------------------------- 1 | [versions] 2 | jackson = "2.20.0" 3 | junit = "5.14.0" 4 | mockito = "5.20.0" 5 | 6 | [libraries] 7 | kafka-bom = { group = "com.bakdata.kafka", name = "kafka-bom", version = "1.3.0" } 8 | kafka-streams = { group = "org.apache.kafka", name = "kafka-streams" } 9 | kafka-streams-avro-serde = { group = "io.confluent", name = "kafka-streams-avro-serde" } 10 | kafka-streams-protobuf-serde = { group = "io.confluent", name = "kafka-streams-protobuf-serde" } 11 | avro = { group = "org.apache.avro", name = "avro", version = "1.12.0" } 12 | protobuf = { group = "com.google.protobuf", name = "protobuf-java", version = "4.33.0" } 13 | jool = { group = "org.jooq", name = "jool", version = "0.9.15" } 14 | commons-lang = { group = "org.apache.commons", name = "commons-lang3", version = "3.18.0" } 15 | jackson-core = { group = "com.fasterxml.jackson.core", name = "jackson-core", version.ref = "jackson" } 16 | jackson-databind = { group = "com.fasterxml.jackson.core", name = "jackson-databind", version.ref = "jackson" } 17 | jackson-datatype-jsr310 = { group = "com.fasterxml.jackson.datatype", name = "jackson-datatype-jsr310", version.ref = "jackson" } 18 | 19 | junit-platform-launcher = { group = "org.junit.platform", name = "junit-platform-launcher" } 20 | junit-jupiter = { group = "org.junit.jupiter", name = "junit-jupiter", version.ref = "junit" } 21 | assertj = { group = "org.assertj", name = "assertj-core", version = "3.27.6" } 22 | mockito-core = { group = "org.mockito", name = "mockito-core", version.ref = "mockito" } 23 | mockito-junit = { group = "org.mockito", name = "mockito-junit-jupiter", version.ref = "mockito" } 24 | fluentKafkaStreamsTests = { group = "com.bakdata.fluent-kafka-streams-tests", name = "fluent-kafka-streams-tests-junit5", version = "3.5.0" } 25 | log4j-slf4j2 = { group = "org.apache.logging.log4j", name = "log4j-slf4j2-impl", version = "2.25.2" } 26 | 27 | [plugins] 28 | release = { id = "com.bakdata.release", version = "1.11.0" } 29 | sonar = { id = "com.bakdata.sonar", version = "1.11.0" } 30 | sonatype = { id = "com.bakdata.sonatype", version = "1.11.0" } 31 | lombok = { id = "io.freefair.lombok", version = "8.14" } 32 | avro = { id = "com.github.davidmc24.gradle.plugin.avro", version = "1.9.1" } 33 | protobuf = { id = "com.google.protobuf", version = "0.9.5" } 34 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorCapturingProcessorContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.NonNull; 28 | import org.apache.kafka.streams.processor.ProcessorContext; 29 | import org.apache.kafka.streams.processor.To; 30 | 31 | final class ErrorCapturingProcessorContext extends DecoratorProcessorContext { 32 | ErrorCapturingProcessorContext(final @NonNull ProcessorContext wrapped) { 33 | super(wrapped); 34 | } 35 | 36 | private static ProcessedKeyValue getValue(final VR value) { 37 | return SuccessKeyValue.of(value); 38 | } 39 | 40 | @Override 41 | public void forward(final K key, final V value, final To to) { 42 | final ProcessedKeyValue recordWithOldKey = getValue(value); 43 | super.forward(key, recordWithOldKey, to); 44 | } 45 | 46 | @Override 47 | public void forward(final K key, final V value) { 48 | final ProcessedKeyValue recordWithOldKey = getValue(value); 49 | super.forward(key, recordWithOldKey); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /error-handling-core/src/test/java/com/bakdata/kafka/ProcessingErrorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static org.assertj.core.api.Assertions.assertThat; 28 | import static org.assertj.core.api.Assertions.assertThatNullPointerException; 29 | 30 | import org.junit.jupiter.api.Test; 31 | import org.junit.jupiter.api.extension.ExtendWith; 32 | import org.mockito.Mock; 33 | import org.mockito.junit.jupiter.MockitoExtension; 34 | import org.mockito.junit.jupiter.MockitoSettings; 35 | import org.mockito.quality.Strictness; 36 | 37 | @ExtendWith(MockitoExtension.class) 38 | @MockitoSettings(strictness = Strictness.STRICT_STUBS) 39 | class ProcessingErrorTest { 40 | 41 | @Mock 42 | private Throwable throwable; 43 | 44 | @Test 45 | void shouldNotAllowNullCause() { 46 | assertThatNullPointerException() 47 | .isThrownBy(() -> ProcessingError.builder().value("foo").throwable(null).build()); 48 | } 49 | 50 | @Test 51 | void shouldAllowNullValue() { 52 | assertThat(ProcessingError.builder().value(null).throwable(this.throwable).build()) 53 | .isNotNull(); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/DecoratorProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.NonNull; 28 | import lombok.RequiredArgsConstructor; 29 | import org.apache.kafka.streams.processor.api.Processor; 30 | import org.apache.kafka.streams.processor.api.ProcessorContext; 31 | import org.apache.kafka.streams.processor.api.Record; 32 | 33 | /** 34 | * Base class for decorating a {@code Processor} 35 | * 36 | * @param type of input keys 37 | * @param type of input values 38 | * @param type of output keys 39 | * @param type of output values 40 | */ 41 | @RequiredArgsConstructor 42 | public abstract class DecoratorProcessor implements Processor { 43 | private final @NonNull Processor wrapped; 44 | 45 | @Override 46 | public void close() { 47 | this.wrapped.close(); 48 | } 49 | 50 | @Override 51 | public void process(final Record inputRecord) { 52 | this.wrapped.process(inputRecord); 53 | } 54 | 55 | @Override 56 | public void init(final ProcessorContext context) { 57 | this.wrapped.init(context); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorKeyValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2020 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static java.util.Collections.emptyList; 28 | 29 | import java.util.List; 30 | import lombok.AccessLevel; 31 | import lombok.NonNull; 32 | import lombok.RequiredArgsConstructor; 33 | import org.apache.kafka.streams.KeyValue; 34 | 35 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 36 | final class ErrorKeyValue implements ProcessedKeyValue { 37 | private final K oldKey; 38 | private final @NonNull ProcessingError error; 39 | 40 | static ProcessedKeyValue of(final K oldKey, final V value, 41 | final Throwable throwable) { 42 | return new ErrorKeyValue<>(oldKey, ProcessingError.builder() 43 | .throwable(throwable) 44 | .value(value) 45 | .build()); 46 | } 47 | 48 | @Override 49 | public Iterable>> getErrors() { 50 | return List.of(KeyValue.pair(this.oldKey, this.error)); 51 | } 52 | 53 | @Override 54 | public Iterable getValues() { 55 | return emptyList(); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/DecoratorValueProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.NonNull; 28 | import lombok.RequiredArgsConstructor; 29 | import org.apache.kafka.streams.processor.api.FixedKeyProcessor; 30 | import org.apache.kafka.streams.processor.api.FixedKeyProcessorContext; 31 | import org.apache.kafka.streams.processor.api.FixedKeyRecord; 32 | 33 | /** 34 | * Base class for decorating a {@code ValueTransformerWithKey} 35 | * 36 | * @param type of input keys 37 | * @param type of input values 38 | * @param type of output values 39 | */ 40 | @RequiredArgsConstructor 41 | public abstract class DecoratorValueProcessor implements FixedKeyProcessor { 42 | private final @NonNull FixedKeyProcessor wrapped; 43 | 44 | @Override 45 | public void close() { 46 | this.wrapped.close(); 47 | } 48 | 49 | @Override 50 | public void process(final FixedKeyRecord inputRecord) { 51 | this.wrapped.process(inputRecord); 52 | } 53 | 54 | @Override 55 | public void init(final FixedKeyProcessorContext context) { 56 | this.wrapped.init(context); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ProcessedValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2020 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | /** 28 | * A processed value is created upon capturing errors in a streams topology. It can either contain a successfully 29 | * processed value or a {@link ProcessingError} describing the input value and the {@link Exception} that has been 30 | * thrown. 31 | * 32 | * @param the type of the old value before applying the error capturer 33 | * @param the type of the new value after applying the error capturer 34 | */ 35 | public interface ProcessedValue { 36 | 37 | /** 38 | * Extract errors from a processed value. If an error is available, it will give information about the input value 39 | * and the {@link Exception} that was thrown while attempting to map it to a new value. 40 | * 41 | * @return A single {@link ProcessingError} if an exception was thrown upon processing the input value or an empty 42 | * list 43 | */ 44 | Iterable> getErrors(); 45 | 46 | /** 47 | * Extract successfully processed values from a processed value. 48 | * 49 | * @return A single value if processing was successful or an empty list 50 | */ 51 | Iterable getValues(); 52 | } 53 | -------------------------------------------------------------------------------- /error-handling-core/src/testFixtures/java/com/bakdata/kafka/TestDeadLetterSerde.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import com.fasterxml.jackson.core.JsonProcessingException; 28 | import com.fasterxml.jackson.databind.ObjectMapper; 29 | import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; 30 | import java.io.IOException; 31 | import org.apache.kafka.common.errors.SerializationException; 32 | import org.apache.kafka.common.serialization.Deserializer; 33 | import org.apache.kafka.common.serialization.Serde; 34 | import org.apache.kafka.common.serialization.Serializer; 35 | 36 | public class TestDeadLetterSerde implements Serde { 37 | static final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule()); 38 | private static final Serializer serializer = (topic, data) -> { 39 | try { 40 | return objectMapper.writeValueAsBytes(data); 41 | } catch (final JsonProcessingException e) { 42 | throw new SerializationException(e); 43 | } 44 | }; 45 | private static final Deserializer deserializer = (topic, data) -> { 46 | try { 47 | return objectMapper.readValue(data, DeadLetterDescription.class); 48 | } catch (IOException e) { 49 | throw new SerializationException(e); 50 | } 51 | }; 52 | 53 | @Override 54 | public Serializer serializer() { 55 | return serializer; 56 | } 57 | 58 | @Override 59 | public Deserializer deserializer() { 60 | return deserializer; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorDescribingValueMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2020 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.AccessLevel; 28 | import lombok.NonNull; 29 | import lombok.RequiredArgsConstructor; 30 | import org.apache.kafka.streams.kstream.ValueMapper; 31 | 32 | /** 33 | * Wrap a {@code ValueMapper} and describe thrown exceptions with input key and value. 34 | * 35 | * @param type of input values 36 | * @param type of output values 37 | * @see #describeErrors(ValueMapper) 38 | */ 39 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 40 | public final class ErrorDescribingValueMapper implements ValueMapper { 41 | private final @NonNull ValueMapper wrapped; 42 | 43 | /** 44 | * Wrap a {@code ValueMapper} and describe thrown exceptions with input key and value. 45 | * 
{@code
46 |      * final ValueMapper mapper = ...;
47 |      * final KStream input = ...;
48 |      * final KStream output = input.mapValues(describeErrors(mapper));
49 |      * }
50 |      *
51 | * 52 | * @param mapper {@code ValueMapper} whose exceptions should be described 53 | * @param type of input values 54 | * @param type of output values 55 | * @return {@code ValueMapper} 56 | */ 57 | public static ValueMapper describeErrors( 58 | final @NonNull ValueMapper mapper) { 59 | return new ErrorDescribingValueMapper<>(mapper); 60 | } 61 | 62 | @Override 63 | public VR apply(final V value) { 64 | try { 65 | return this.wrapped.apply(value); 66 | } catch (final Exception e) { 67 | throw new ProcessingException(value, e); 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorCapturingApiProcessorContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.NonNull; 28 | import org.apache.kafka.streams.processor.api.ProcessorContext; 29 | import org.apache.kafka.streams.processor.api.Record; 30 | 31 | final class ErrorCapturingApiProcessorContext extends DecoratorProcessingContext 32 | implements ProcessorContext { 33 | private final @NonNull ProcessorContext> wrapped; 34 | 35 | ErrorCapturingApiProcessorContext( 36 | final @NonNull ProcessorContext> wrapped) { 37 | super(wrapped); 38 | this.wrapped = wrapped; 39 | } 40 | 41 | private static Record> getValue(final Record outputRecord) { 43 | final VR value = outputRecord.value(); 44 | final ProcessedKeyValue recordWithOldKey = SuccessKeyValue.of(value); 45 | return outputRecord.withValue(recordWithOldKey); 46 | } 47 | 48 | @Override 49 | public void forward(final Record outputRecord) { 50 | final Record> recordWithOldKey = getValue(outputRecord); 51 | this.wrapped.forward(recordWithOldKey); 52 | } 53 | 54 | @Override 55 | public void forward(final Record outputRecord, 56 | final String childName) { 57 | final Record> recordWithOldKey = getValue(outputRecord); 58 | this.wrapped.forward(recordWithOldKey, childName); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorCapturingFixedKeyProcessorContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.NonNull; 28 | import org.apache.kafka.streams.processor.api.FixedKeyProcessorContext; 29 | import org.apache.kafka.streams.processor.api.FixedKeyRecord; 30 | 31 | final class ErrorCapturingFixedKeyProcessorContext extends DecoratorProcessingContext 32 | implements FixedKeyProcessorContext { 33 | private final @NonNull FixedKeyProcessorContext> wrapped; 34 | 35 | ErrorCapturingFixedKeyProcessorContext( 36 | final @NonNull FixedKeyProcessorContext> wrapped) { 37 | super(wrapped); 38 | this.wrapped = wrapped; 39 | } 40 | 41 | private static FixedKeyRecord> getValue( 42 | final FixedKeyRecord outputRecord) { 43 | final VR value = outputRecord.value(); 44 | final ProcessedValue recordWithOldKey = SuccessValue.of(value); 45 | return outputRecord.withValue(recordWithOldKey); 46 | } 47 | 48 | @Override 49 | public void forward(final FixedKeyRecord outputRecord) { 50 | final FixedKeyRecord> recordWithOldKey = getValue(outputRecord); 51 | this.wrapped.forward(recordWithOldKey); 52 | } 53 | 54 | @Override 55 | public void forward(final FixedKeyRecord outputRecord, 56 | final String childName) { 57 | final FixedKeyRecord> recordWithOldKey = getValue(outputRecord); 58 | this.wrapped.forward(recordWithOldKey, childName); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorDescribingKeyValueMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2020 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.AccessLevel; 28 | import lombok.NonNull; 29 | import lombok.RequiredArgsConstructor; 30 | import org.apache.kafka.streams.kstream.KeyValueMapper; 31 | 32 | /** 33 | * Wrap a {@code KeyValueMapper} and describe thrown exceptions with input key and value. 34 | * 35 | * @param type of input keys 36 | * @param type of input values 37 | * @param type of map result 38 | * @see #describeErrors(KeyValueMapper) 39 | */ 40 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 41 | public final class ErrorDescribingKeyValueMapper implements KeyValueMapper { 42 | private final @NonNull KeyValueMapper wrapped; 43 | 44 | /** 45 | * Wrap a {@code KeyValueMapper} and describe thrown exceptions with input key and value. 46 | * 
{@code
47 |      * final KeyValueMapper> mapper = ...;
48 |      * final KStream input = ...;
49 |      * final KStream output = input.map(describeErrors(mapper));
50 |      * }
51 |      *
52 | * 53 | * @param mapper {@code KeyValueMapper} whose exceptions should be described 54 | * @param type of input keys 55 | * @param type of input values 56 | * @param type of map result 57 | * @return {@code KeyValueMapper} 58 | */ 59 | public static KeyValueMapper describeErrors( 60 | final @NonNull KeyValueMapper mapper) { 61 | return new ErrorDescribingKeyValueMapper<>(mapper); 62 | } 63 | 64 | @Override 65 | public R apply(final K key, final V value) { 66 | try { 67 | return this.wrapped.apply(key, value); 68 | } catch (final Exception e) { 69 | throw new ProcessingException(key, value, e); 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorDescribingValueMapperWithKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2020 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.AccessLevel; 28 | import lombok.NonNull; 29 | import lombok.RequiredArgsConstructor; 30 | import org.apache.kafka.streams.kstream.ValueMapperWithKey; 31 | 32 | /** 33 | * Wrap a {@code ValueMapperWithKey} and describe thrown exceptions with input key and value. 34 | * 35 | * @param type of input keys 36 | * @param type of input values 37 | * @param type of output values 38 | */ 39 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 40 | public final class ErrorDescribingValueMapperWithKey implements ValueMapperWithKey { 41 | private final @NonNull ValueMapperWithKey wrapped; 42 | 43 | /** 44 | * Wrap a {@code ValueMapperWithKey} and describe thrown exceptions with input key and value. 45 | * 
{@code
46 |      * final ValueMapperWithKey mapper = ...;
47 |      * final KStream input = ...;
48 |      * final KStream output = input.mapValues(describeErrors(mapper));
49 |      * }
50 |      *
51 | * 52 | * @param mapper {@code ValueMapperWithKey} whose exceptions should be described 53 | * @param type of input keys 54 | * @param type of input values 55 | * @param type of output values 56 | * @return {@code ValueMapperWithKey} 57 | */ 58 | public static ValueMapperWithKey describeErrors( 59 | final @NonNull ValueMapperWithKey mapper) { 60 | return new ErrorDescribingValueMapperWithKey<>(mapper); 61 | } 62 | 63 | @Override 64 | public VR apply(final K key, final V value) { 65 | try { 66 | return this.wrapped.apply(key, value); 67 | } catch (final Exception e) { 68 | throw new ProcessingException(key, value, e); 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /error-handling-core/src/test/java/com/bakdata/kafka/ErrorUtilTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static org.assertj.core.api.Assertions.assertThat; 28 | import static org.mockito.Mockito.mock; 29 | 30 | import java.util.stream.Stream; 31 | import org.apache.kafka.common.errors.SerializationException; 32 | import org.apache.kafka.streams.errors.StreamsException; 33 | import org.junit.jupiter.params.ParameterizedTest; 34 | import org.junit.jupiter.params.provider.Arguments; 35 | import org.junit.jupiter.params.provider.MethodSource; 36 | 37 | class ErrorUtilTest { 38 | 39 | static Stream generateConvertToStringParameters() { 40 | return Stream.of( 41 | Arguments.of(1, "1"), 42 | Arguments.of(null, "null"), 43 | Arguments.of(TestValue.newBuilder().setField1("foo").setField2("bar").build(), 44 | "{\"optional_field\":null,\"field1\":{\"string\":\"foo\"},\"field2\":{\"string\":\"bar\"}}") 45 | ); 46 | } 47 | 48 | static Stream generateIsRecoverableExceptionParameters() { 49 | return Stream.of( 50 | Arguments.of(mock(Exception.class), false), 51 | Arguments.of(new SerializationException(), true), 52 | Arguments.of(new StreamsException("message"), true) 53 | ); 54 | } 55 | 56 | @ParameterizedTest 57 | @MethodSource("generateConvertToStringParameters") 58 | void shouldConvertToString(final Object object, final String expected) { 59 | assertThat(ErrorUtil.toString(object)) 60 | .as("Convert %s", object) 61 | .isEqualTo(expected); 62 | } 63 | 64 | @ParameterizedTest 65 | @MethodSource("generateIsRecoverableExceptionParameters") 66 | void shouldClassifyRecoverableErrors(final Exception exception, final boolean isRecoverable) { 67 | assertThat(ErrorUtil.isRecoverable(exception)) 68 | .isEqualTo(isRecoverable); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | @rem SPDX-License-Identifier: Apache-2.0 17 | @rem 18 | 19 | @if "%DEBUG%"=="" @echo off 20 | @rem ########################################################################## 21 | @rem 22 | @rem Gradle startup script for Windows 23 | @rem 24 | @rem ########################################################################## 25 | 26 | @rem Set local scope for the variables with windows NT shell 27 | if "%OS%"=="Windows_NT" setlocal 28 | 29 | set DIRNAME=%~dp0 30 | if "%DIRNAME%"=="" set DIRNAME=. 31 | @rem This is normally unused 32 | set APP_BASE_NAME=%~n0 33 | set APP_HOME=%DIRNAME% 34 | 35 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 36 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 37 | 38 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 39 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 40 | 41 | @rem Find java.exe 42 | if defined JAVA_HOME goto findJavaFromJavaHome 43 | 44 | set JAVA_EXE=java.exe 45 | %JAVA_EXE% -version >NUL 2>&1 46 | if %ERRORLEVEL% equ 0 goto execute 47 | 48 | echo. 1>&2 49 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 50 | echo. 1>&2 51 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 52 | echo location of your Java installation. 1>&2 53 | 54 | goto fail 55 | 56 | :findJavaFromJavaHome 57 | set JAVA_HOME=%JAVA_HOME:"=% 58 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 59 | 60 | if exist "%JAVA_EXE%" goto execute 61 | 62 | echo. 1>&2 63 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 64 | echo. 1>&2 65 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 66 | echo location of your Java installation. 1>&2 67 | 68 | goto fail 69 | 70 | :execute 71 | @rem Setup the command line 72 | 73 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 74 | 75 | 76 | @rem Execute Gradle 77 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 78 | 79 | :end 80 | @rem End local scope for the variables with windows NT shell 81 | if %ERRORLEVEL% equ 0 goto mainEnd 82 | 83 | :fail 84 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 85 | rem the _cmd.exe /c_ return code! 86 | set EXIT_CODE=%ERRORLEVEL% 87 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 88 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 89 | exit /b %EXIT_CODE% 90 | 91 | :mainEnd 92 | if "%OS%"=="Windows_NT" endlocal 93 | 94 | :omega 95 | -------------------------------------------------------------------------------- /error-handling-core/src/testFixtures/java/com/bakdata/kafka/ErrorCaptureTopologyTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import com.bakdata.fluent_kafka_streams_tests.TestTopology; 28 | import java.util.HashMap; 29 | import java.util.Map; 30 | import org.apache.kafka.clients.producer.ProducerConfig; 31 | import org.apache.kafka.common.errors.SerializationException; 32 | import org.apache.kafka.common.serialization.Serdes.IntegerSerde; 33 | import org.apache.kafka.streams.StreamsBuilder; 34 | import org.apache.kafka.streams.StreamsConfig; 35 | import org.apache.kafka.streams.Topology; 36 | import org.junit.jupiter.api.AfterEach; 37 | 38 | public abstract class ErrorCaptureTopologyTest { 39 | protected TestTopology topology = null; 40 | 41 | protected static RuntimeException createRecoverableException() { 42 | return new SerializationException(); 43 | } 44 | 45 | @AfterEach 46 | void tearDown() { 47 | if (this.topology != null) { 48 | this.topology.stop(); 49 | } 50 | } 51 | 52 | protected Map getKafkaProperties() { 53 | final Map kafkaConfig = new HashMap<>(); 54 | 55 | // exactly once and order 56 | kafkaConfig.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE_V2); 57 | kafkaConfig.put(StreamsConfig.producerPrefix(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), 1); 58 | 59 | kafkaConfig.put(StreamsConfig.producerPrefix(ProducerConfig.ACKS_CONFIG), "all"); 60 | 61 | // topology 62 | kafkaConfig.put(StreamsConfig.APPLICATION_ID_CONFIG, "fake"); 63 | kafkaConfig.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, IntegerSerde.class); 64 | kafkaConfig.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, TestDeadLetterSerde.class); 65 | kafkaConfig.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "fake"); 66 | 67 | return kafkaConfig; 68 | } 69 | 70 | protected void createTopology() { 71 | final StreamsBuilder builder = new StreamsBuilder(); 72 | this.buildTopology(builder); 73 | final Topology topology = builder.build(); 74 | this.topology = new TestTopology<>(topology, this.getKafkaProperties()); 75 | this.topology.start(); 76 | } 77 | 78 | protected abstract void buildTopology(StreamsBuilder builder); 79 | } 80 | -------------------------------------------------------------------------------- /error-handling-avro/src/main/java/com/bakdata/kafka/AvroDeadLetterConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; 28 | 29 | /** 30 | * Convert a {@code DeadLetterDescription} to an Avro {@code DeadLetter} 31 | */ 32 | public final class AvroDeadLetterConverter implements DeadLetterConverter { 33 | 34 | @Override 35 | public DeadLetter convert(final DeadLetterDescription deadLetterDescription) { 36 | return DeadLetter.newBuilder() 37 | .setInputValue(deadLetterDescription.getInputValue()) 38 | .setCause(ErrorDescription.newBuilder() 39 | .setMessage(deadLetterDescription.getCause().getMessage()) 40 | .setStackTrace(deadLetterDescription.getCause().getStackTrace()) 41 | .setErrorClass(deadLetterDescription.getCause().getErrorClass()) 42 | .build()) 43 | .setDescription(deadLetterDescription.getDescription()) 44 | .setTopic(deadLetterDescription.getTopic()) 45 | .setPartition(deadLetterDescription.getPartition()) 46 | .setOffset(deadLetterDescription.getOffset()) 47 | .setInputTimestamp(deadLetterDescription.getInputTimestamp()) 48 | .build(); 49 | } 50 | 51 | /** 52 | * Creates a processor that uses the AvroDeadLetterConverter 53 | * 54 | * 
{@code
55 |      * // Example, this works for all error capturing topologies
56 |      * final KeyValueMapper> mapper = ...;
57 |      * final KStream input = ...;
58 |      * final KStream> processed = input.map(captureErrors(mapper));
59 |      * final KStream output = processed.flatMapValues(ProcessedKeyValue::getValues);
60 |      * final KStream> errors = processed.flatMap(ProcessedKeyValue::getErrors);
61 |      * final KStream deadLetters = errors.processValues(
62 |      *                      AvroDeadLetterConverter.asProcessor("Description"));
63 |      * deadLetters.to(ERROR_TOPIC);
64 |      * }
65 |      *
66 | * 67 | * @param description shared description for all errors 68 | * @param type of the input key 69 | * @param type of the input value 70 | * @return a processor supplier 71 | */ 72 | public static FixedKeyProcessorSupplier, DeadLetter> asProcessor( 73 | final String description) { 74 | return DeadLetterProcessor.create(description, new AvroDeadLetterConverter()); 75 | } 76 | 77 | } 78 | -------------------------------------------------------------------------------- /error-handling-avro/src/test/java/com/bakdata/kafka/AvroDeadLetterConverterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import org.assertj.core.api.SoftAssertions; 28 | import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; 29 | import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; 30 | import org.junit.jupiter.api.Test; 31 | import org.junit.jupiter.api.extension.ExtendWith; 32 | 33 | @ExtendWith(SoftAssertionsExtension.class) 34 | class AvroDeadLetterConverterTest { 35 | 36 | @InjectSoftAssertions 37 | private SoftAssertions softly; 38 | 39 | @Test 40 | void shouldConvertDeadLetterDescriptionWithOptionalFields() { 41 | final AvroDeadLetterConverter converter = new AvroDeadLetterConverter(); 42 | final DeadLetterDescription deadLetterDescription = DeadLetterDescription.builder() 43 | .inputValue("inputValue") 44 | .cause(DeadLetterDescription.Cause.builder() 45 | .message("message") 46 | .stackTrace("stackTrace") 47 | .errorClass("errorClass") 48 | .build()) 49 | .description("description") 50 | .topic("topic") 51 | .partition(1) 52 | .offset(1L) 53 | .build(); 54 | 55 | final DeadLetter deadLetter = converter.convert(deadLetterDescription); 56 | this.softly.assertThat(deadLetter.getInputValue()).hasValue("inputValue"); 57 | this.softly.assertThat(deadLetter.getCause().getMessage()).hasValue("message"); 58 | this.softly.assertThat(deadLetter.getCause().getStackTrace()).hasValue("stackTrace"); 59 | this.softly.assertThat(deadLetter.getCause().getErrorClass()).hasValue("errorClass"); 60 | this.softly.assertThat(deadLetter.getDescription()).isEqualTo("description"); 61 | this.softly.assertThat(deadLetter.getPartition()).hasValue(1); 62 | this.softly.assertThat(deadLetter.getOffset()).hasValue(1L); 63 | } 64 | 65 | @Test 66 | void shouldConvertDeadLetterDescriptionWithoutOptionalFields() { 67 | final AvroDeadLetterConverter converter = new AvroDeadLetterConverter(); 68 | final DeadLetterDescription onlyRequiredFieldsDeadLetterDescription = DeadLetterDescription.builder() 69 | .description("description") 70 | .cause(DeadLetterDescription.Cause.builder().build()) 71 | .build(); 72 | final DeadLetter deadLetter = converter.convert(onlyRequiredFieldsDeadLetterDescription); 73 | this.softly.assertThat(deadLetter.getInputValue()).isNotPresent(); 74 | this.softly.assertThat(deadLetter.getCause().getMessage()).isNotPresent(); 75 | this.softly.assertThat(deadLetter.getCause().getStackTrace()).isNotPresent(); 76 | this.softly.assertThat(deadLetter.getCause().getErrorClass()).isNotPresent(); 77 | this.softly.assertThat(deadLetter.getDescription()).isEqualTo("description"); 78 | this.softly.assertThat(deadLetter.getPartition()).isNotPresent(); 79 | this.softly.assertThat(deadLetter.getOffset()).isNotPresent(); 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorCapturingValueMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2020 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.function.Predicate; 28 | import lombok.AccessLevel; 29 | import lombok.NonNull; 30 | import lombok.RequiredArgsConstructor; 31 | import org.apache.kafka.streams.kstream.ValueMapper; 32 | 33 | /** 34 | * Wrap a {@code ValueMapper} and capture thrown exceptions. 35 | * 36 | * @param type of input values 37 | * @param type of output values 38 | * @see #captureErrors(ValueMapper) 39 | * @see #captureErrors(ValueMapper, Predicate) 40 | */ 41 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 42 | public final class ErrorCapturingValueMapper implements ValueMapper> { 43 | private final @NonNull ValueMapper wrapped; 44 | private final @NonNull Predicate errorFilter; 45 | 46 | /** 47 | * Wrap a {@code ValueMapper} and capture thrown exceptions. Recoverable Kafka exceptions such as a schema registry 48 | * timeout are forwarded and not captured. 49 | * 50 | * @param mapper {@code ValueMapper} whose exceptions should be captured 51 | * @param type of input values 52 | * @param type of output values 53 | * @return {@code ValueMapper} 54 | * @see #captureErrors(ValueMapper, Predicate) 55 | * @see ErrorUtil#isRecoverable(Exception) 56 | */ 57 | public static ValueMapper> captureErrors( 58 | final @NonNull ValueMapper mapper) { 59 | return captureErrors(mapper, ErrorUtil::isRecoverable); 60 | } 61 | 62 | /** 63 | * Wrap a {@code ValueMapper} and capture thrown exceptions. 64 | * 
{@code
65 |      * final ValueMapper mapper = ...;
66 |      * final KStream input = ...;
67 |      * final KStream> processed = input.mapValues(captureErrors(mapper));
68 |      * final KStream output = processed.flatMapValues(ProcessedValue::getValues);
69 |      * final KStream> errors = processed.flatMapValues(ProcessedValue::getErrors);
70 |      * }
71 |      *
72 | * 73 | * @param mapper {@code ValueMapper} whose exceptions should be captured 74 | * @param errorFilter expression that filters errors which should be thrown and not captured 75 | * @param type of input values 76 | * @param type of output values 77 | * @return {@code ValueMapper} 78 | */ 79 | public static ValueMapper> captureErrors( 80 | final @NonNull ValueMapper mapper, 81 | final @NonNull Predicate errorFilter) { 82 | return new ErrorCapturingValueMapper<>(mapper, errorFilter); 83 | } 84 | 85 | @Override 86 | public ProcessedValue apply(final V value) { 87 | try { 88 | final VR newValue = this.wrapped.apply(value); 89 | return SuccessValue.of(newValue); 90 | } catch (final Exception e) { 91 | if (this.errorFilter.test(e)) { 92 | throw e; 93 | } 94 | return ErrorValue.of(value, e); 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ProcessedKeyValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2020 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import org.apache.kafka.streams.KeyValue; 28 | 29 | /** 30 | * A processed key-value is created upon capturing errors in a streams topology. It can either contain a successfully 31 | * processed value or the old key along with a {@link ProcessingError} describing the input value and the {@link 32 | * Exception} that has been thrown. 33 | * 34 | * @param the type of the old key before applying the error capturer 35 | * @param the type of the old value before applying the error capturer 36 | * @param the type of the new value after applying the error capturer 37 | */ 38 | public interface ProcessedKeyValue { 39 | 40 | /** 41 | *

This method serves as a utility method to extract errors from a previous key-value mapper step. It can be 42 | * used as a lambda method reference and simply delegates to {@link #getErrors()}. The new key is not relevant and 43 | * thus omitted.

44 | * 45 | * Usage example: 46 | * 
{@code
47 |      * final KStream> input = ...;
48 |      * final KStream> errors = input.flatMap(ProcessedKeyValue::getErrors);
49 |      * }
50 |      *
51 | * 52 | * @param newKey the new key of a processed key-value pair. As this method extracts errors, the new key is not 53 | * relevant and omitted. It is only used as a parameter to use a method reference lambda when creating a streams 54 | * topology. 55 | * @param recordWithOldKey a processed key-value pair containing a successfully extracted record or a {@link 56 | * ProcessingError} 57 | * @param the type of the old key before applying the error capturer 58 | * @param the type of the old value before applying the error capturer 59 | * @param the type of the new key after applying the error capturer 60 | * @param the type of the new value after applying the error capturer 61 | * @return A single key-value pair containing the old key and a {@link ProcessingError} if an exception was thrown 62 | * upon processing the input key-value pair or an empty list 63 | */ 64 | static Iterable>> getErrors( 65 | @SuppressWarnings("unused") final KR newKey, final ProcessedKeyValue recordWithOldKey) { 66 | return recordWithOldKey.getErrors(); 67 | } 68 | 69 | /** 70 | * Extract errors from a processed key-value. If an error is available, it will give information about the input 71 | * key, value, and the {@link Exception} that was thrown while attempting to map it to a new value. 72 | * 73 | * @return A single key-value pair containing the old key and a {@link ProcessingError} if an exception was thrown 74 | * upon processing the input key-value pair or an empty list 75 | */ 76 | Iterable>> getErrors(); 77 | 78 | /** 79 | * Extract successfully processed values from a processed key-value. 80 | * 81 | * @return A single value if processing was successful or an empty list; 82 | */ 83 | Iterable getValues(); 84 | } 85 | -------------------------------------------------------------------------------- /error-handling-proto/src/test/java/com/bakdata/kafka/ProtoDeadLetterConverterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import com.bakdata.kafka.proto.v1.ProtoDeadLetter; 28 | import org.assertj.core.api.SoftAssertions; 29 | import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; 30 | import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; 31 | import org.junit.jupiter.api.Test; 32 | import org.junit.jupiter.api.extension.ExtendWith; 33 | 34 | @ExtendWith(SoftAssertionsExtension.class) 35 | class ProtoDeadLetterConverterTest { 36 | 37 | @InjectSoftAssertions 38 | private SoftAssertions softly; 39 | 40 | @Test 41 | void shouldConvertDeadletterDescriptionWithOptionalFields() { 42 | final ProtoDeadLetterConverter converter = new ProtoDeadLetterConverter(); 43 | final DeadLetterDescription deadLetterDescription = DeadLetterDescription.builder() 44 | .inputValue("inputValue") 45 | .cause(DeadLetterDescription.Cause.builder() 46 | .message("message") 47 | .stackTrace("stackTrace") 48 | .errorClass("errorClass") 49 | .build()) 50 | .description("description") 51 | .topic("topic") 52 | .partition(1) 53 | .offset(1L) 54 | .build(); 55 | 56 | final ProtoDeadLetter deadLetter = converter.convert(deadLetterDescription); 57 | this.softly.assertThat(deadLetter.getInputValue().getValue()).isEqualTo("inputValue"); 58 | this.softly.assertThat(deadLetter.getCause().getMessage().getValue()).isEqualTo("message"); 59 | this.softly.assertThat(deadLetter.getCause().getStackTrace().getValue()).isEqualTo("stackTrace"); 60 | this.softly.assertThat(deadLetter.getCause().getErrorClass().getValue()).isEqualTo("errorClass"); 61 | this.softly.assertThat(deadLetter.getDescription()).isEqualTo("description"); 62 | this.softly.assertThat(deadLetter.getPartition().getValue()).isEqualTo(1); 63 | this.softly.assertThat(deadLetter.getOffset().getValue()).isEqualTo(1L); 64 | } 65 | 66 | @Test 67 | void shouldConvertDeadletterDescriptionWithoutOptionalFields() { 68 | final ProtoDeadLetterConverter converter = new ProtoDeadLetterConverter(); 69 | final DeadLetterDescription onlyRequiredFieldsDeadLetterDescription = DeadLetterDescription.builder() 70 | .description("description") 71 | .cause(DeadLetterDescription.Cause.builder().build()) 72 | .build(); 73 | final ProtoDeadLetter deadLetter = converter.convert(onlyRequiredFieldsDeadLetterDescription); 74 | this.softly.assertThat(deadLetter.hasInputValue()).isFalse(); 75 | this.softly.assertThat(deadLetter.getCause().hasMessage()).isFalse(); 76 | this.softly.assertThat(deadLetter.getCause().hasStackTrace()).isFalse(); 77 | this.softly.assertThat(deadLetter.getCause().hasErrorClass()).isFalse(); 78 | this.softly.assertThat(deadLetter.getDescription()).isEqualTo("description"); 79 | this.softly.assertThat(deadLetter.hasPartition()).isFalse(); 80 | this.softly.assertThat(deadLetter.hasOffset()).isFalse(); 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorLoggingFlatValueMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static java.util.Collections.emptyList; 28 | 29 | import java.util.function.Predicate; 30 | import lombok.AccessLevel; 31 | import lombok.NonNull; 32 | import lombok.RequiredArgsConstructor; 33 | import lombok.extern.slf4j.Slf4j; 34 | import org.apache.kafka.streams.kstream.ValueMapper; 35 | 36 | /** 37 | * Wrap a {@code ValueMapper} and log thrown exceptions with input key and value. 38 | * 39 | * @param type of input values 40 | * @param type of output values 41 | * @see #logErrors(ValueMapper) 42 | * @see #logErrors(ValueMapper, Predicate) 43 | */ 44 | @Slf4j 45 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 46 | public final class ErrorLoggingFlatValueMapper implements ValueMapper> { 47 | private final @NonNull ValueMapper> wrapped; 48 | private final @NonNull Predicate errorFilter; 49 | 50 | /** 51 | * Wrap a {@code ValueMapper} and log thrown exceptions with input key and value. Recoverable Kafka exceptions such 52 | * as a schema registry timeout are forwarded and not captured. 53 | * 54 | * @param mapper {@code ValueMapper} whose exceptions should be logged 55 | * @param type of input values 56 | * @param type of output values 57 | * @return {@code ValueMapper} 58 | * @see #logErrors(ValueMapper, Predicate) 59 | * @see ErrorUtil#isRecoverable(Exception) 60 | */ 61 | public static ValueMapper> logErrors( 62 | final @NonNull ValueMapper> mapper) { 63 | return logErrors(mapper, ErrorUtil::isRecoverable); 64 | } 65 | 66 | /** 67 | * Wrap a {@code ValueMapper} and log thrown exceptions with input key and value. 68 | * 
{@code
 69 |      * final ValueMapper> mapper = ...;
 70 |      * final KStream input = ...;
 71 |      * final KStream output = input.flatMapValues(logErrors(mapper));
 72 |      * }
 73 |      *
74 | * 75 | * @param mapper {@code ValueMapper} whose exceptions should be logged 76 | * @param errorFilter expression that filters errors which should be thrown and not logged 77 | * @param type of input values 78 | * @param type of output values 79 | * @return {@code ValueMapper} 80 | */ 81 | public static ValueMapper> logErrors( 82 | final @NonNull ValueMapper> mapper, 83 | final @NonNull Predicate errorFilter) { 84 | return new ErrorLoggingFlatValueMapper<>(mapper, errorFilter); 85 | } 86 | 87 | @Override 88 | public Iterable apply(final V value) { 89 | try { 90 | return this.wrapped.apply(value); 91 | } catch (final Exception e) { 92 | if (this.errorFilter.test(e)) { 93 | throw e; 94 | } 95 | log.error("Cannot process {}", ErrorUtil.toString(value), e); 96 | return emptyList(); 97 | } 98 | } 99 | 100 | } 101 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorLoggingValueMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static java.util.Collections.emptyList; 28 | 29 | import java.util.Collections; 30 | import java.util.function.Predicate; 31 | import lombok.AccessLevel; 32 | import lombok.NonNull; 33 | import lombok.RequiredArgsConstructor; 34 | import lombok.extern.slf4j.Slf4j; 35 | import org.apache.kafka.streams.kstream.ValueMapper; 36 | 37 | /** 38 | * Wrap a {@code ValueMapper} and log thrown exceptions with input key and value. 39 | * 40 | * @param type of input values 41 | * @param type of output values 42 | * @see #logErrors(ValueMapper) 43 | * @see #logErrors(ValueMapper, Predicate) 44 | */ 45 | @Slf4j 46 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 47 | public final class ErrorLoggingValueMapper implements ValueMapper> { 48 | private final @NonNull ValueMapper wrapped; 49 | private final @NonNull Predicate errorFilter; 50 | 51 | /** 52 | * Wrap a {@code ValueMapper} and log thrown exceptions with input key and value. Recoverable Kafka exceptions such 53 | * as a schema registry timeout are forwarded and not captured. 54 | * 55 | * @param mapper {@code ValueMapper} whose exceptions should be logged 56 | * @param type of input values 57 | * @param type of output values 58 | * @return {@code ValueMapper} 59 | * @see #logErrors(ValueMapper, Predicate) 60 | * @see ErrorUtil#isRecoverable(Exception) 61 | */ 62 | public static ValueMapper> logErrors( 63 | final @NonNull ValueMapper mapper) { 64 | return logErrors(mapper, ErrorUtil::isRecoverable); 65 | } 66 | 67 | /** 68 | * Wrap a {@code ValueMapper} and log thrown exceptions with input key and value. 69 | * 
{@code
 70 |      * final ValueMapper mapper = ...;
 71 |      * final KStream input = ...;
 72 |      * final KStream output = input.mapValues(logErrors(mapper));
 73 |      * }
 74 |      *
75 | * 76 | * @param mapper {@code ValueMapper} whose exceptions should be logged 77 | * @param errorFilter expression that filters errors which should be thrown and not logged 78 | * @param type of input values 79 | * @param type of output values 80 | * @return {@code ValueMapper} 81 | */ 82 | public static ValueMapper> logErrors( 83 | final @NonNull ValueMapper mapper, 84 | final @NonNull Predicate errorFilter) { 85 | return new ErrorLoggingValueMapper<>(mapper, errorFilter); 86 | } 87 | 88 | @Override 89 | public Iterable apply(final V value) { 90 | try { 91 | final VR newValue = this.wrapped.apply(value); 92 | // allow null values 93 | return Collections.singletonList(newValue); 94 | } catch (final Exception e) { 95 | if (this.errorFilter.test(e)) { 96 | throw e; 97 | } 98 | log.error("Cannot process {}", ErrorUtil.toString(value), e); 99 | return emptyList(); 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorDescribingProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.Set; 28 | import lombok.NonNull; 29 | import org.apache.kafka.streams.processor.api.Processor; 30 | import org.apache.kafka.streams.processor.api.ProcessorSupplier; 31 | import org.apache.kafka.streams.processor.api.Record; 32 | import org.apache.kafka.streams.state.StoreBuilder; 33 | 34 | /** 35 | * Wrap a {@code Processor} and describe thrown exceptions with input key and value. 36 | * 37 | * @param type of input keys 38 | * @param type of input values 39 | * @param type of output keys 40 | * @param type of output values 41 | * @see #describeErrors(Processor) 42 | */ 43 | public final class ErrorDescribingProcessor extends DecoratorProcessor { 44 | 45 | private ErrorDescribingProcessor(final @NonNull Processor wrapped) { 46 | super(wrapped); 47 | } 48 | 49 | /** 50 | * Wrap a {@code Processor} and describe thrown exceptions with input key and value. 51 | * 
{@code
 52 |      * final KStream input = ...;
 53 |      * final KStream output = input.process(() -> describeErrors(new Processor() {...}));
 54 |      * }
 55 |      *
56 | * 57 | * @param processor {@code Processor} whose exceptions should be described 58 | * @param type of input keys 59 | * @param type of input values 60 | * @param type of output keys 61 | * @param type of output values 62 | * @return {@code Processor} 63 | */ 64 | public static Processor describeErrors( 65 | final @NonNull Processor processor) { 66 | return new ErrorDescribingProcessor<>(processor); 67 | } 68 | 69 | /** 70 | * Wrap a {@code ProcessorSupplier} and describe thrown exceptions with input key and value. 71 | * 
{@code
 72 |      * final ProcessorSupplier processor = ...;
 73 |      * final KStream input = ...;
 74 |      * final KStream output = input.process(describeErrors(processor));
 75 |      * }
 76 |      *
77 | * 78 | * @param supplier {@code ProcessorSupplier} whose exceptions should be described 79 | * @param type of input keys 80 | * @param type of input values 81 | * @param type of output keys 82 | * @param type of output values 83 | * @return {@code ProcessorSupplier} 84 | */ 85 | public static ProcessorSupplier describeErrors( 86 | final @NonNull ProcessorSupplier supplier) { 87 | return new ProcessorSupplier<>() { 88 | @Override 89 | public Set> stores() { 90 | return supplier.stores(); 91 | } 92 | 93 | @Override 94 | public Processor get() { 95 | return describeErrors(supplier.get()); 96 | } 97 | }; 98 | } 99 | 100 | @Override 101 | public void process(final Record inputRecord) { 102 | try { 103 | super.process(inputRecord); 104 | } catch (final Exception e) { 105 | throw new ProcessingException(inputRecord.key(), inputRecord.value(), e); 106 | } 107 | } 108 | 109 | } 110 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorDescribingValueProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.Set; 28 | import lombok.NonNull; 29 | import org.apache.kafka.streams.processor.api.FixedKeyProcessor; 30 | import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; 31 | import org.apache.kafka.streams.processor.api.FixedKeyRecord; 32 | import org.apache.kafka.streams.state.StoreBuilder; 33 | 34 | /** 35 | * Wrap a {@code FixedKeyProcessor} and describe thrown exceptions with input key and value. 36 | * 37 | * @param type of input keys 38 | * @param type of input values 39 | * @param type of output values 40 | * @see #describeErrors(FixedKeyProcessor) 41 | */ 42 | public final class ErrorDescribingValueProcessor extends DecoratorValueProcessor { 43 | 44 | private ErrorDescribingValueProcessor(final @NonNull FixedKeyProcessor wrapped) { 45 | super(wrapped); 46 | } 47 | 48 | /** 49 | * Wrap a {@code FixedKeyProcessor} and describe thrown exceptions with input key and value. 50 | * 
{@code
 51 |      * final KStream input = ...;
 52 |      * final KStream output = input.processValues(() -> describeErrors(new FixedKeyProcessor() {...}));
 53 |      * }
 54 |      *
55 | * 56 | * @param processor {@code FixedKeyProcessor} whose exceptions should be described 57 | * @param type of input keys 58 | * @param type of input values 59 | * @param type of output values 60 | * @return {@code FixedKeyProcessor} 61 | */ 62 | public static FixedKeyProcessor describeErrors( 63 | final @NonNull FixedKeyProcessor processor) { 64 | return new ErrorDescribingValueProcessor<>(processor); 65 | } 66 | 67 | /** 68 | * Wrap a {@code FixedKeyProcessorSupplier} and describe thrown exceptions with input key and value. 69 | * 
{@code
 70 |      * final FixedKeyProcessorSupplier processor = ...;
 71 |      * final KStream input = ...;
 72 |      * final KStream output = input.processValues(describeErrors(processor));
 73 |      * }
 74 |      *
75 | * 76 | * @param supplier {@code FixedKeyProcessorSupplier} whose exceptions should be described 77 | * @param type of input keys 78 | * @param type of input values 79 | * @param type of output values 80 | * @return {@code FixedKeyProcessorSupplier} 81 | */ 82 | public static FixedKeyProcessorSupplier describeErrors( 83 | final @NonNull FixedKeyProcessorSupplier supplier) { 84 | return new FixedKeyProcessorSupplier<>() { 85 | @Override 86 | public Set> stores() { 87 | return supplier.stores(); 88 | } 89 | 90 | @Override 91 | public FixedKeyProcessor get() { 92 | return describeErrors(supplier.get()); 93 | } 94 | }; 95 | } 96 | 97 | @Override 98 | public void process(final FixedKeyRecord inputRecord) { 99 | try { 100 | super.process(inputRecord); 101 | } catch (final Exception e) { 102 | throw new ProcessingException(inputRecord.key(), inputRecord.value(), e); 103 | } 104 | } 105 | 106 | } 107 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorLoggingFlatKeyValueMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static java.util.Collections.emptyList; 28 | 29 | import java.util.function.Predicate; 30 | import lombok.AccessLevel; 31 | import lombok.NonNull; 32 | import lombok.RequiredArgsConstructor; 33 | import lombok.extern.slf4j.Slf4j; 34 | import org.apache.kafka.streams.kstream.KeyValueMapper; 35 | 36 | /** 37 | * Wrap a {@code KeyValueMapper} and log thrown exceptions with input key and value. 38 | * 39 | * @param type of input keys 40 | * @param type of input values 41 | * @param type of map result 42 | * @see #logErrors(KeyValueMapper) 43 | * @see #logErrors(KeyValueMapper, Predicate) 44 | */ 45 | @Slf4j 46 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 47 | public final class ErrorLoggingFlatKeyValueMapper implements KeyValueMapper> { 48 | private final @NonNull KeyValueMapper> wrapped; 49 | private final @NonNull Predicate errorFilter; 50 | 51 | /** 52 | * Wrap a {@code KeyValueMapper} and log thrown exceptions with input key and value. Recoverable Kafka exceptions 53 | * such as a schema registry timeout are forwarded and not captured. 54 | * 55 | * @param mapper {@code KeyValueMapper} whose exceptions should be logged 56 | * @param type of input keys 57 | * @param type of input values 58 | * @param type of map result 59 | * @return {@code KeyValueMapper} 60 | * @see #logErrors(KeyValueMapper, Predicate) 61 | * @see ErrorUtil#isRecoverable(Exception) 62 | */ 63 | public static KeyValueMapper> logErrors( 64 | final @NonNull KeyValueMapper> mapper) { 65 | return logErrors(mapper, ErrorUtil::isRecoverable); 66 | } 67 | 68 | /** 69 | * Wrap a {@code KeyValueMapper} and log thrown exceptions with input key and value. 70 | * 
{@code
 71 |      * final KeyValueMapper>> mapper = ...;
 72 |      * final KStream input = ...;
 73 |      * final KStream output = input.flatMap(logErrors(mapper));
 74 |      * }
 75 |      *
76 | * 77 | * @param mapper {@code KeyValueMapper} whose exceptions should be logged 78 | * @param errorFilter expression that filters errors which should be thrown and not logged 79 | * @param type of input keys 80 | * @param type of input values 81 | * @param type of map result 82 | * @return {@code KeyValueMapper} 83 | */ 84 | public static KeyValueMapper> logErrors( 85 | final @NonNull KeyValueMapper> mapper, 86 | final @NonNull Predicate errorFilter) { 87 | return new ErrorLoggingFlatKeyValueMapper<>(mapper, errorFilter); 88 | } 89 | 90 | @Override 91 | public Iterable apply(final K key, final V value) { 92 | try { 93 | return this.wrapped.apply(key, value); 94 | } catch (final Exception e) { 95 | if (this.errorFilter.test(e)) { 96 | throw e; 97 | } 98 | log.error("Cannot process ('{}', '{}')", ErrorUtil.toString(key), ErrorUtil.toString(value), e); 99 | return emptyList(); 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorLoggingKeyValueMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static java.util.Collections.emptyList; 28 | 29 | import java.util.List; 30 | import java.util.function.Predicate; 31 | import lombok.AccessLevel; 32 | import lombok.NonNull; 33 | import lombok.RequiredArgsConstructor; 34 | import lombok.extern.slf4j.Slf4j; 35 | import org.apache.kafka.streams.kstream.KeyValueMapper; 36 | 37 | /** 38 | * Wrap a {@code KeyValueMapper} and log thrown exceptions with input key and value. 39 | * 40 | * @param type of input keys 41 | * @param type of input values 42 | * @param type of map result 43 | * @see #logErrors(KeyValueMapper) 44 | * @see #logErrors(KeyValueMapper, Predicate) 45 | */ 46 | @Slf4j 47 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 48 | public final class ErrorLoggingKeyValueMapper implements KeyValueMapper> { 49 | private final @NonNull KeyValueMapper wrapped; 50 | private final @NonNull Predicate errorFilter; 51 | 52 | /** 53 | * Wrap a {@code KeyValueMapper} and log thrown exceptions with input key and value. Recoverable Kafka exceptions 54 | * such as a schema registry timeout are forwarded and not captured. 55 | * 56 | * @param mapper {@code KeyValueMapper} whose exceptions should be logged 57 | * @param type of input keys 58 | * @param type of input values 59 | * @param type of map result 60 | * @return {@code KeyValueMapper} 61 | * @see #logErrors(KeyValueMapper, Predicate) 62 | * @see ErrorUtil#isRecoverable(Exception) 63 | */ 64 | public static KeyValueMapper> logErrors( 65 | final @NonNull KeyValueMapper mapper) { 66 | return logErrors(mapper, ErrorUtil::isRecoverable); 67 | } 68 | 69 | /** 70 | * Wrap a {@code KeyValueMapper} and log thrown exceptions with input key and value. 71 | * 
{@code
 72 |      * final KeyValueMapper> mapper = ...;
 73 |      * final KStream input = ...;
 74 |      * final KStream output = input.map(logErrors(mapper));
 75 |      * }
 76 |      *
77 | * 78 | * @param mapper {@code KeyValueMapper} whose exceptions should be logged 79 | * @param errorFilter expression that filters errors which should be thrown and not logged 80 | * @param type of input keys 81 | * @param type of input values 82 | * @param type of map result 83 | * @return {@code KeyValueMapper} 84 | */ 85 | public static KeyValueMapper> logErrors( 86 | final @NonNull KeyValueMapper mapper, 87 | final @NonNull Predicate errorFilter) { 88 | return new ErrorLoggingKeyValueMapper<>(mapper, errorFilter); 89 | } 90 | 91 | @Override 92 | public Iterable apply(final K key, final V value) { 93 | try { 94 | final R newKeyValue = this.wrapped.apply(key, value); 95 | return List.of(newKeyValue); 96 | } catch (final Exception e) { 97 | if (this.errorFilter.test(e)) { 98 | throw e; 99 | } 100 | log.error("Cannot process ('{}', '{}')", ErrorUtil.toString(key), ErrorUtil.toString(value), e); 101 | return emptyList(); 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorCapturingFlatValueMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static org.jooq.lambda.Seq.seq; 28 | 29 | import java.util.List; 30 | import java.util.function.Predicate; 31 | import lombok.AccessLevel; 32 | import lombok.NonNull; 33 | import lombok.RequiredArgsConstructor; 34 | import org.apache.kafka.streams.kstream.ValueMapper; 35 | 36 | /** 37 | * Wrap a {@code ValueMapper} and capture thrown exceptions. 38 | * 39 | * @param type of input values 40 | * @param type of output values 41 | * @see #captureErrors(ValueMapper) 42 | * @see #captureErrors(ValueMapper, Predicate) 43 | */ 44 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 45 | public final class ErrorCapturingFlatValueMapper implements ValueMapper>> { 46 | private final @NonNull ValueMapper> wrapped; 47 | private final @NonNull Predicate errorFilter; 48 | 49 | /** 50 | * Wrap a {@code ValueMapper} and capture thrown exceptions. Recoverable Kafka exceptions such as a schema registry 51 | * timeout are forwarded and not captured. 52 | * 53 | * @param mapper {@code ValueMapper} whose exceptions should be captured 54 | * @param type of input values 55 | * @param type of output values 56 | * @return {@code ValueMapper} 57 | * @see #captureErrors(ValueMapper, Predicate) 58 | * @see ErrorUtil#isRecoverable(Exception) 59 | */ 60 | public static ValueMapper>> captureErrors( 61 | final @NonNull ValueMapper> mapper) { 62 | return captureErrors(mapper, ErrorUtil::isRecoverable); 63 | } 64 | 65 | /** 66 | * Wrap a {@code ValueMapper} and capture thrown exceptions. 67 | * 
{@code
 68 |      * final ValueMapper> mapper = ...;
 69 |      * final KStream input = ...;
 70 |      * final KStream> processed = input.flatMapValues(captureErrors(mapper));
 71 |      * final KStream output = processed.flatMapValues(ProcessedValue::getValues);
 72 |      * final KStream> errors = processed.flatMapValues(ProcessedValue::getErrors);
 73 |      * }
 74 |      *
75 | * 76 | * @param mapper {@code ValueMapper} whose exceptions should be captured 77 | * @param errorFilter expression that filters errors which should be thrown and not captured 78 | * @param type of input values 79 | * @param type of output values 80 | * @return {@code ValueMapper} 81 | */ 82 | public static ValueMapper>> captureErrors( 83 | final @NonNull ValueMapper> mapper, 84 | final @NonNull Predicate errorFilter) { 85 | return new ErrorCapturingFlatValueMapper<>(mapper, errorFilter); 86 | } 87 | 88 | @Override 89 | public Iterable> apply(final V value) { 90 | try { 91 | final Iterable newValues = this.wrapped.apply(value); 92 | return seq(newValues).map(SuccessValue::of); 93 | } catch (final Exception e) { 94 | if (this.errorFilter.test(e)) { 95 | throw e; 96 | } 97 | return List.of(ErrorValue.of(value, e)); 98 | } 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorLoggingFlatValueMapperWithKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static java.util.Collections.emptyList; 28 | 29 | import java.util.function.Predicate; 30 | import lombok.AccessLevel; 31 | import lombok.NonNull; 32 | import lombok.RequiredArgsConstructor; 33 | import lombok.extern.slf4j.Slf4j; 34 | import org.apache.kafka.streams.kstream.ValueMapperWithKey; 35 | 36 | /** 37 | * Wrap a {@code ValueMapperWithKey} and log thrown exceptions with input key and value. 38 | * 39 | * @param type of input keys 40 | * @param type of input values 41 | * @param type of output values 42 | * @see #logErrors(ValueMapperWithKey) 43 | * @see #logErrors(ValueMapperWithKey, Predicate) 44 | */ 45 | @Slf4j 46 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 47 | public final class ErrorLoggingFlatValueMapperWithKey implements ValueMapperWithKey> { 48 | private final @NonNull ValueMapperWithKey> wrapped; 49 | private final @NonNull Predicate errorFilter; 50 | 51 | /** 52 | * Wrap a {@code ValueMapperWithKey} and log thrown exceptions with input key and value. Recoverable Kafka 53 | * exceptions such as a schema registry timeout are forwarded and not captured. 54 | * 55 | * @param mapper {@code ValueMapperWithKey} whose exceptions should be logged 56 | * @param type of input keys 57 | * @param type of input values 58 | * @param type of output values 59 | * @return {@code ValueMapperWithKey} 60 | * @see #logErrors(ValueMapperWithKey, Predicate) 61 | * @see ErrorUtil#isRecoverable(Exception) 62 | */ 63 | public static ValueMapperWithKey> logErrors( 64 | final @NonNull ValueMapperWithKey> mapper) { 65 | return logErrors(mapper, ErrorUtil::isRecoverable); 66 | } 67 | 68 | /** 69 | * Wrap a {@code ValueMapperWithKey} and log thrown exceptions with input key and value. 70 | * 
{@code
 71 |      * final ValueMapperWithKey> mapper = ...;
 72 |      * final KStream input = ...;
 73 |      * final KStream output = input.flatMapValues(logErrors(mapper));
 74 |      * }
 75 |      *
76 | * 77 | * @param mapper {@code ValueMapperWithKey} whose exceptions should be logged 78 | * @param errorFilter expression that filters errors which should be thrown and not logged 79 | * @param type of input keys 80 | * @param type of input values 81 | * @param type of output values 82 | * @return {@code ValueMapperWithKey} 83 | */ 84 | public static ValueMapperWithKey> logErrors( 85 | final @NonNull ValueMapperWithKey> mapper, 86 | final @NonNull Predicate errorFilter) { 87 | return new ErrorLoggingFlatValueMapperWithKey<>(mapper, errorFilter); 88 | } 89 | 90 | @Override 91 | public Iterable apply(final K key, final V value) { 92 | try { 93 | return this.wrapped.apply(key, value); 94 | } catch (final Exception e) { 95 | if (this.errorFilter.test(e)) { 96 | throw e; 97 | } 98 | log.error("Cannot process ('{}', '{}')", ErrorUtil.toString(key), ErrorUtil.toString(value), e); 99 | return emptyList(); 100 | } 101 | } 102 | } 103 | 104 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorCapturingValueMapperWithKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2020 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.function.Predicate; 28 | import lombok.AccessLevel; 29 | import lombok.NonNull; 30 | import lombok.RequiredArgsConstructor; 31 | import org.apache.kafka.streams.kstream.ValueMapperWithKey; 32 | 33 | /** 34 | * Wrap a {@code ValueMapperWithKey} and capture thrown exceptions. 35 | * 36 | * @param type of input keys 37 | * @param type of input values 38 | * @param type of output values 39 | * @see #captureErrors(ValueMapperWithKey) 40 | * @see #captureErrors(ValueMapperWithKey, Predicate) 41 | */ 42 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 43 | public final class ErrorCapturingValueMapperWithKey 44 | implements ValueMapperWithKey> { 45 | private final @NonNull ValueMapperWithKey wrapped; 46 | private final @NonNull Predicate errorFilter; 47 | 48 | /** 49 | * Wrap a {@code ValueMapperWithKey} and capture thrown exceptions. Recoverable Kafka exceptions such as a schema 50 | * registry timeout are forwarded and not captured. 51 | * 52 | * @param mapper {@code ValueMapperWithKey} whose exceptions should be captured 53 | * @param type of input keys 54 | * @param type of input values 55 | * @param type of output values 56 | * @return {@code ValueMapperWithKey} 57 | * @see #captureErrors(ValueMapperWithKey, Predicate) 58 | * @see ErrorUtil#isRecoverable(Exception) 59 | */ 60 | public static ValueMapperWithKey> captureErrors( 61 | final @NonNull ValueMapperWithKey mapper) { 62 | return captureErrors(mapper, ErrorUtil::isRecoverable); 63 | } 64 | 65 | /** 66 | * Wrap a {@code ValueMapperWithKey} and capture thrown exceptions. 67 | * 
{@code
 68 |      * final ValueMapperWithKey mapper = ...;
 69 |      * final KStream input = ...;
 70 |      * final KStream> processed = input.mapValues(captureErrors(mapper));
 71 |      * final KStream output = processed.flatMapValues(ProcessedValue::getValues);
 72 |      * final KStream> errors = processed.flatMapValues(ProcessedValue::getErrors);
 73 |      * }
 74 |      *
75 | * 76 | * @param mapper {@code ValueMapperWithKey} whose exceptions should be captured 77 | * @param errorFilter expression that filters errors which should be thrown and not captured 78 | * @param type of input keys 79 | * @param type of input values 80 | * @param type of output values 81 | * @return {@code ValueMapperWithKey} 82 | */ 83 | public static ValueMapperWithKey> captureErrors( 84 | final @NonNull ValueMapperWithKey mapper, 85 | final @NonNull Predicate errorFilter) { 86 | return new ErrorCapturingValueMapperWithKey<>(mapper, errorFilter); 87 | } 88 | 89 | @Override 90 | public ProcessedValue apply(final K key, final V value) { 91 | try { 92 | final VR newValue = this.wrapped.apply(key, value); 93 | return SuccessValue.of(newValue); 94 | } catch (final Exception e) { 95 | if (this.errorFilter.test(e)) { 96 | throw e; 97 | } 98 | return ErrorValue.of(value, e); 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorLoggingValueMapperWithKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static java.util.Collections.emptyList; 28 | 29 | import java.util.Collections; 30 | import java.util.function.Predicate; 31 | import lombok.AccessLevel; 32 | import lombok.NonNull; 33 | import lombok.RequiredArgsConstructor; 34 | import lombok.extern.slf4j.Slf4j; 35 | import org.apache.kafka.streams.kstream.ValueMapperWithKey; 36 | 37 | /** 38 | * Wrap a {@code ValueMapperWithKey} and log thrown exceptions with input key and value. 39 | * 40 | * @param type of input keys 41 | * @param type of input values 42 | * @param type of output values 43 | * @see #logErrors(ValueMapperWithKey) 44 | * @see #logErrors(ValueMapperWithKey, Predicate) 45 | */ 46 | @Slf4j 47 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 48 | public final class ErrorLoggingValueMapperWithKey implements ValueMapperWithKey> { 49 | private final @NonNull ValueMapperWithKey wrapped; 50 | private final @NonNull Predicate errorFilter; 51 | 52 | /** 53 | * Wrap a {@code ValueMapperWithKey} and log thrown exceptions with input key and value. Recoverable Kafka 54 | * exceptions such as a schema registry timeout are forwarded and not captured. 55 | * 56 | * @param mapper {@code ValueMapperWithKey} whose exceptions should be logged 57 | * @param type of input keys 58 | * @param type of input values 59 | * @param type of output values 60 | * @return {@code ValueMapperWithKey} 61 | * @see #logErrors(ValueMapperWithKey, Predicate) 62 | * @see ErrorUtil#isRecoverable(Exception) 63 | */ 64 | public static ValueMapperWithKey> logErrors( 65 | final @NonNull ValueMapperWithKey mapper) { 66 | return logErrors(mapper, ErrorUtil::isRecoverable); 67 | } 68 | 69 | /** 70 | * Wrap a {@code ValueMapperWithKey} and log thrown exceptions with input key and value. 71 | * 
{@code
 72 |      * final ValueMapperWithKey mapper = ...;
 73 |      * final KStream input = ...;
 74 |      * final KStream output = input.mapValues(logErrors(mapper));
 75 |      * }
 76 |      *
77 | * 78 | * @param mapper {@code ValueMapperWithKey} whose exceptions should be logged 79 | * @param errorFilter expression that filters errors which should be thrown and not logged 80 | * @param type of input keys 81 | * @param type of input values 82 | * @param type of output values 83 | * @return {@code ValueMapperWithKey} 84 | */ 85 | public static ValueMapperWithKey> logErrors( 86 | final @NonNull ValueMapperWithKey mapper, 87 | final @NonNull Predicate errorFilter) { 88 | return new ErrorLoggingValueMapperWithKey<>(mapper, errorFilter); 89 | } 90 | 91 | @Override 92 | public Iterable apply(final K key, final V value) { 93 | try { 94 | final VR newValue = this.wrapped.apply(key, value); 95 | // allow null values 96 | return Collections.singletonList(newValue); 97 | } catch (final Exception e) { 98 | if (this.errorFilter.test(e)) { 99 | throw e; 100 | } 101 | log.error("Cannot process ('{}', '{}')", ErrorUtil.toString(key), ErrorUtil.toString(value), e); 102 | return emptyList(); 103 | } 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorCapturingFlatValueMapperWithKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.List; 28 | import java.util.function.Predicate; 29 | import lombok.AccessLevel; 30 | import lombok.NonNull; 31 | import lombok.RequiredArgsConstructor; 32 | import org.apache.kafka.streams.kstream.ValueMapperWithKey; 33 | import org.jooq.lambda.Seq; 34 | 35 | /** 36 | * Wrap a {@code ValueMapperWithKey} and capture thrown exceptions. 37 | * 38 | * @param type of input keys 39 | * @param type of input values 40 | * @param type of output values 41 | * @see #captureErrors(ValueMapperWithKey) 42 | * @see #captureErrors(ValueMapperWithKey, Predicate) 43 | */ 44 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 45 | public final class ErrorCapturingFlatValueMapperWithKey 46 | implements ValueMapperWithKey>> { 47 | private final @NonNull ValueMapperWithKey> wrapped; 48 | private final @NonNull Predicate errorFilter; 49 | 50 | /** 51 | * Wrap a {@code ValueMapperWithKey} and capture thrown exceptions. Recoverable Kafka exceptions such as a schema 52 | * registry timeout are forwarded and not captured. 53 | * 54 | * @param mapper {@code ValueMapperWithKey} whose exceptions should be captured 55 | * @param type of input keys 56 | * @param type of input values 57 | * @param type of output values 58 | * @return {@code ValueMapperWithKey} 59 | * @see #captureErrors(ValueMapperWithKey, Predicate) 60 | * @see ErrorUtil#isRecoverable(Exception) 61 | */ 62 | public static ValueMapperWithKey>> captureErrors( 63 | final @NonNull ValueMapperWithKey> mapper) { 64 | return captureErrors(mapper, ErrorUtil::isRecoverable); 65 | } 66 | 67 | /** 68 | * Wrap a {@code ValueMapperWithKey} and capture thrown exceptions. 69 | * 
{@code
 70 |      * final ValueMapperWithKey> mapper = ...;
 71 |      * final KStream input = ...;
 72 |      * final KStream> processed = input.flatMapValues(captureErrors(mapper));
 73 |      * final KStream output = processed.flatMapValues(ProcessedValue::getValues);
 74 |      * final KStream> errors = processed.flatMapValues(ProcessedValue::getErrors);
 75 |      * }
 76 |      *
77 | * 78 | * @param mapper {@code ValueMapperWithKey} whose exceptions should be captured 79 | * @param errorFilter expression that filters errors which should be thrown and not captured 80 | * @param type of input keys 81 | * @param type of input values 82 | * @param type of output values 83 | * @return {@code ValueMapperWithKey} 84 | */ 85 | public static ValueMapperWithKey>> captureErrors( 86 | final @NonNull ValueMapperWithKey> mapper, 87 | final @NonNull Predicate errorFilter) { 88 | return new ErrorCapturingFlatValueMapperWithKey<>(mapper, errorFilter); 89 | } 90 | 91 | @Override 92 | public Iterable> apply(final K key, final V value) { 93 | try { 94 | final Iterable newValues = this.wrapped.apply(key, value); 95 | return Seq.seq(newValues).map(SuccessValue::of); 96 | } catch (final Exception e) { 97 | if (this.errorFilter.test(e)) { 98 | throw e; 99 | } 100 | return List.of(ErrorValue.of(value, e)); 101 | } 102 | } 103 | } 104 | 105 | -------------------------------------------------------------------------------- /error-handling-proto/src/main/java/com/bakdata/kafka/ProtoDeadLetterConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import com.bakdata.kafka.proto.v1.ProtoDeadLetter; 28 | import com.google.protobuf.Int32Value; 29 | import com.google.protobuf.Int64Value; 30 | import com.google.protobuf.StringValue; 31 | import com.google.protobuf.Timestamp; 32 | import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; 33 | 34 | 35 | /** 36 | * Convert a {@code DeadLetterDescription} to a {@code ProtoDeadLetter} message 37 | */ 38 | public class ProtoDeadLetterConverter implements DeadLetterConverter { 39 | 40 | @Override 41 | public ProtoDeadLetter convert(final DeadLetterDescription deadLetterDescription) { 42 | final ProtoDeadLetter.Builder builder = ProtoDeadLetter.newBuilder(); 43 | final DeadLetterDescription.Cause cause = deadLetterDescription.getCause(); 44 | final ProtoDeadLetter.Cause.Builder causeBuilder = builder.getCauseBuilder(); 45 | // Everything is optional with fix defaults in proto3, so use wrappers 46 | if (cause.getMessage() != null) { 47 | causeBuilder.setMessage(StringValue.of(cause.getMessage())); 48 | } 49 | if (cause.getStackTrace() != null) { 50 | causeBuilder.setStackTrace(StringValue.of(cause.getStackTrace())); 51 | } 52 | if (cause.getErrorClass() != null) { 53 | causeBuilder.setErrorClass(StringValue.of(cause.getErrorClass())); 54 | } 55 | builder.setDescription(deadLetterDescription.getDescription()); 56 | if (deadLetterDescription.getInputValue() != null) { 57 | builder.setInputValue(StringValue.of(deadLetterDescription.getInputValue())); 58 | } 59 | if (deadLetterDescription.getTopic() != null) { 60 | builder.setTopic(StringValue.of(deadLetterDescription.getTopic())); 61 | } 62 | if (deadLetterDescription.getPartition() != null) { 63 | builder.setPartition(Int32Value.of(deadLetterDescription.getPartition())); 64 | } 65 | if (deadLetterDescription.getOffset() != null) { 66 | builder.setOffset(Int64Value.of(deadLetterDescription.getOffset())); 67 | } 68 | 69 | if (deadLetterDescription.getInputTimestamp() != null) { 70 | final Timestamp timestamp = Timestamp.newBuilder() 71 | .setSeconds(deadLetterDescription.getInputTimestamp().getEpochSecond()) 72 | .setNanos(deadLetterDescription.getInputTimestamp().getNano()) 73 | .build(); 74 | builder.setInputTimestamp(timestamp); 75 | } 76 | 77 | return builder.build(); 78 | } 79 | 80 | /** 81 | * Creates a processor that uses the ProtoDeadLetterConverter 82 | * 83 | * 
{@code
 84 |      * // Example, this works for all error capturing topologies
 85 |      * final KeyValueMapper> mapper = ...;
 86 |      * final KStream input = ...;
 87 |      * final KStream> processed = input.map(captureErrors(mapper));
 88 |      * final KStream output = processed.flatMapValues(ProcessedKeyValue::getValues);
 89 |      * final KStream> errors = processed.flatMapValues(ProcessedKeyValue::getErrors);
 90 |      * final KStream deadLetters = errors.processValues(
 91 |      *                      ProtoDeadLetterConverter.asProcessor("Description"));
 92 |      * deadLetters.to(OUTPUT_TOPIC);
 93 |      * }
 94 |      *
95 | * 96 | * @param description shared description for all errors 97 | * @param type of the input key 98 | * @param type of the input value 99 | * @return a processor supplier 100 | */ 101 | public static FixedKeyProcessorSupplier, ProtoDeadLetter> asProcessor( 102 | final String description) { 103 | return DeadLetterProcessor.create(description, new ProtoDeadLetterConverter()); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorCapturingKeyValueMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.function.Predicate; 28 | import lombok.AccessLevel; 29 | import lombok.NonNull; 30 | import lombok.RequiredArgsConstructor; 31 | import org.apache.kafka.streams.KeyValue; 32 | import org.apache.kafka.streams.kstream.KeyValueMapper; 33 | 34 | /** 35 | * Wrap a {@code KeyValueMapper} and capture thrown exceptions. 36 | * 37 | * @param type of input keys 38 | * @param type of input values 39 | * @param type of output keys 40 | * @param type of output values 41 | * @see #captureErrors(KeyValueMapper) 42 | * @see #captureErrors(KeyValueMapper, Predicate) 43 | */ 44 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 45 | public final class ErrorCapturingKeyValueMapper 46 | implements KeyValueMapper>> { 47 | private final @NonNull KeyValueMapper> wrapped; 48 | private final @NonNull Predicate errorFilter; 49 | 50 | /** 51 | * Wrap a {@code KeyValueMapper} and capture thrown exceptions. Recoverable Kafka exceptions such as a schema 52 | * registry timeout are forwarded and not captured. 53 | * 54 | * @param mapper {@code KeyValueMapper} whose exceptions should be captured 55 | * @param type of input keys 56 | * @param type of input values 57 | * @param type of output keys 58 | * @param type of output values 59 | * @return {@code KeyValueMapper} 60 | * @see #captureErrors 61 | * @see ErrorUtil#isRecoverable(Exception) 62 | */ 63 | public static KeyValueMapper>> captureErrors( 64 | final @NonNull KeyValueMapper> mapper) { 65 | return captureErrors(mapper, ErrorUtil::isRecoverable); 66 | } 67 | 68 | /** 69 | * Wrap a {@code KeyValueMapper} and capture thrown exceptions. 70 | * 
{@code
 71 |      * final KeyValueMapper> mapper = ...;
 72 |      * final KStream input = ...;
 73 |      * final KStream> processed = input.map(captureErrors(mapper));
 74 |      * final KStream output = processed.flatMapValues(ProcessedKeyValue::getValues);
 75 |      * final KStream> errors = processed.flatMap(ProcessedKeyValue::getErrors);
 76 |      * }
 77 |      *
78 | * 79 | * @param mapper {@code KeyValueMapper} whose exceptions should be captured 80 | * @param errorFilter expression that filters errors which should be thrown and not captured 81 | * @param type of input keys 82 | * @param type of input values 83 | * @param type of output keys 84 | * @param type of output values 85 | * @return {@code KeyValueMapper} 86 | */ 87 | public static KeyValueMapper>> captureErrors( 88 | final @NonNull KeyValueMapper> mapper, 89 | final @NonNull Predicate errorFilter) { 90 | return new ErrorCapturingKeyValueMapper<>(mapper, errorFilter); 91 | } 92 | 93 | @Override 94 | public KeyValue> apply(final K key, final V value) { 95 | try { 96 | final KeyValue newKeyValue = this.wrapped.apply(key, value); 97 | final ProcessedKeyValue recordWithOldKey = SuccessKeyValue.of(newKeyValue.value); 98 | return KeyValue.pair(newKeyValue.key, recordWithOldKey); 99 | } catch (final Exception e) { 100 | if (this.errorFilter.test(e)) { 101 | throw e; 102 | } 103 | final ProcessedKeyValue errorWithOldKey = ErrorKeyValue.of(key, value, e); 104 | // new key is only relevant if no error occurs 105 | return KeyValue.pair(null, errorWithOldKey); 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/DeadLetterProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.time.Instant; 28 | import java.util.Optional; 29 | import lombok.Getter; 30 | import lombok.NonNull; 31 | import lombok.RequiredArgsConstructor; 32 | import org.apache.commons.lang3.exception.ExceptionUtils; 33 | import org.apache.kafka.streams.processor.api.FixedKeyProcessor; 34 | import org.apache.kafka.streams.processor.api.FixedKeyProcessorContext; 35 | import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; 36 | import org.apache.kafka.streams.processor.api.FixedKeyRecord; 37 | import org.apache.kafka.streams.processor.api.RecordMetadata; 38 | 39 | /** 40 | * {@link FixedKeyProcessor} that creates a {@code DeadLetter} from a processing error. 41 | * 42 | * @param type of key 43 | * @param type of value 44 | * @param the DeadLetter type 45 | */ 46 | @Getter 47 | @RequiredArgsConstructor 48 | public class DeadLetterProcessor implements FixedKeyProcessor, T> { 49 | private final @NonNull String description; 50 | private final @NonNull DeadLetterConverter deadLetterConverter; 51 | private FixedKeyProcessorContext context; 52 | 53 | /** 54 | * Transforms captured errors for serialization 55 | * 56 | * 
{@code
 57 |      * // Example, this works for all error capturing topologies
 58 |      * final KeyValueMapper> mapper = ...;
 59 |      * final KStream input = ...;
 60 |      * final KStream> processed = input.map(captureErrors(mapper));
 61 |      * final KStream output = processed.flatMapValues(ProcessedKeyValue::getValues);
 62 |      * final KStream> errors = processed.flatMap(ProcessedKeyValue::getErrors);
 63 |      * final DeadLetterConverter deadLetterConverter = ...
 64 |      * final KStream deadLetters = errors.processValues(
 65 |      *                      DeadLetterProcessor.create("Description", deadLetterConverter));
 66 |      * deadLetters.to(ERROR_TOPIC);
 67 |      * }
 68 |      *
69 | * 70 | * @param description shared description for all errors 71 | * @param deadLetterConverter converter from DeadLetterDescriptions to VR 72 | * @param type of the input key 73 | * @param type of the input value 74 | * @param type of the output value 75 | * @return a processor supplier 76 | */ 77 | public static FixedKeyProcessorSupplier, VR> create(final String description, 78 | final DeadLetterConverter deadLetterConverter) { 79 | return () -> new DeadLetterProcessor<>(description, deadLetterConverter); 80 | } 81 | 82 | @Override 83 | public void init(final FixedKeyProcessorContext context) { 84 | this.context = context; 85 | } 86 | 87 | @Override 88 | public void process(final FixedKeyRecord> inputRecord) { 89 | final ProcessingError error = inputRecord.value(); 90 | final Throwable throwable = error.getThrowable(); 91 | final Optional metadata = this.context.recordMetadata(); 92 | final DeadLetterDescription deadLetterDescription = DeadLetterDescription.builder() 93 | .inputValue(Optional.ofNullable(error.getValue()).map(ErrorUtil::toString).orElse(null)) 94 | .cause(DeadLetterDescription.Cause.builder() 95 | .message(throwable.getMessage()) 96 | .stackTrace(ExceptionUtils.getStackTrace(throwable)) 97 | .errorClass(throwable.getClass().getName()) 98 | .build()) 99 | .description(this.description) 100 | .topic(metadata.map(RecordMetadata::topic).orElse(null)) 101 | .partition(metadata.map(RecordMetadata::partition).orElse(null)) 102 | .offset(metadata.map(RecordMetadata::offset).orElse(null)) 103 | .inputTimestamp(Instant.ofEpochMilli(inputRecord.timestamp())) 104 | .build(); 105 | 106 | final FixedKeyRecord outputRecord = inputRecord 107 | .withValue(this.deadLetterConverter.convert(deadLetterDescription)) 108 | .withTimestamp(this.context.currentSystemTimeMs()); 109 | 110 | this.context.forward(outputRecord); 111 | } 112 | 113 | @Override 114 | public void close() { 115 | // do nothing 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorCapturingFlatKeyValueMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.List; 28 | import java.util.function.Predicate; 29 | import lombok.AccessLevel; 30 | import lombok.NonNull; 31 | import lombok.RequiredArgsConstructor; 32 | import org.apache.kafka.streams.KeyValue; 33 | import org.apache.kafka.streams.kstream.KeyValueMapper; 34 | import org.jooq.lambda.Seq; 35 | 36 | /** 37 | * Wrap a {@code KeyValueMapper} and capture thrown exceptions. 38 | * 39 | * @param type of input keys 40 | * @param type of input values 41 | * @param type of output keys 42 | * @param type of output values 43 | * @see #captureErrors(KeyValueMapper) 44 | * @see #captureErrors(KeyValueMapper, Predicate) 45 | */ 46 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 47 | public final class ErrorCapturingFlatKeyValueMapper 48 | implements KeyValueMapper>>> { 49 | private final @NonNull KeyValueMapper>> 51 | wrapped; 52 | private final @NonNull Predicate errorFilter; 53 | 54 | /** 55 | * Wrap a {@code KeyValueMapper} and capture thrown exceptions. Recoverable Kafka exceptions such as a schema 56 | * registry timeout are forwarded and not captured. 57 | * 58 | * @param mapper {@code KeyValueMapper} whose exceptions should be captured 59 | * @param type of input keys 60 | * @param type of input values 61 | * @param type of output keys 62 | * @param type of output values 63 | * @return {@code KeyValueMapper} 64 | * @see #captureErrors(KeyValueMapper, Predicate) 65 | * @see ErrorUtil#isRecoverable(Exception) 66 | */ 67 | public static KeyValueMapper>>> 68 | captureErrors( 69 | final @NonNull KeyValueMapper>> mapper) { 71 | return captureErrors(mapper, ErrorUtil::isRecoverable); 72 | } 73 | 74 | /** 75 | * Wrap a {@code KeyValueMapper} and capture thrown exceptions. 76 | * 
{@code
 77 |      * final KeyValueMapper>> mapper = ...;
 78 |      * final KStream input = ...;
 79 |      * final KStream> processed = input.flatMap(captureErrors(mapper));
 80 |      * final KStream output = processed.flatMapValues(ProcessedKeyValue::getValues);
 81 |      * final KStream> errors = processed.flatMap(ProcessedKeyValue::getErrors);
 82 |      * }
 83 |      *
84 | * 85 | * @param mapper {@code KeyValueMapper} whose exceptions should be captured 86 | * @param errorFilter expression that filters errors which should be thrown and not captured 87 | * @param type of input keys 88 | * @param type of input values 89 | * @param type of output keys 90 | * @param type of output values 91 | * @return {@code KeyValueMapper} 92 | */ 93 | public static KeyValueMapper>>> 94 | captureErrors( 95 | final @NonNull KeyValueMapper>> mapper, 97 | final @NonNull Predicate errorFilter) { 98 | return new ErrorCapturingFlatKeyValueMapper<>(mapper, errorFilter); 99 | } 100 | 101 | @Override 102 | public Iterable>> apply(final K key, final V value) { 103 | try { 104 | final Iterable> newKeyValues = 105 | this.wrapped.apply(key, value); 106 | return Seq.seq(newKeyValues) 107 | .map(kv -> KeyValue.pair(kv.key, SuccessKeyValue.of(kv.value))); 108 | } catch (final Exception e) { 109 | if (this.errorFilter.test(e)) { 110 | throw e; 111 | } 112 | final ProcessedKeyValue errorWithOldKey = ErrorKeyValue.of(key, value, e); 113 | // new key is only relevant if no error occurs 114 | return List.of(KeyValue.pair(null, errorWithOldKey)); 115 | } 116 | } 117 | 118 | } 119 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://dev.azure.com/bakdata/public/_apis/build/status/bakdata.kafka-error-handling?branchName=master)](https://dev.azure.com/bakdata/public/_build/latest?definitionId=23&branchName=master) 2 | [![Sonarcloud status](https://sonarcloud.io/api/project_badges/measure?project=com.bakdata.kafka%3Aerror-handling&metric=alert_status)](https://sonarcloud.io/dashboard?id=com.bakdata.kafka%3Aerror-handling) 3 | [![Code coverage](https://sonarcloud.io/api/project_badges/measure?project=com.bakdata.kafka%3Aerror-handling&metric=coverage)](https://sonarcloud.io/dashboard?id=com.bakdata.kafka%3Aerror-handling) 4 | [![Maven](https://img.shields.io/maven-central/v/com.bakdata.kafka/error-handling-core.svg)](https://search.maven.org/search?q=g:com.bakdata.kafka%20AND%20a:error-handling-core&core=gav) 5 | 6 | # Kafka error handling 7 | Libraries for error handling in [Kafka Streams](https://kafka.apache.org/documentation/streams/). 8 | 9 | ## Getting Started 10 | 11 | You can add Kafka error handling via Maven Central. 12 | Depending on how you want to store the dead letters in Kafka, you can use the Avro or Protobuf converter. 13 | 14 | #### Gradle 15 | ```gradle 16 | // for Avro dead letters 17 | compile group: 'com.bakdata.kafka', name: 'error-handling-avro', version: '1.3.0' 18 | // or, for Protobuf dead letters 19 | compile group: 'com.bakdata.kafka', name: 'error-handling-proto', version: '1.3.0' 20 | // or, for custom dead letters 21 | compile group: 'com.bakdata.kafka', name: 'error-handling-core', version: '1.3.0' 22 | ``` 23 | 24 | #### Maven 25 | ```xml 26 | 27 | 28 | com.bakdata.kafka 29 | error-handling-avro 30 | 1.3.0 31 | 32 | 33 | 34 | 35 | com.bakdata.kafka 36 | error-handling-proto 37 | 1.3.0 38 | 39 | 40 | 41 | 42 | com.bakdata.kafka 43 | error-handling-core 44 | 1.3.0 45 | 46 | ``` 47 | 48 | For other build tools or versions, refer to the [latest version in MvnRepository](https://mvnrepository.com/artifact/com.bakdata.kafka/error-handling/latest). 49 | 50 | ### Usage 51 | 52 | If you use Kafka Streams to process your data, you will sooner or later get to the point where processing of a message throws an exception. 53 | In case your streams application is configured to process every message at least once, which is the case most times, 54 | your application will crash upon encountering an error and retry processing the erroneous message. 55 | If the error was just temporary, processing will continue as if nothing has happened. 56 | However, if the error is related to the specific message, then your streams application will be stuck processing the record. 57 | For such cases we developed three solutions that help handling errors in your Kafka Streams application. 58 | 59 | Consider the following Topology: 60 | 61 | ```java 62 | final KeyValueMapper> mapper = … 63 | final KStream input = 64 | builder.stream(INPUT_TOPIC, Consumed.with(Serdes.Integer(), Serdes.String())); 65 | 66 | final KStream mapped = input.map(mapper); 67 | mapped.to(OUTPUT_TOPIC, Produced.with(Serdes.Double(), Serdes.Long())); 68 | ``` 69 | 70 | You can simply add a dead letter queue to your topology with our libraries: 71 | 72 | ```java 73 | final KeyValueMapper> mapper = … 74 | final KStream input = 75 | builder.stream(INPUT_TOPIC, Consumed.with(Serdes.Integer(), Serdes.String())); 76 | 77 | final KStream> mappedWithErrors = 78 | input.map(captureErrors(mapper)); 79 | mappedWithErrors.flatMap(ProcessedKeyValue::getErrors) 80 | .processValues(AvroDeadLetterConverter.asProcessor("A good description where the pipeline broke")) 81 | .to(ERROR_TOPIC); 82 | 83 | final KStream mapped = mappedWithErrors.flatMapValues(ProcessedKeyValue::getValues); 84 | mapped.to(OUTPUT_TOPIC, Produced.with(Serdes.Double(), Serdes.Long())); 85 | ``` 86 | 87 | Successfully processed messages are sent to the output topic as before. 88 | However, errors are sent to a specific error topic. 89 | This error topic contains dead letters describing the input value, error message and stack trace of any error that is raised in that part of your topology. 90 | 91 | The example uses the `AvroDeadLetterConverter` from `error-handling-avro`. 92 | Analogously, `error-handling-proto` implements a `ProtoDeadLetterConverter`. 93 | A custom `DeadLetterConverter` can be passed to `DeadLetterProcessor.create`. 94 | 95 | ## Development 96 | 97 | If you want to contribute to this project, you can simply clone the repository and build it via Gradle. 98 | All dependencies should be included in the Gradle files, there are no external prerequisites. 99 | 100 | ```bash 101 | > git clone git@github.com:bakdata/kafka-error-handling.git 102 | > cd kafka-error-handling && ./gradlew build 103 | ``` 104 | 105 | Please note, that we have [code styles](https://github.com/bakdata/bakdata-code-styles) for Java. 106 | They are basically the Google style guide, with some small modifications. 107 | 108 | ## Contributing 109 | 110 | We are happy if you want to contribute to this project. 111 | If you find any bugs or have suggestions for improvements, please open an issue. 112 | We are also happy to accept your PRs. 113 | Just open an issue beforehand and let us know what you want to do and why. 114 | 115 | ## License 116 | This project is licensed under the MIT license. 117 | Have a look at the [LICENSE](https://github.com/bakdata/kafka-error-handling/blob/master/LICENSE) for more details. 118 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorLoggingFlatValueTransformer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static java.util.Collections.emptyList; 28 | 29 | import java.util.Set; 30 | import java.util.function.Predicate; 31 | import lombok.AccessLevel; 32 | import lombok.NonNull; 33 | import lombok.RequiredArgsConstructor; 34 | import lombok.extern.slf4j.Slf4j; 35 | import org.apache.kafka.streams.kstream.ValueTransformer; 36 | import org.apache.kafka.streams.kstream.ValueTransformerSupplier; 37 | import org.apache.kafka.streams.processor.ProcessorContext; 38 | import org.apache.kafka.streams.state.StoreBuilder; 39 | 40 | /** 41 | * Wrap a {@code ValueTransformer} and log thrown exceptions with input key and value. 42 | * 43 | * @param type of input values 44 | * @param type of output values 45 | * @see #logErrors(ValueTransformer) 46 | * @see #logErrors(ValueTransformer, Predicate) 47 | */ 48 | @Slf4j 49 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 50 | public final class ErrorLoggingFlatValueTransformer implements ValueTransformer> { 51 | private final @NonNull ValueTransformer> wrapped; 52 | private final @NonNull Predicate errorFilter; 53 | 54 | /** 55 | * Wrap a {@code ValueTransformer} and log thrown exceptions with input key and value. Recoverable Kafka exceptions 56 | * such as a schema registry timeout are forwarded and not captured. 57 | * 58 | * @param transformer {@code ValueTransformer} whose exceptions should be logged 59 | * @param type of input values 60 | * @param type of output values 61 | * @return {@code ValueTransformer} 62 | * @see #logErrors(ValueTransformer, Predicate) 63 | * @see ErrorUtil#isRecoverable(Exception) 64 | */ 65 | public static ValueTransformer> logErrors( 66 | final @NonNull ValueTransformer> transformer) { 67 | return logErrors(transformer, ErrorUtil::isRecoverable); 68 | } 69 | 70 | /** 71 | * Wrap a {@code ValueTransformer} and log thrown exceptions with input key and value. 72 | * 
{@code
 73 |      * final KStream input = ...;
 74 |      * final KStream output = input.transformValues(() -> logErrors(new ValueTransformer>() {...}));
 75 |      * }
 76 |      *
77 | * 78 | * @param transformer {@code ValueTransformer} whose exceptions should be logged 79 | * @param errorFilter expression that filters errors which should be thrown and not logged 80 | * @param type of input values 81 | * @param type of output values 82 | * @return {@code ValueTransformer} 83 | */ 84 | public static ValueTransformer> logErrors( 85 | final @NonNull ValueTransformer> transformer, 86 | final @NonNull Predicate errorFilter) { 87 | return new ErrorLoggingFlatValueTransformer<>(transformer, errorFilter); 88 | } 89 | 90 | /** 91 | * Wrap a {@code ValueTransformerSupplier} and log thrown exceptions with input key and value. Recoverable Kafka 92 | * exceptions such as a schema registry timeout are forwarded and not captured. 93 | * 94 | * @param supplier {@code ValueTransformerSupplier} whose exceptions should be logged 95 | * @param type of input values 96 | * @param type of output values 97 | * @return {@code ValueTransformerSupplier} 98 | * @see #logErrors(ValueTransformerSupplier, Predicate) 99 | * @see ErrorUtil#isRecoverable(Exception) 100 | */ 101 | public static ValueTransformerSupplier> logErrors( 102 | final @NonNull ValueTransformerSupplier> supplier) { 103 | return logErrors(supplier, ErrorUtil::isRecoverable); 104 | } 105 | 106 | /** 107 | * Wrap a {@code ValueTransformerSupplier} and log thrown exceptions with input key and value. 108 | * 
{@code
109 |      * final ValueTransformerSupplier> transformer = ...;
110 |      * final KStream input = ...;
111 |      * final KStream output = input.transformValues(logErrors(transformer));
112 |      * }
113 |      *
114 | * 115 | * @param supplier {@code ValueTransformerSupplier} whose exceptions should be logged 116 | * @param errorFilter expression that filters errors which should be thrown and not logged 117 | * @param type of input values 118 | * @param type of output values 119 | * @return {@code ValueTransformerSupplier} 120 | */ 121 | public static ValueTransformerSupplier> logErrors( 122 | final @NonNull ValueTransformerSupplier> supplier, 123 | final @NonNull Predicate errorFilter) { 124 | return new ValueTransformerSupplier<>() { 125 | @Override 126 | public Set> stores() { 127 | return supplier.stores(); 128 | } 129 | 130 | @Override 131 | public ValueTransformer> get() { 132 | return logErrors(supplier.get(), errorFilter); 133 | } 134 | }; 135 | } 136 | 137 | @Override 138 | public void close() { 139 | this.wrapped.close(); 140 | } 141 | 142 | @Override 143 | public void init(final ProcessorContext context) { 144 | this.wrapped.init(context); 145 | } 146 | 147 | @Override 148 | public Iterable transform(final V value) { 149 | try { 150 | return this.wrapped.transform(value); 151 | } catch (final Exception e) { 152 | if (this.errorFilter.test(e)) { 153 | throw e; 154 | } 155 | log.error("Cannot process {}", ErrorUtil.toString(value), e); 156 | return emptyList(); 157 | } 158 | } 159 | 160 | } 161 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorHeaderProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.nio.charset.StandardCharsets; 28 | import java.util.Optional; 29 | import lombok.NonNull; 30 | import lombok.RequiredArgsConstructor; 31 | import org.apache.commons.lang3.exception.ExceptionUtils; 32 | import org.apache.kafka.common.header.Headers; 33 | import org.apache.kafka.streams.processor.api.FixedKeyProcessor; 34 | import org.apache.kafka.streams.processor.api.FixedKeyProcessorContext; 35 | import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; 36 | import org.apache.kafka.streams.processor.api.FixedKeyRecord; 37 | import org.apache.kafka.streams.processor.api.RecordMetadata; 38 | 39 | /** 40 | * {@link FixedKeyProcessor} that produces a message with the original value and headers detailing the error that has 41 | * been captured. 42 | * 43 | * Headers added by this FixedKeyProcessor: 44 | *
    45 | *
  • {@code __streams.errors.topic}: original input topic of the erroneous record
    • 46 | *
  • {@code __streams.errors.partition}: original partition of the input topic of the erroneous record
    • 47 | *
  • {@code __streams.errors.offset}: original offset in the partition of the input topic of the erroneous 48 | * record
    • 49 | *
  • {@code __streams.errors.description}: description of the context in which an exception has been thrown
    • 50 | *
  • {@code __streams.errors.exception.class.name}: class of the exception that was captured
    • 51 | *
  • {@code __streams.errors.exception.message}: message of the exception that was captured
    • 52 | *
  • {@code __streams.errors.exception.stack_trace}: stack trace of the exception that was captured
    • 53 | *
54 | * 55 | * @param type of key 56 | * @param type of value 57 | */ 58 | @RequiredArgsConstructor 59 | public class ErrorHeaderProcessor implements FixedKeyProcessor, V> { 60 | /** 61 | * Prefix of all headers added by this FixedKeyProcessor 62 | */ 63 | public static final String HEADER_PREFIX = "__streams.errors."; 64 | /** 65 | * Header indicating the original input topic of the erroneous record 66 | */ 67 | public static final String TOPIC = HEADER_PREFIX + "topic"; 68 | /** 69 | * Header indicating the original partition in the input topic of the erroneous record 70 | */ 71 | public static final String PARTITION = HEADER_PREFIX + "partition"; 72 | /** 73 | * Header indicating the original offset in the partition of the input topic of the erroneous record 74 | */ 75 | public static final String OFFSET = HEADER_PREFIX + "offset"; 76 | /** 77 | * Header indicating the description of the context in which an exception has been thrown 78 | */ 79 | public static final String DESCRIPTION = HEADER_PREFIX + "description"; 80 | /** 81 | * Prefix of all headers detailing the error message added by this FixedKeyProcessor 82 | */ 83 | public static final String EXCEPTION_PREFIX = HEADER_PREFIX + "exception."; 84 | /** 85 | * Header indicating the class of the exception that was captured 86 | */ 87 | public static final String EXCEPTION_CLASS_NAME = EXCEPTION_PREFIX + "class.name"; 88 | /** 89 | * Header indicating the message of the exception that was captured 90 | */ 91 | public static final String EXCEPTION_MESSAGE = EXCEPTION_PREFIX + "message"; 92 | /** 93 | * Header indicating the stack trace of the exception that was captured 94 | */ 95 | public static final String EXCEPTION_STACK_TRACE = EXCEPTION_PREFIX + "stack_trace"; 96 | private final @NonNull String description; 97 | private FixedKeyProcessorContext context; 98 | 99 | /** 100 | * Create a new {@code ErrorHeaderProcessor} with the provided description 101 | * 102 | * @param description description of the context in which an exception has been thrown 103 | * @param type of key 104 | * @param type of value 105 | * @return {@code FixedKeyProcessorSupplier} that produces a message with the original value and headers detailing 106 | * the error that has been captured. 107 | */ 108 | public static FixedKeyProcessorSupplier, V> withErrorHeaders( 109 | final String description) { 110 | return () -> new ErrorHeaderProcessor<>(description); 111 | } 112 | 113 | static void addHeader(final String key, final String value, final Headers headers) { 114 | headers.remove(key); 115 | headers.add(key, value == null ? null : value.getBytes(StandardCharsets.UTF_8)); 116 | } 117 | 118 | @Override 119 | public void init(final FixedKeyProcessorContext context) { 120 | this.context = context; 121 | } 122 | 123 | @Override 124 | public void process(final FixedKeyRecord> inputRecord) { 125 | final Headers headers = inputRecord.headers(); 126 | final Optional metadata = this.context.recordMetadata(); 127 | addHeader(TOPIC, metadata.map(RecordMetadata::topic).orElse(null), headers); 128 | addHeader(PARTITION, metadata.map(RecordMetadata::partition) 129 | .map(p -> Integer.toString(p)) 130 | .orElse(null), headers); 131 | addHeader(OFFSET, metadata.map(RecordMetadata::offset) 132 | .map(p -> Long.toString(p)) 133 | .orElse(null), headers); 134 | final ProcessingError value = inputRecord.value(); 135 | addHeader(EXCEPTION_CLASS_NAME, value.getThrowable().getClass().getName(), headers); 136 | addHeader(EXCEPTION_MESSAGE, value.getThrowable().getMessage(), headers); 137 | addHeader(EXCEPTION_STACK_TRACE, ExceptionUtils.getStackTrace(value.getThrowable()), headers); 138 | addHeader(DESCRIPTION, this.description, headers); 139 | this.context.forward(inputRecord.withValue(value.getValue())); 140 | } 141 | 142 | @Override 143 | public void close() { 144 | // do nothing 145 | } 146 | 147 | } 148 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorLoggingProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.Set; 28 | import java.util.function.Predicate; 29 | import lombok.AccessLevel; 30 | import lombok.NonNull; 31 | import lombok.RequiredArgsConstructor; 32 | import lombok.extern.slf4j.Slf4j; 33 | import org.apache.kafka.streams.processor.api.Processor; 34 | import org.apache.kafka.streams.processor.api.ProcessorContext; 35 | import org.apache.kafka.streams.processor.api.ProcessorSupplier; 36 | import org.apache.kafka.streams.processor.api.Record; 37 | import org.apache.kafka.streams.state.StoreBuilder; 38 | 39 | /** 40 | * Wrap a {@code Processor} and log thrown exceptions with input key and value. 41 | * 42 | * @param type of input keys 43 | * @param type of input values 44 | * @param type of output keys 45 | * @param type of output values 46 | * @see #logErrors(Processor) 47 | * @see #logErrors(Processor, Predicate) 48 | */ 49 | @Slf4j 50 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 51 | public final class ErrorLoggingProcessor implements Processor { 52 | private final @NonNull Processor wrapped; 53 | private final @NonNull Predicate errorFilter; 54 | 55 | /** 56 | * Wrap a {@code Processor} and log thrown exceptions with input key and value. Recoverable Kafka exceptions such as 57 | * a schema registry timeout are forwarded and not captured. 58 | * 59 | * @param processor {@code Processor} whose exceptions should be logged 60 | * @param type of input keys 61 | * @param type of input values 62 | * @param type of output keys 63 | * @param type of output values 64 | * @return {@code Processor} 65 | * @see #logErrors(Processor, Predicate) 66 | * @see ErrorUtil#isRecoverable(Exception) 67 | */ 68 | public static Processor logErrors( 69 | final @NonNull Processor processor) { 70 | return logErrors(processor, ErrorUtil::isRecoverable); 71 | } 72 | 73 | /** 74 | * Wrap a {@code Processor} and log thrown exceptions with input key and value. 75 | * 
{@code
 76 |      * final KStream input = ...;
 77 |      * final KStream output = input.process(() -> logErrors(new Processor() {...}));
 78 |      * }
 79 |      *
80 | * 81 | * @param processor {@code Processor} whose exceptions should be logged 82 | * @param errorFilter expression that filters errors which should be thrown and not logged 83 | * @param type of input keys 84 | * @param type of input values 85 | * @param type of output keys 86 | * @param type of output values 87 | * @return {@code Processor} 88 | */ 89 | public static Processor logErrors( 90 | final @NonNull Processor processor, 91 | final @NonNull Predicate errorFilter) { 92 | return new ErrorLoggingProcessor<>(processor, errorFilter); 93 | } 94 | 95 | /** 96 | * Wrap a {@code ProcessorSupplier} and log thrown exceptions with input key and value. Recoverable Kafka exceptions 97 | * such as a schema registry timeout are forwarded and not captured. 98 | * 99 | * @param supplier {@code ProcessorSupplier} whose exceptions should be logged 100 | * @param type of input keys 101 | * @param type of input values 102 | * @param type of output keys 103 | * @param type of output values 104 | * @return {@code ProcessorSupplier} 105 | * @see #logErrors(ProcessorSupplier, Predicate) 106 | * @see ErrorUtil#isRecoverable(Exception) 107 | */ 108 | public static ProcessorSupplier logErrors( 109 | final @NonNull ProcessorSupplier supplier) { 110 | return logErrors(supplier, ErrorUtil::isRecoverable); 111 | } 112 | 113 | /** 114 | * Wrap a {@code ProcessorSupplier} and log thrown exceptions with input key and value. 115 | * 
{@code
116 |      * final ProcessorSupplier processor = ...;
117 |      * final KStream input = ...;
118 |      * final KStream output = input.process(logErrors(processor));
119 |      * }
120 |      *
121 | * 122 | * @param supplier {@code ProcessorSupplier} whose exceptions should be logged 123 | * @param errorFilter expression that filters errors which should be thrown and not logged 124 | * @param type of input keys 125 | * @param type of input values 126 | * @param type of output keys 127 | * @param type of output values 128 | * @return {@code ProcessorSupplier} 129 | */ 130 | public static ProcessorSupplier logErrors( 131 | final @NonNull ProcessorSupplier supplier, 132 | final @NonNull Predicate errorFilter) { 133 | return new ProcessorSupplier<>() { 134 | @Override 135 | public Set> stores() { 136 | return supplier.stores(); 137 | } 138 | 139 | @Override 140 | public Processor get() { 141 | return logErrors(supplier.get(), errorFilter); 142 | } 143 | }; 144 | } 145 | 146 | @Override 147 | public void close() { 148 | this.wrapped.close(); 149 | } 150 | 151 | @Override 152 | public void init(final ProcessorContext context) { 153 | this.wrapped.init(context); 154 | } 155 | 156 | @Override 157 | public void process(final Record inputRecord) { 158 | try { 159 | this.wrapped.process(inputRecord); 160 | } catch (final Exception e) { 161 | if (this.errorFilter.test(e)) { 162 | throw e; 163 | } 164 | log.error("Cannot process ('{}', '{}')", ErrorUtil.toString(inputRecord.key()), 165 | ErrorUtil.toString(inputRecord.value()), e); 166 | } 167 | } 168 | 169 | } 170 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorLoggingValueProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2022 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.Set; 28 | import java.util.function.Predicate; 29 | import lombok.AccessLevel; 30 | import lombok.NonNull; 31 | import lombok.RequiredArgsConstructor; 32 | import lombok.extern.slf4j.Slf4j; 33 | import org.apache.kafka.streams.processor.api.FixedKeyProcessor; 34 | import org.apache.kafka.streams.processor.api.FixedKeyProcessorContext; 35 | import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; 36 | import org.apache.kafka.streams.processor.api.FixedKeyRecord; 37 | import org.apache.kafka.streams.state.StoreBuilder; 38 | 39 | /** 40 | * Wrap a {@code FixedKeyProcessor} and log thrown exceptions with input key and value. 41 | * 42 | * @param type of input keys 43 | * @param type of input values 44 | * @param type of output values 45 | * @see #logErrors(FixedKeyProcessor) 46 | * @see #logErrors(FixedKeyProcessor, Predicate) 47 | */ 48 | @Slf4j 49 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 50 | public final class ErrorLoggingValueProcessor implements FixedKeyProcessor { 51 | private final @NonNull FixedKeyProcessor wrapped; 52 | private final @NonNull Predicate errorFilter; 53 | 54 | /** 55 | * Wrap a {@code FixedKeyProcessor} and log thrown exceptions with input key and value. Recoverable Kafka exceptions 56 | * such as a schema registry timeout are forwarded and not captured. 57 | * 58 | * @param processor {@code FixedKeyProcessor} whose exceptions should be logged 59 | * @param type of input keys 60 | * @param type of input values 61 | * @param type of output values 62 | * @return {@code FixedKeyProcessor} 63 | * @see #logErrors(FixedKeyProcessor, Predicate) 64 | * @see ErrorUtil#isRecoverable(Exception) 65 | */ 66 | public static FixedKeyProcessor logErrors( 67 | final @NonNull FixedKeyProcessor processor) { 68 | return logErrors(processor, ErrorUtil::isRecoverable); 69 | } 70 | 71 | /** 72 | * Wrap a {@code FixedKeyProcessor} and log thrown exceptions with input key and value. 73 | * 
{@code
 74 |      * final KStream input = ...;
 75 |      * final KStream output = input.processValues(() -> logErrors(new FixedKeyProcessor() {...}));
 76 |      * }
 77 |      *
78 | * 79 | * @param processor {@code FixedKeyProcessor} whose exceptions should be logged 80 | * @param errorFilter expression that filters errors which should be thrown and not logged 81 | * @param type of input keys 82 | * @param type of input values 83 | * @param type of output values 84 | * @return {@code FixedKeyProcessor} 85 | */ 86 | public static FixedKeyProcessor logErrors( 87 | final @NonNull FixedKeyProcessor processor, 88 | final @NonNull Predicate errorFilter) { 89 | return new ErrorLoggingValueProcessor<>(processor, errorFilter); 90 | } 91 | 92 | /** 93 | * Wrap a {@code FixedKeyProcessorSupplier} and log thrown exceptions with input key and value. Recoverable Kafka 94 | * exceptions such as a schema registry timeout are forwarded and not captured. 95 | * 96 | * @param supplier {@code FixedKeyProcessorSupplier} whose exceptions should be logged 97 | * @param type of input keys 98 | * @param type of input values 99 | * @param type of output values 100 | * @return {@code FixedKeyProcessorSupplier} 101 | * @see #logErrors(FixedKeyProcessorSupplier, Predicate) 102 | * @see ErrorUtil#isRecoverable(Exception) 103 | */ 104 | public static FixedKeyProcessorSupplier logErrors( 105 | final @NonNull FixedKeyProcessorSupplier supplier) { 106 | return logErrors(supplier, ErrorUtil::isRecoverable); 107 | } 108 | 109 | /** 110 | * Wrap a {@code FixedKeyProcessorSupplier} and log thrown exceptions with input key and value. 111 | * 
{@code
112 |      * final FixedKeyProcessorSupplier processor = ...;
113 |      * final KStream input = ...;
114 |      * final KStream output = input.processValues(logErrors(processor));
115 |      * }
116 |      *
117 | * 118 | * @param supplier {@code FixedKeyProcessorSupplier} whose exceptions should be logged 119 | * @param errorFilter expression that filters errors which should be thrown and not logged 120 | * @param type of input keys 121 | * @param type of input values 122 | * @param type of output values 123 | * @return {@code FixedKeyProcessorSupplier} 124 | */ 125 | public static FixedKeyProcessorSupplier logErrors( 126 | final @NonNull FixedKeyProcessorSupplier supplier, 127 | final @NonNull Predicate errorFilter) { 128 | return new FixedKeyProcessorSupplier<>() { 129 | @Override 130 | public Set> stores() { 131 | return supplier.stores(); 132 | } 133 | 134 | @Override 135 | public FixedKeyProcessor get() { 136 | return logErrors(supplier.get(), errorFilter); 137 | } 138 | }; 139 | } 140 | 141 | @Override 142 | public void close() { 143 | this.wrapped.close(); 144 | } 145 | 146 | @Override 147 | public void init(final FixedKeyProcessorContext context) { 148 | this.wrapped.init(context); 149 | } 150 | 151 | @Override 152 | public void process(final FixedKeyRecord inputRecord) { 153 | try { 154 | this.wrapped.process(inputRecord); 155 | } catch (final Exception e) { 156 | if (this.errorFilter.test(e)) { 157 | throw e; 158 | } 159 | log.error("Cannot process ('{}', '{}')", ErrorUtil.toString(inputRecord.key()), 160 | ErrorUtil.toString(inputRecord.value()), e); 161 | } 162 | } 163 | 164 | } 165 | -------------------------------------------------------------------------------- /error-handling-core/src/main/java/com/bakdata/kafka/ErrorUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.io.ByteArrayOutputStream; 28 | import java.io.IOException; 29 | import java.io.OutputStream; 30 | import java.nio.charset.StandardCharsets; 31 | import java.util.Objects; 32 | import java.util.Set; 33 | import lombok.experimental.UtilityClass; 34 | import lombok.extern.slf4j.Slf4j; 35 | import org.apache.avro.Schema; 36 | import org.apache.avro.generic.GenericContainer; 37 | import org.apache.avro.generic.GenericDatumWriter; 38 | import org.apache.avro.generic.GenericRecord; 39 | import org.apache.avro.io.DatumWriter; 40 | import org.apache.avro.io.EncoderFactory; 41 | import org.apache.avro.io.JsonEncoder; 42 | import org.apache.avro.specific.SpecificDatumWriter; 43 | import org.apache.avro.specific.SpecificRecord; 44 | import org.apache.kafka.common.errors.RecordTooLargeException; 45 | 46 | /** 47 | * This class provides utility methods for dealing with errors in Kafka streams, such as serializing values to string 48 | * and classifying errors as recoverable. 49 | */ 50 | @Slf4j 51 | @UtilityClass 52 | public class ErrorUtil { 53 | 54 | private static final String ORG_APACHE_KAFKA_COMMON_ERRORS = "org.apache.kafka.common.errors"; 55 | private static final String ORG_APACHE_KAFKA_STREAMS_ERRORS = "org.apache.kafka.streams.errors"; 56 | private static final Set RECOVERABLE_ERROR_PACKAGES = Set.of( 57 | ORG_APACHE_KAFKA_COMMON_ERRORS, 58 | ORG_APACHE_KAFKA_STREAMS_ERRORS 59 | ); 60 | 61 | /** 62 | * Check if an exception is recoverable and thus should be thrown so that the process is restarted by the execution 63 | * environment. 64 | *

Recoverable errors are: 65 | *

    66 | *
  • {@link #isRecoverableKafkaError(Exception)} 67 | *
68 | * 69 | * @param e exception 70 | * @return whether exception is recoverable or not 71 | */ 72 | public static boolean isRecoverable(final Exception e) { 73 | return isRecoverableKafkaError(e); 74 | } 75 | 76 | /** 77 | * Check if an exception is thrown by Kafka, i.e., located in package {@code org.apache.kafka.common.errors} or 78 | * {@code org.apache.kafka.streams.errors}, and is recoverable. 79 | *

Non-recoverable Kafka errors are: 80 | *

    81 | *
  • {@link RecordTooLargeException} 82 | *
83 | * 84 | * @param e exception 85 | * @return whether exception is thrown by Kafka and recoverable or not 86 | */ 87 | public static boolean isRecoverableKafkaError(final Exception e) { 88 | if (RECOVERABLE_ERROR_PACKAGES.contains(e.getClass().getPackageName())) { 89 | return !(e instanceof RecordTooLargeException); 90 | } 91 | return false; 92 | } 93 | 94 | /** 95 | * Convert an object to {@code String}. {@code SpecificRecord} will be serialized using 96 | * {@link #toString(SpecificRecord)}. {@code GenericRecord} will be serialized using 97 | * {@link #toString(GenericRecord)}. 98 | * 99 | * @param o object to be serialized 100 | * @return {@code String} representation of record 101 | */ 102 | public static String toString(final Object o) { 103 | final Object o1; 104 | if (o instanceof SpecificRecord) { 105 | o1 = toString((SpecificRecord) o); 106 | } else if (o instanceof GenericRecord) { 107 | o1 = toString((GenericRecord) o); 108 | } else { 109 | o1 = o; 110 | } 111 | return Objects.toString(o1); 112 | } 113 | 114 | /** 115 | * Convert a {@code SpecificRecord} to {@code String} using JSON serialization. 116 | * 117 | * @param specificRecord record to be serialized 118 | * @return JSON representation of record or record if an error occurred 119 | */ 120 | private static Object toString(final SpecificRecord specificRecord) { 121 | try { 122 | return writeAsJson(specificRecord); 123 | } catch (final IOException ex) { 124 | log.warn("Failed to write to json", ex); 125 | return specificRecord; 126 | } 127 | } 128 | 129 | private static String writeAsJson(final SpecificRecord itemRecord) throws IOException { 130 | final Schema targetSchema = itemRecord.getSchema(); 131 | final DatumWriter writer = new SpecificDatumWriter<>(targetSchema); 132 | return writeJson(itemRecord, writer); 133 | } 134 | 135 | /** 136 | * Convert a {@code GenericRecord} to {@code String} using JSON serialization. 137 | * 138 | * @param genericRecord record to be serialized 139 | * @return JSON representation of record or record if an error occurred 140 | */ 141 | private static Object toString(final GenericRecord genericRecord) { 142 | try { 143 | return writeAsJson(genericRecord); 144 | } catch (final IOException ex) { 145 | log.warn("Failed to write to json", ex); 146 | return genericRecord; 147 | } 148 | } 149 | 150 | private static String writeAsJson(final GenericRecord itemRecord) throws IOException { 151 | final Schema targetSchema = itemRecord.getSchema(); 152 | final DatumWriter writer = new GenericDatumWriter<>(targetSchema); 153 | return writeJson(itemRecord, writer); 154 | } 155 | 156 | private static String writeJson(final T itemRecord, final DatumWriter writer) 157 | throws IOException { 158 | try (final ByteArrayOutputStream out = new ByteArrayOutputStream()) { 159 | writeJson(itemRecord, writer, out); 160 | return out.toString(StandardCharsets.ISO_8859_1); 161 | } 162 | } 163 | 164 | private static void writeJson(final T itemRecord, final DatumWriter writer, 165 | final OutputStream out) throws IOException { 166 | final JsonEncoder jsonEncoder = EncoderFactory.get().jsonEncoder(itemRecord.getSchema(), out); 167 | writer.write(itemRecord, jsonEncoder); 168 | jsonEncoder.flush(); 169 | } 170 | } 171 | --------------------------------------------------------------------------------