├── .github ├── dependabot.yaml ├── lint-config.yaml └── workflows │ ├── build-and-publish.yaml │ ├── helm-lint.yaml │ ├── helm-publish.yaml │ └── release.yaml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── build.gradle.kts ├── charts ├── README.md ├── producer-app-cleanup-job │ ├── .helmignore │ ├── Chart.yaml │ ├── README.md │ ├── templates │ │ ├── _helpers.tpl │ │ ├── configmap.yaml │ │ ├── job.yaml │ │ └── secrets.yaml │ └── values.yaml ├── producer-app │ ├── .helmignore │ ├── Chart.yaml │ ├── README.md │ ├── templates │ │ ├── _helpers.tpl │ │ ├── _pod.yaml │ │ ├── configmap.yaml │ │ ├── deployment.yaml │ │ ├── job.yaml │ │ ├── secrets.yaml │ │ └── service.yaml │ └── values.yaml ├── streams-app-cleanup-job │ ├── .helmignore │ ├── Chart.yaml │ ├── README.md │ ├── templates │ │ ├── _helpers.tpl │ │ ├── configmap.yaml │ │ ├── job.yaml │ │ └── secrets.yaml │ └── values.yaml └── streams-app │ ├── .helmignore │ ├── Chart.yaml │ ├── README.md │ ├── templates │ ├── _helpers.tpl │ ├── configmap.yaml │ ├── deployment.yaml │ ├── jmx-configmap.yaml │ ├── scaled-object.yaml │ ├── secrets.yaml │ └── service.yaml │ └── values.yaml ├── gradle.properties ├── gradle ├── libs.versions.toml └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── monitoring └── pod_monitor.yaml ├── settings.gradle.kts ├── streams-bootstrap-bom └── build.gradle.kts ├── streams-bootstrap-cli ├── build.gradle.kts └── src │ ├── main │ └── java │ │ └── com │ │ └── bakdata │ │ └── kafka │ │ ├── EnvironmentArgumentsParser.java │ │ ├── KafkaApplication.java │ │ ├── KafkaProducerApplication.java │ │ ├── KafkaStreamsApplication.java │ │ ├── SimpleKafkaProducerApplication.java │ │ ├── SimpleKafkaStreamsApplication.java │ │ └── StringListConverter.java │ └── test │ ├── avro │ └── TestRecord.avsc │ ├── java │ └── com │ │ └── bakdata │ │ └── kafka │ │ ├── CliTest.java │ │ ├── CloseFlagApp.java │ │ ├── EnvironmentArgumentsParserTest.java │ │ ├── StringListConverterTest.java │ │ ├── integration │ │ ├── RunProducerAppTest.java │ │ ├── RunStreamsAppTest.java │ │ └── StreamsCleanUpTest.java │ │ └── test_applications │ │ ├── Mirror.java │ │ └── WordCount.java │ └── resources │ └── log4j2.xml ├── streams-bootstrap-core ├── build.gradle.kts ├── lombok.config └── src │ ├── main │ └── java │ │ └── com │ │ └── bakdata │ │ └── kafka │ │ ├── App.java │ │ ├── AppConfiguration.java │ │ ├── BranchedKStreamX.java │ │ ├── BranchedKStreamXImpl.java │ │ ├── BranchedX.java │ │ ├── CapturingStreamsUncaughtExceptionHandler.java │ │ ├── CleanUpException.java │ │ ├── CleanUpRunner.java │ │ ├── CogroupedKStreamX.java │ │ ├── CogroupedStreamXImpl.java │ │ ├── ConfiguredApp.java │ │ ├── ConfiguredProducerApp.java │ │ ├── ConfiguredStreamsApp.java │ │ ├── ConsumedX.java │ │ ├── DefaultStreamsUncaughtExceptionHandler.java │ │ ├── EffectiveAppConfiguration.java │ │ ├── EnvironmentKafkaConfigParser.java │ │ ├── ExecutableApp.java │ │ ├── ExecutableProducerApp.java │ │ ├── ExecutableStreamsApp.java │ │ ├── GroupedX.java │ │ ├── HasCleanHook.java │ │ ├── HasTopicHooks.java │ │ ├── ImprovedStreamsConfig.java │ │ ├── JoinedX.java │ │ ├── KErrorStreamX.java │ │ ├── KGroupedStreamX.java │ │ ├── KGroupedStreamXImpl.java │ │ ├── KGroupedTableX.java │ │ ├── KGroupedTableXImpl.java │ │ ├── KStreamX.java │ │ ├── KStreamXImpl.java │ │ ├── KTableX.java │ │ ├── KTableXImpl.java │ │ ├── KafkaEndpointConfig.java │ │ ├── KafkaPropertiesFactory.java │ │ ├── KeyValueKErrorStreamX.java │ │ ├── MaterializedX.java │ │ ├── ModifierChain.java │ │ ├── NoOpStateListener.java │ │ ├── ProducedX.java │ │ ├── ProducerApp.java │ │ ├── ProducerBuilder.java │ │ ├── ProducerCleanUpConfiguration.java │ │ ├── ProducerCleanUpRunner.java │ │ ├── ProducerExecutionOptions.java │ │ ├── ProducerRunnable.java │ │ ├── ProducerRunner.java │ │ ├── ProducerTopicConfig.java │ │ ├── RepartitionedX.java │ │ ├── Runner.java │ │ ├── RunningStreams.java │ │ ├── SerdeConfig.java │ │ ├── SerializationConfig.java │ │ ├── SerializerConfig.java │ │ ├── SessionWindowedCogroupedKStreamX.java │ │ ├── SessionWindowedCogroupedStreamXImpl.java │ │ ├── SessionWindowedKStreamX.java │ │ ├── SessionWindowedStreamXImpl.java │ │ ├── StoresX.java │ │ ├── StreamJoinedX.java │ │ ├── StreamsApp.java │ │ ├── StreamsApplicationException.java │ │ ├── StreamsBuilderX.java │ │ ├── StreamsCleanUpConfiguration.java │ │ ├── StreamsCleanUpRunner.java │ │ ├── StreamsContext.java │ │ ├── StreamsExecutionOptions.java │ │ ├── StreamsRunner.java │ │ ├── StreamsShutdownStateListener.java │ │ ├── StreamsTopicConfig.java │ │ ├── TimeWindowedCogroupedKStreamX.java │ │ ├── TimeWindowedCogroupedStreamXImpl.java │ │ ├── TimeWindowedKStreamX.java │ │ ├── TimeWindowedStreamXImpl.java │ │ ├── ValueKErrorStreamX.java │ │ └── util │ │ ├── ConsumerGroupClient.java │ │ ├── ImprovedAdminClient.java │ │ ├── KafkaAdminException.java │ │ ├── SchemaTopicClient.java │ │ ├── TopicClient.java │ │ └── TopicSettings.java │ └── test │ ├── avro │ └── TestRecord.avsc │ ├── java │ └── com │ │ └── bakdata │ │ └── kafka │ │ ├── AvroMirrorTest.java │ │ ├── BranchedKStreamXTest.java │ │ ├── BranchedXTest.java │ │ ├── CogroupedStreamXTest.java │ │ ├── ConfiguredProducerAppTest.java │ │ ├── ConfiguredStreamsAppTest.java │ │ ├── ConsumedXTest.java │ │ ├── DoubleApp.java │ │ ├── EnvironmentKafkaConfigParserTest.java │ │ ├── ExecutableProducerAppTest.java │ │ ├── ExecutableStreamsAppTest.java │ │ ├── GroupedXTest.java │ │ ├── ImprovedStreamsConfigTest.java │ │ ├── JoinedXTest.java │ │ ├── KErrorStreamXTest.java │ │ ├── KGroupedStreamXTest.java │ │ ├── KGroupedTableXTest.java │ │ ├── KStreamXTest.java │ │ ├── KTableXTest.java │ │ ├── MaterializedXTest.java │ │ ├── ProducedXTest.java │ │ ├── RepartitionedXTest.java │ │ ├── SessionWindowedCogroupedKStreamXTest.java │ │ ├── SessionWindowedKStreamXTest.java │ │ ├── SimpleFixedKeyProcessor.java │ │ ├── SimpleLegacyProcessor.java │ │ ├── SimpleProcessor.java │ │ ├── SimpleTransformer.java │ │ ├── SimpleValueTransformer.java │ │ ├── SimpleValueTransformerWithKey.java │ │ ├── StoresXTest.java │ │ ├── StreamJoinedXTest.java │ │ ├── StreamsBuilderXTest.java │ │ ├── StreamsCleanUpRunnerTest.java │ │ ├── StreamsContextTest.java │ │ ├── StreamsExecutionOptionsTest.java │ │ ├── StringApp.java │ │ ├── TestHelper.java │ │ ├── TimeWindowedCogroupedKStreamXTest.java │ │ ├── TimeWindowedKStreamXTest.java │ │ ├── integration │ │ ├── ProducerCleanUpRunnerTest.java │ │ ├── ProducerRunnerTest.java │ │ ├── StreamsCleanUpRunnerTest.java │ │ └── StreamsRunnerTest.java │ │ ├── test_applications │ │ ├── AvroKeyProducer.java │ │ ├── AvroValueProducer.java │ │ ├── ComplexTopologyApplication.java │ │ ├── LabeledInputTopics.java │ │ ├── Mirror.java │ │ ├── MirrorKeyWithAvro.java │ │ ├── MirrorValueWithAvro.java │ │ ├── MirrorWithNonDefaultSerde.java │ │ ├── StringProducer.java │ │ ├── WordCount.java │ │ └── WordCountPattern.java │ │ └── util │ │ ├── SchemaTopicClientTest.java │ │ ├── TopicClientClusterTest.java │ │ └── TopicClientTest.java │ └── resources │ └── log4j2.xml ├── streams-bootstrap-large-messages ├── build.gradle.kts ├── lombok.config └── src │ └── main │ └── java │ └── com │ └── bakdata │ └── kafka │ ├── LargeMessageAppUtils.java │ ├── LargeMessageProducerApp.java │ └── LargeMessageStreamsApp.java └── streams-bootstrap-test ├── build.gradle.kts ├── lombok.config └── src ├── main └── java │ └── com │ └── bakdata │ └── kafka │ ├── ConsumerGroupVerifier.java │ ├── KafkaTestClient.java │ ├── ReaderBuilder.java │ ├── SenderBuilder.java │ └── TestTopologyFactory.java ├── test ├── java │ └── com │ │ └── bakdata │ │ └── kafka │ │ ├── ConsumerGroupVerifierTest.java │ │ ├── SimpleStreamsApp.java │ │ └── TestTopologyFactoryTest.java └── resources │ └── log4j2.xml └── testFixtures └── java └── com └── bakdata └── kafka ├── ApacheKafkaContainerCluster.java └── KafkaTest.java /.github/dependabot.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gradle" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | groups: 8 | kafka-dependencies: 9 | patterns: 10 | - "com.bakdata.kafka*" 11 | - "com.bakdata.fluent-kafka-streams-tests*" 12 | - "io.confluent*" 13 | - "org.apache.kafka*" 14 | log-dependencies: 15 | patterns: 16 | - "org.slf4j*" 17 | - "org.apache.logging.log4j*" 18 | test-dependencies: 19 | patterns: 20 | - "org.junit*" 21 | - "org.assertj*" 22 | - "*junit*" 23 | - "org.mockito*" 24 | - "org.testcontainers*" 25 | - "org.awaitility*" 26 | plugins: 27 | patterns: 28 | - "com.bakdata.release" 29 | - "com.bakdata.sonar" 30 | - "com.bakdata.sonatype" 31 | 32 | - package-ecosystem: "github-actions" 33 | directory: "/" 34 | schedule: 35 | interval: "daily" 36 | -------------------------------------------------------------------------------- /.github/lint-config.yaml: -------------------------------------------------------------------------------- 1 | target-branch: "master" 2 | -------------------------------------------------------------------------------- /.github/workflows/build-and-publish.yaml: -------------------------------------------------------------------------------- 1 | name: Build and Publish 2 | 3 | on: 4 | push: 5 | tags: ["**"] 6 | branches: 7 | - master 8 | pull_request: 9 | 10 | jobs: 11 | build-and-publish: 12 | name: Java Gradle 13 | uses: bakdata/ci-templates/.github/workflows/java-gradle-library.yaml@1.66.1 14 | with: 15 | java-version: 17 16 | secrets: 17 | sonar-token: ${{ secrets.SONARCLOUD_TOKEN }} 18 | sonar-organization: ${{ secrets.SONARCLOUD_ORGANIZATION }} 19 | signing-secret-key-ring: ${{ secrets.SONATYPE_SIGNING_SECRET_KEY_RING }} 20 | signing-key-id: ${{ secrets.SONATYPE_SIGNING_KEY_ID }} 21 | signing-password: ${{ secrets.SONATYPE_SIGNING_PASSWORD }} 22 | ossrh-username: ${{ secrets.SONATYPE_OSSRH_USERNAME }} 23 | ossrh-password: ${{ secrets.SONATYPE_OSSRH_PASSWORD }} 24 | github-token: ${{ secrets.GH_TOKEN }} 25 | -------------------------------------------------------------------------------- /.github/workflows/helm-lint.yaml: -------------------------------------------------------------------------------- 1 | name: Helm lint 2 | 3 | on: 4 | push: 5 | 6 | jobs: 7 | helm-lint: 8 | runs-on: ubuntu-22.04 9 | steps: 10 | - name: Lint Helm chart 11 | uses: bakdata/ci-templates/actions/helm-lint@1.66.1 12 | with: 13 | lint-config-path: ".github/lint-config.yaml" 14 | ref: ${{ github.ref_name }} 15 | -------------------------------------------------------------------------------- /.github/workflows/helm-publish.yaml: -------------------------------------------------------------------------------- 1 | name: Publish Helm Charts 2 | 3 | on: 4 | push: 5 | 6 | jobs: 7 | get-gradle-version: 8 | name: Get Gradle Version 9 | runs-on: ubuntu-22.04 10 | outputs: 11 | version: ${{ steps.get-version.outputs.version }} 12 | steps: 13 | - name: Check out repository 14 | uses: bakdata/ci-templates/actions/checkout@1.66.1 15 | 16 | - name: Set up Gradle with version ${{ inputs.gradle-version }} 17 | uses: bakdata/ci-templates/actions/java-gradle-setup@1.66.1 18 | with: 19 | java-distribution: "microsoft" 20 | java-version: "11" 21 | gradle-version: "wrapper" 22 | gradle-cache: "true" 23 | gradle-cache-read-only: "false" 24 | 25 | - name: Get version 26 | id: get-version 27 | run: | 28 | version=$(./gradlew properties -q | grep "^version:" | awk '{print $2}' | tr -d '[:space:]' | xargs) 29 | echo "version=$version" >> "$GITHUB_OUTPUT" 30 | shell: bash 31 | 32 | 33 | helm-publish: 34 | name: Publish Helm chart 35 | uses: bakdata/ci-templates/.github/workflows/helm-multi-release.yaml@1.66.1 36 | needs: get-gradle-version 37 | with: 38 | charts-path: "./charts" 39 | subdirs: "['producer-app','producer-app-cleanup-job','streams-app','streams-app-cleanup-job']" 40 | gh-pages-branch: gh-pages 41 | version: ${{ needs.get-gradle-version.outputs.version }} 42 | secrets: 43 | github-username: ${{ secrets.GH_USERNAME }} 44 | github-email: ${{ secrets.GH_EMAIL }} 45 | github-token: ${{ secrets.GH_TOKEN }} 46 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | release-type: 7 | description: "The scope of the release (major, minor or patch)." 8 | type: choice 9 | required: true 10 | default: patch 11 | options: 12 | - patch 13 | - minor 14 | - major 15 | 16 | jobs: 17 | java-gradle-release: 18 | name: Java Gradle 19 | uses: bakdata/ci-templates/.github/workflows/java-gradle-release.yaml@1.66.1 20 | with: 21 | java-version: 17 22 | release-type: "${{ inputs.release-type }}" 23 | secrets: 24 | github-email: "${{ secrets.GH_EMAIL }}" 25 | github-username: "${{ secrets.GH_USERNAME }}" 26 | github-token: "${{ secrets.GH_TOKEN }}" 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | !.github 3 | !.gitignore 4 | build/ 5 | out/ 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) {{year}} bakdata 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | alias(libs.plugins.release) 3 | alias(libs.plugins.sonar) 4 | alias(libs.plugins.sonatype) 5 | alias(libs.plugins.lombok) 6 | } 7 | 8 | allprojects { 9 | group = "com.bakdata.kafka" 10 | 11 | repositories { 12 | mavenCentral() 13 | maven(url = "https://packages.confluent.io/maven/") 14 | maven(url = "https://s01.oss.sonatype.org/content/repositories/snapshots") 15 | } 16 | } 17 | 18 | subprojects { 19 | plugins.matching { it is JavaPlugin }.all { 20 | apply(plugin = "java-test-fixtures") 21 | apply(plugin = "io.freefair.lombok") 22 | 23 | configure { 24 | toolchain { 25 | languageVersion = JavaLanguageVersion.of(11) 26 | } 27 | } 28 | } 29 | 30 | publication { 31 | developers { 32 | developer { 33 | name.set("Lawrence Benson") 34 | id.set("lawben") 35 | } 36 | developer { 37 | name.set("Benjamin Feldmann") 38 | id.set("BJennWare") 39 | } 40 | developer { 41 | name.set("Ramin Gharib") 42 | id.set("raminqaf") 43 | } 44 | developer { 45 | name.set("Arvid Heise") 46 | id.set("AHeise") 47 | } 48 | developer { 49 | name.set("Victor Künstler") 50 | id.set("VictorKuenstler") 51 | } 52 | developer { 53 | name.set("Sven Lehmann") 54 | id.set("SvenLehmann") 55 | } 56 | developer { 57 | name.set("Torben Meyer") 58 | id.set("torbsto") 59 | } 60 | developer { 61 | name.set("Fabian Paul") 62 | id.set("fapaul") 63 | } 64 | developer { 65 | name.set("Yannick Roeder") 66 | id.set("yannick-roeder") 67 | } 68 | developer { 69 | name.set("Philipp Schirmer") 70 | id.set("philipp94831") 71 | } 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /charts/README.md: -------------------------------------------------------------------------------- 1 | # Kafka Streams Bakdata Helm Repository 2 | 3 | Collection of commonly used charts associated with bakdata Kafka streaming applications. 4 | 5 | ## Install 6 | 7 | ``` 8 | helm repo add bakdata-common https://bakdata.github.io/streams-bootstrap/ 9 | helm install bakdata-common/ 10 | ``` 11 | -------------------------------------------------------------------------------- /charts/producer-app-cleanup-job/.helmignore: -------------------------------------------------------------------------------- 1 | *.tgz 2 | .* 3 | -------------------------------------------------------------------------------- /charts/producer-app-cleanup-job/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | name: producer-app-cleanup-job 3 | description: A helm chart for deploying a clean up job for a Kafka producer application. 4 | version: 0.1.0 5 | maintainers: 6 | - name: bakdata 7 | email: opensource@bakdata.com 8 | url: bakdata.com 9 | -------------------------------------------------------------------------------- /charts/producer-app-cleanup-job/README.md: -------------------------------------------------------------------------------- 1 | # producer-app cleanup helm chart 2 | 3 | This chart can be used to deploy a cleanup job for your Kafka producer app developed using streams-bootstrap. 4 | Make sure to destroy the corresponding producer deployment before running the cleanup job. 5 | 6 | ## Configuration 7 | 8 | You can specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. 9 | 10 | Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. 11 | 12 | All relevant configurations of 13 | the [producer app](https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app) are available as well. 14 | Therefore, you can just reuse your `values.yaml` file. 15 | -------------------------------------------------------------------------------- /charts/producer-app-cleanup-job/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "producer-app.name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 7 | {{- end }} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | If release name contains chart name it will be used as a full name. 13 | */}} 14 | {{- define "producer-app.fullname" -}} 15 | {{- if .Values.fullnameOverride }} 16 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 17 | {{- else }} 18 | {{- $name := default .Chart.Name .Values.nameOverride }} 19 | {{- if contains $name .Release.Name }} 20 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 21 | {{- else }} 22 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 23 | {{- end }} 24 | {{- end }} 25 | {{- end }} 26 | 27 | {{/* 28 | Create chart name and version as used by the chart label. 29 | */}} 30 | {{- define "producer-app.chart" -}} 31 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 32 | {{- end }} 33 | 34 | {{/* 35 | Common labels 36 | */}} 37 | {{- define "producer-app.labels" -}} 38 | helm.sh/chart: {{ include "producer-app.chart" . }} 39 | {{ include "producer-app.selectorLabels" . }} 40 | {{- if .Chart.AppVersion }} 41 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 42 | {{- end }} 43 | app.kubernetes.io/managed-by: {{ .Release.Service }} 44 | {{- end }} 45 | 46 | {{/* 47 | Selector labels 48 | */}} 49 | {{- define "producer-app.selectorLabels" -}} 50 | app.kubernetes.io/name: {{ include "producer-app.name" . }} 51 | app.kubernetes.io/instance: {{ .Release.Name }} 52 | {{- end }} 53 | 54 | {{/* 55 | Helper function to add annotations to resources 56 | */}} 57 | {{- define "producer-app.annotations" -}} 58 | {{- if .Values.annotations }} 59 | annotations: 60 | {{- range $key, $value := .Values.annotations }} 61 | {{ $key | quote }}: {{ $value | quote }} 62 | {{- end }} 63 | {{- end }} 64 | {{- end }} 65 | -------------------------------------------------------------------------------- /charts/producer-app-cleanup-job/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.files }} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: {{ include "producer-app.fullname" . }} 6 | {{- include "producer-app.annotations" . }} 7 | labels: 8 | {{- include "producer-app.labels" . | nindent 4 }} 9 | {{- range $key, $value := .Values.labels }} 10 | {{ $key | quote }}: {{ $value | quote }} 11 | {{- end }} 12 | data: 13 | {{- range $key, $value := .Values.files }} 14 | {{ $key }}: {{ $value.content | quote }} 15 | {{- end }} 16 | {{ end }} 17 | -------------------------------------------------------------------------------- /charts/producer-app-cleanup-job/templates/secrets.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.secrets }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ include "producer-app.fullname" . }} 6 | {{- include "producer-app.annotations" . }} 7 | labels: 8 | {{- include "producer-app.labels" . | nindent 4 }} 9 | {{- range $key, $value := .Values.labels }} 10 | {{ $key | quote }}: {{ $value | quote }} 11 | {{- end }} 12 | type: Opaque 13 | data: 14 | {{- range $key, $value := .Values.secrets }} 15 | {{ $key }}: {{ $value | b64enc }} 16 | {{- end }} 17 | {{- end }} 18 | -------------------------------------------------------------------------------- /charts/producer-app-cleanup-job/values.yaml: -------------------------------------------------------------------------------- 1 | nameOverride: "" 2 | fullnameOverride: "" 3 | 4 | image: producerApp 5 | imageTag: latest 6 | imagePullPolicy: Always 7 | 8 | imagePullSecrets: [] 9 | 10 | restartPolicy: OnFailure 11 | 12 | configurationEnvPrefix: "APP" 13 | 14 | files: {} 15 | # log4j2.xml: 16 | # mountPath: app/resources 17 | # content: "foo bar" 18 | 19 | kafka: 20 | # bootstrapServers: "test:9092" 21 | # schemaRegistryUrl: "url:1234" 22 | config: {} 23 | # max.poll.records: 500 24 | # Note that YAML may convert large integers to scientific notation. Use Strings to avoid this. 25 | # max.request.size: "1000000" 26 | # outputTopic: output 27 | labeledOutputTopics: {} 28 | # label: output 29 | 30 | commandLine: {} 31 | # MY_CLI_PARAM: "foo-bar" 32 | 33 | env: {} 34 | # MY_ENV_VARIABLE: foo-bar 35 | 36 | secrets: {} 37 | # MY_SECRET: fo-bar 38 | secretRefs: {} 39 | # MY_SECRET: 40 | # name: secretName 41 | # key: secretKey 42 | secretFilesRefs: [] 43 | # - name: my-secret 44 | # volume: secret-volume 45 | # mountPath: /etc/test 46 | # readOnly: true 47 | # subPath: optional-subpath 48 | 49 | annotations: {} 50 | # MY_ANNOTATION: "foo-bar" 51 | 52 | labels: {} 53 | # MY_LABEL: "foo-bar" 54 | 55 | # serviceAccountName: foo 56 | 57 | tolerations: [] 58 | # - key: "foo" 59 | # operator: "Exists" 60 | # effect: "NoSchedule" 61 | # - key: "bar" 62 | # operator: "Exists" 63 | # effect: "NoSchedule" 64 | 65 | ## Affinity for pod assignment (evaluated as template) 66 | ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity 67 | ## 68 | affinity: {} 69 | 70 | resources: 71 | requests: 72 | cpu: 200m 73 | memory: 300Mi 74 | limits: 75 | cpu: 500m 76 | memory: 2G 77 | 78 | javaOptions: 79 | maxRAMPercentage: 75 80 | others: [] 81 | # - "-XX:MinRAMPercentage=50.0" 82 | 83 | backoffLimit: 6 84 | 85 | podAnnotations: {} 86 | 87 | podLabels: {} 88 | -------------------------------------------------------------------------------- /charts/producer-app/.helmignore: -------------------------------------------------------------------------------- 1 | *.tgz 2 | .* 3 | -------------------------------------------------------------------------------- /charts/producer-app/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | name: producer-app 3 | description: A helm chart for deploying a Kafka producer application based on the bakdata KafkaProducerApplication template. 4 | version: 0.1.0 5 | maintainers: 6 | - name: bakdata 7 | email: opensource@bakdata.com 8 | url: bakdata.com 9 | -------------------------------------------------------------------------------- /charts/producer-app/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "producer-app.name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 7 | {{- end }} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | If release name contains chart name it will be used as a full name. 13 | */}} 14 | {{- define "producer-app.fullname" -}} 15 | {{- if .Values.fullnameOverride }} 16 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 17 | {{- else }} 18 | {{- $name := default .Chart.Name .Values.nameOverride }} 19 | {{- if contains $name .Release.Name }} 20 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 21 | {{- else }} 22 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 23 | {{- end }} 24 | {{- end }} 25 | {{- end }} 26 | 27 | {{/* 28 | Create chart name and version as used by the chart label. 29 | */}} 30 | {{- define "producer-app.chart" -}} 31 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 32 | {{- end }} 33 | 34 | {{/* 35 | Common labels 36 | */}} 37 | {{- define "producer-app.labels" -}} 38 | helm.sh/chart: {{ include "producer-app.chart" . }} 39 | {{ include "producer-app.selectorLabels" . }} 40 | {{- if .Chart.AppVersion }} 41 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 42 | {{- end }} 43 | app.kubernetes.io/managed-by: {{ .Release.Service }} 44 | {{- end }} 45 | 46 | {{/* 47 | Selector labels 48 | */}} 49 | {{- define "producer-app.selectorLabels" -}} 50 | app.kubernetes.io/name: {{ include "producer-app.name" . }} 51 | app.kubernetes.io/instance: {{ .Release.Name }} 52 | {{- end }} 53 | 54 | {{/* 55 | Helper function to add annotations to resources 56 | */}} 57 | {{- define "producer-app.annotations" -}} 58 | {{- if .Values.annotations }} 59 | annotations: 60 | {{- range $key, $value := .Values.annotations }} 61 | {{ $key | quote }}: {{ $value | quote }} 62 | {{- end }} 63 | {{- end }} 64 | {{- end }} 65 | -------------------------------------------------------------------------------- /charts/producer-app/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.files }} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: {{ include "producer-app.fullname" . }} 6 | {{- include "producer-app.annotations" . }} 7 | labels: 8 | {{- include "producer-app.labels" . | nindent 4 }} 9 | {{- range $key, $value := .Values.labels }} 10 | {{ $key | quote }}: {{ $value | quote }} 11 | {{- end }} 12 | data: 13 | {{- range $key, $value := .Values.files }} 14 | {{ $key }}: {{ $value.content | quote }} 15 | {{- end }} 16 | {{ end }} 17 | -------------------------------------------------------------------------------- /charts/producer-app/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.deployment }} 2 | {{- $root := . -}} 3 | {{- if .Capabilities.APIVersions.Has "apps/v1" }} 4 | apiVersion: apps/v1 5 | {{- else }} 6 | apiVersion: apps/v1beta1 7 | {{- end }} 8 | kind: Deployment 9 | metadata: 10 | name: {{ include "producer-app.fullname" . }} 11 | {{- include "producer-app.annotations" . }} 12 | labels: 13 | {{- include "producer-app.labels" . | nindent 4 }} 14 | streams-bootstrap/kind: {{ .Chart.Name }} 15 | {{- range $key, $value := .Values.labels }} 16 | {{ $key | quote }}: {{ $value | quote }} 17 | {{- end }} 18 | spec: 19 | replicas: {{ .Values.replicaCount }} 20 | selector: 21 | matchLabels: 22 | {{- include "producer-app.selectorLabels" . | nindent 6 }} 23 | template: 24 | {{ include "producer-app.podTemplate" . | indent 4 }} 25 | {{- end }} 26 | -------------------------------------------------------------------------------- /charts/producer-app/templates/job.yaml: -------------------------------------------------------------------------------- 1 | {{- if not .Values.deployment -}} 2 | {{- $root := . -}} 3 | {{- if .Values.schedule -}} 4 | {{- if .Capabilities.APIVersions.Has "batch/v1/CronJob" }} 5 | apiVersion: batch/v1 6 | {{- else }} 7 | apiVersion: batch/v1beta1 8 | {{- end }} 9 | kind: CronJob 10 | {{- else -}} 11 | apiVersion: batch/v1 12 | kind: Job 13 | {{- end }} 14 | metadata: 15 | name: {{ include "producer-app.fullname" . }} 16 | {{- include "producer-app.annotations" . }} 17 | labels: 18 | {{- include "producer-app.labels" . | nindent 4 }} 19 | streams-bootstrap/kind: {{ .Chart.Name }} 20 | {{- range $key, $value := .Values.labels }} 21 | {{ $key | quote }}: {{ $value | quote }} 22 | {{- end }} 23 | spec: 24 | {{- if .Values.schedule }} 25 | suspend: {{ .Values.suspend }} 26 | schedule: {{ quote .Values.schedule }} 27 | successfulJobsHistoryLimit: {{ .Values.successfulJobsHistoryLimit }} 28 | failedJobsHistoryLimit: {{ .Values.failedJobsHistoryLimit }} 29 | concurrencyPolicy: Replace 30 | jobTemplate: 31 | metadata: 32 | {{- include "producer-app.annotations" . | indent 4 }} 33 | labels: 34 | {{- include "producer-app.selectorLabels" . | nindent 8 }} 35 | streams-bootstrap/kind: {{ .Chart.Name }} 36 | {{- range $key, $value := .Values.labels }} 37 | {{ $key | quote }}: {{ $value | quote }} 38 | {{- end }} 39 | spec: 40 | {{- if .Values.ttlSecondsAfterFinished }} 41 | ttlSecondsAfterFinished: {{ .Values.ttlSecondsAfterFinished }} 42 | {{- end }} 43 | template: 44 | {{ include "producer-app.podTemplate" . | indent 8 }} 45 | backoffLimit: {{ .Values.backoffLimit }} 46 | {{ else }} 47 | {{- if .Values.ttlSecondsAfterFinished }} 48 | ttlSecondsAfterFinished: {{ .Values.ttlSecondsAfterFinished }} 49 | {{- end }} 50 | template: 51 | {{ include "producer-app.podTemplate" . | indent 4 }} 52 | backoffLimit: {{ .Values.backoffLimit }} 53 | {{- end -}} 54 | {{- end }} 55 | -------------------------------------------------------------------------------- /charts/producer-app/templates/secrets.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.secrets }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ include "producer-app.fullname" . }} 6 | {{- include "producer-app.annotations" . }} 7 | labels: 8 | {{- include "producer-app.labels" . | nindent 4 }} 9 | {{- range $key, $value := .Values.labels }} 10 | {{ $key | quote }}: {{ $value | quote }} 11 | {{- end }} 12 | type: Opaque 13 | data: 14 | {{- range $key, $value := .Values.secrets }} 15 | {{ $key }}: {{ $value | b64enc }} 16 | {{- end }} 17 | {{- end }} 18 | -------------------------------------------------------------------------------- /charts/producer-app/templates/service.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.service.enabled }} 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: {{ include "producer-app.fullname" . }} 6 | {{- include "producer-app.annotations" . }} 7 | labels: 8 | {{- include "producer-app.labels" . | nindent 4 }} 9 | {{- range $key, $value := .Values.service.labels }} 10 | {{ $key }}: {{ $value }} 11 | {{- end }} 12 | spec: 13 | selector: 14 | {{- include "producer-app.selectorLabels" . | nindent 4 }} 15 | ports: 16 | {{- range .Values.ports }} 17 | {{- if .servicePort }} 18 | - targetPort: {{ .containerPort }} 19 | port: {{ .servicePort }} 20 | name: {{ .name }} 21 | {{- if .protocol }} 22 | protocol: {{ .protocol }} 23 | {{- end }} 24 | {{- end }} 25 | {{- end }} 26 | type: {{ .Values.service.type }} 27 | {{- end }} 28 | -------------------------------------------------------------------------------- /charts/producer-app/values.yaml: -------------------------------------------------------------------------------- 1 | nameOverride: "" 2 | fullnameOverride: "" 3 | 4 | image: producerApp 5 | imageTag: latest 6 | imagePullPolicy: Always 7 | 8 | imagePullSecrets: [] 9 | 10 | # if true, deploy as kubernetes deployment instead of CronJob/Job 11 | deployment: false 12 | # Optional, applied for deployment: 13 | # replicaCount: 1 14 | 15 | restartPolicy: OnFailure 16 | 17 | configurationEnvPrefix: "APP" 18 | 19 | files: {} 20 | # log4j2.xml: 21 | # mountPath: app/resources 22 | # content: "foo bar" 23 | 24 | # Optional: Cron schedule for this producer job 25 | # schedule: "0 12 * * *" 26 | 27 | suspend: false 28 | 29 | # serviceAccountName: foo 30 | 31 | tolerations: [] 32 | # - key: "foo" 33 | # operator: "Exists" 34 | # effect: "NoSchedule" 35 | # - key: "bar" 36 | # operator: "Exists" 37 | # effect: "NoSchedule" 38 | 39 | ## Affinity for pod assignment (evaluated as template) 40 | ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity 41 | ## 42 | affinity: {} 43 | 44 | resources: 45 | requests: 46 | cpu: 200m 47 | memory: 300Mi 48 | limits: 49 | memory: 2G 50 | cpu: 500m 51 | 52 | kafka: 53 | # bootstrapServers: "test:9092" 54 | # schemaRegistryUrl: "url:1234" 55 | config: {} 56 | # max.poll.records: 500 57 | # Note that YAML may convert large integers to scientific notation. Use Strings to avoid this. 58 | # max.request.size: "1000000" 59 | # outputTopic: output 60 | labeledOutputTopics: {} 61 | # label: output 62 | 63 | commandLine: {} 64 | # MY_CLI_PARAM: "foo-bar" 65 | 66 | env: {} 67 | # MY_ENV_VARIABLE: foo-bar 68 | 69 | secrets: {} 70 | # MY_SECRET: fo-bar 71 | secretRefs: {} 72 | # MY_SECRET: 73 | # name: secretName 74 | # key: secretKey 75 | secretFilesRefs: [] 76 | # - name: my-secret 77 | # volume: secret-volume 78 | # mountPath: /etc/test 79 | # readOnly: true 80 | # subPath: optional-subpath 81 | 82 | annotations: {} 83 | # MY_ANNOTATION: "foo-bar" 84 | 85 | labels: {} 86 | # MY_LABEL: "foo-bar" 87 | 88 | javaOptions: 89 | maxRAMPercentage: 75 90 | others: [] 91 | # - "-XX:MinRAMPercentage=50.0" 92 | 93 | successfulJobsHistoryLimit: 1 94 | failedJobsHistoryLimit: 1 95 | backoffLimit: 6 96 | # ttlSecondsAfterFinished: 100 97 | 98 | # Producer's container ports 99 | ports: [] 100 | # - containerPort: 8080 101 | # # Service can reference port by name 102 | # name: http 103 | # # Optional: If not set, kubernetes will use 'TCP' 104 | # protocol: "TCP" 105 | # # The port that will be exposed by the service. 106 | # servicePort: 80 107 | 108 | service: 109 | enabled: false 110 | labels: {} 111 | type: "ClusterIP" 112 | 113 | # Arbitrary Probe v1 definition for producer: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#probe-v1-core 114 | livenessProbe: {} 115 | 116 | # Arbitrary Probe v1 definition for producer: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#probe-v1-core 117 | readinessProbe: {} 118 | 119 | podAnnotations: {} 120 | 121 | podLabels: {} 122 | -------------------------------------------------------------------------------- /charts/streams-app-cleanup-job/.helmignore: -------------------------------------------------------------------------------- 1 | *.tgz 2 | .* 3 | -------------------------------------------------------------------------------- /charts/streams-app-cleanup-job/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | name: streams-app-cleanup-job 3 | description: A helm chart for deploying a kafka streams application based on the bakdata KafkaStreamsApplication template. 4 | version: 0.1.0 5 | maintainers: 6 | - name: bakdata 7 | email: opensource@bakdata.com 8 | url: bakdata.com 9 | -------------------------------------------------------------------------------- /charts/streams-app-cleanup-job/README.md: -------------------------------------------------------------------------------- 1 | # streams-app cleanup helm chart 2 | 3 | This chart can be used to deploy a cleanup job for your Kafka Streams app developed using streams-bootstrap. 4 | Make sure to destroy the corresponding streams deployment before running the cleanup job. 5 | 6 | ## Configuration 7 | 8 | You can specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. 9 | 10 | Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. 11 | 12 | All relevant configurations of 13 | the [streams app](https://github.com/bakdata/streams-bootstrap/tree/master/charts/streams-app) are available as well. 14 | Therefore, you can just reuse your `values.yaml` file. 15 | 16 | Additionally, the following parameters can be configured: 17 | 18 | | Parameter | Description | Default | 19 | |-----------------|----------------------------------------------------------------------------------------------------------------------------------------------------|-------------| 20 | | `restartPolicy` | [Restart policy](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy) to use for the job. | `OnFailure` | 21 | | `deleteOutput` | Whether the output topics with their associated schemas and the consumer group should be deleted. | `false` | 22 | | `backoffLimit` | The number of times to restart an unsuccessful job. See https://kubernetes.io/docs/concepts/workloads/controllers/job/#pod-backoff-failure-policy. | `6` | 23 | -------------------------------------------------------------------------------- /charts/streams-app-cleanup-job/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "streams-app.name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 7 | {{- end }} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | If release name contains chart name it will be used as a full name. 13 | */}} 14 | {{- define "streams-app.fullname" -}} 15 | {{- if .Values.fullnameOverride }} 16 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 17 | {{- else }} 18 | {{- $name := default .Chart.Name .Values.nameOverride }} 19 | {{- if contains $name .Release.Name }} 20 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 21 | {{- else }} 22 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 23 | {{- end }} 24 | {{- end }} 25 | {{- end }} 26 | 27 | {{/* 28 | Create chart name and version as used by the chart label. 29 | */}} 30 | {{- define "streams-app.chart" -}} 31 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 32 | {{- end }} 33 | 34 | {{/* 35 | Common labels 36 | */}} 37 | {{- define "streams-app.labels" -}} 38 | helm.sh/chart: {{ include "streams-app.chart" . }} 39 | {{ include "streams-app.selectorLabels" . }} 40 | {{- if .Chart.AppVersion }} 41 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 42 | {{- end }} 43 | app.kubernetes.io/managed-by: {{ .Release.Service }} 44 | {{- range $key, $value := .Values.labels }} 45 | {{ $key | quote }}: {{ $value | quote }} 46 | {{- end }} 47 | {{- end }} 48 | 49 | {{/* 50 | Selector labels 51 | */}} 52 | {{- define "streams-app.selectorLabels" -}} 53 | app.kubernetes.io/name: {{ include "streams-app.name" . }} 54 | app.kubernetes.io/instance: {{ .Release.Name }} 55 | {{- end }} 56 | 57 | {{/* 58 | Helper function to add annotations to resources 59 | */}} 60 | {{- define "streams-app.annotations" -}} 61 | {{- if .Values.annotations }} 62 | annotations: 63 | {{- range $key, $value := .Values.annotations }} 64 | {{ $key | quote }}: {{ $value | quote }} 65 | {{- end }} 66 | {{- end }} 67 | {{- end }} 68 | -------------------------------------------------------------------------------- /charts/streams-app-cleanup-job/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.files }} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: {{ include "streams-app.fullname" . }} 6 | {{- include "streams-app.annotations" . }} 7 | labels: 8 | {{- include "streams-app.labels" . | nindent 4 }} 9 | {{- range $key, $value := .Values.labels }} 10 | {{ $key | quote }}: {{ $value | quote }} 11 | {{- end }} 12 | data: 13 | {{- range $key, $value := .Values.files }} 14 | {{ $key }}: {{ $value.content | quote }} 15 | {{- end }} 16 | {{ end }} 17 | -------------------------------------------------------------------------------- /charts/streams-app-cleanup-job/templates/secrets.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.secrets }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ include "streams-app.fullname" . }} 6 | {{- include "streams-app.annotations" . }} 7 | labels: 8 | {{- include "streams-app.labels" . | nindent 4 }} 9 | {{- range $key, $value := .Values.labels }} 10 | {{ $key | quote }}: {{ $value | quote }} 11 | {{- end }} 12 | type: Opaque 13 | data: 14 | {{- range $key, $value := .Values.secrets }} 15 | {{ $key }}: {{ $value | b64enc }} 16 | {{- end }} 17 | {{- end }} 18 | -------------------------------------------------------------------------------- /charts/streams-app-cleanup-job/values.yaml: -------------------------------------------------------------------------------- 1 | nameOverride: "" 2 | fullnameOverride: "" 3 | 4 | image: streamsApp 5 | imageTag: latest 6 | imagePullPolicy: Always 7 | 8 | imagePullSecrets: [] 9 | 10 | restartPolicy: OnFailure 11 | 12 | configurationEnvPrefix: "APP" 13 | 14 | files: {} 15 | # log4j2.xml: 16 | # mountPath: app/resources 17 | # content: "foo bar" 18 | 19 | kafka: 20 | # bootstrapServers: "test:9092" 21 | # schemaRegistryUrl: "url:1234" 22 | config: {} 23 | # max.poll.records: 500 24 | # Note that YAML may convert large integers to scientific notation. Use Strings to avoid this. 25 | # max.request.size: "1000000" 26 | inputTopics: [] 27 | # - input 28 | # - input2 29 | labeledInputTopics: {} 30 | # label: 31 | # - input 32 | # - input2 33 | # inputPattern: .*-input 34 | labeledInputPatterns: {} 35 | # label: .*-input 36 | # outputTopic: output 37 | labeledOutputTopics: {} 38 | # label: output 39 | # errorTopic: error 40 | deleteOutput: false 41 | 42 | commandLine: {} 43 | # MY_CLI_PARAM: "foo-bar" 44 | 45 | env: {} 46 | # MY_ENV_VARIABLE: foo-bar 47 | # 48 | secrets: {} 49 | # MY_SECRET: fo-bar 50 | secretRefs: {} 51 | # MY_SECRET: 52 | # name: secretName 53 | # key: secretKey 54 | secretFilesRefs: [] 55 | # - name: my-secret 56 | # volume: secret-volume 57 | # mountPath: /etc/test 58 | # readOnly: true 59 | # subPath: optional-subpath 60 | 61 | annotations: {} 62 | # MY_ANNOTATION: "foo-bar" 63 | 64 | labels: {} 65 | # MY_LABEL: "foo-bar" 66 | 67 | # serviceAccountName: foo 68 | 69 | tolerations: [] 70 | # - key: "foo" 71 | # operator: "Exists" 72 | # effect: "NoSchedule" 73 | # - key: "bar" 74 | # operator: "Exists" 75 | # effect: "NoSchedule" 76 | 77 | 78 | ## Affinity for pod assignment (evaluated as template) 79 | ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity 80 | ## 81 | affinity: {} 82 | 83 | resources: 84 | requests: 85 | cpu: 200m 86 | memory: 300Mi 87 | limits: 88 | cpu: 500m 89 | memory: 2G 90 | 91 | javaOptions: 92 | maxRAMPercentage: 75 93 | others: [] 94 | # - "-XX:MinRAMPercentage=50.0" 95 | 96 | backoffLimit: 6 97 | 98 | podAnnotations: {} 99 | 100 | podLabels: {} 101 | -------------------------------------------------------------------------------- /charts/streams-app/.helmignore: -------------------------------------------------------------------------------- 1 | *.tgz 2 | .* 3 | -------------------------------------------------------------------------------- /charts/streams-app/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | name: streams-app 3 | description: A helm chart for deploying a kafka streams application based on the bakdata KafkaStreamsApplication template. 4 | version: 0.1.0 5 | maintainers: 6 | - name: bakdata 7 | email: opensource@bakdata.com 8 | url: bakdata.com 9 | -------------------------------------------------------------------------------- /charts/streams-app/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "streams-app.name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 7 | {{- end }} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | If release name contains chart name it will be used as a full name. 13 | */}} 14 | {{- define "streams-app.fullname" -}} 15 | {{- if .Values.fullnameOverride }} 16 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 17 | {{- else }} 18 | {{- $name := default .Chart.Name .Values.nameOverride }} 19 | {{- if contains $name .Release.Name }} 20 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 21 | {{- else }} 22 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 23 | {{- end }} 24 | {{- end }} 25 | {{- end }} 26 | 27 | {{/* 28 | Create chart name and version as used by the chart label. 29 | */}} 30 | {{- define "streams-app.chart" -}} 31 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 32 | {{- end }} 33 | 34 | {{/* 35 | Common labels 36 | */}} 37 | {{- define "streams-app.labels" -}} 38 | helm.sh/chart: {{ include "streams-app.chart" . }} 39 | {{ include "streams-app.selectorLabels" . }} 40 | {{- if .Chart.AppVersion }} 41 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 42 | {{- end }} 43 | app.kubernetes.io/managed-by: {{ .Release.Service }} 44 | {{- end }} 45 | 46 | {{/* 47 | Selector labels 48 | */}} 49 | {{- define "streams-app.selectorLabels" -}} 50 | app.kubernetes.io/name: {{ include "streams-app.name" . }} 51 | app.kubernetes.io/instance: {{ .Release.Name }} 52 | {{- end }} 53 | 54 | {{/* 55 | Define default annotations from .Values.annotations. 56 | This will be used across resources. 57 | */}} 58 | {{- define "streams-app.annotations" -}} 59 | {{- if or .Values.annotations }} 60 | annotations: 61 | {{- range $key, $value := .Values.annotations }} 62 | {{ $key | quote }}: {{ $value | quote }} 63 | {{- end }} 64 | {{- end }} 65 | {{- end }} 66 | 67 | {{/* 68 | Define annotations helper for Deployment. 69 | Includes default annotations and conditionally adds consumerGroup if applicable. 70 | */}} 71 | {{- define "streams-app.deployment-annotations" -}} 72 | {{- if or .Values.annotations .Values.kafka.applicationId }} 73 | annotations: 74 | {{- range $key, $value := .Values.annotations }} 75 | {{ $key | quote }}: {{ $value | quote }} 76 | {{- end }} 77 | 78 | {{- /* Conditionally add the consumerGroup annotation if needed */ -}} 79 | {{- if and .Values.kafka.applicationId (not .Values.annotations.consumerGroup) }} 80 | consumerGroup: {{ .Values.kafka.applicationId | quote }} 81 | {{- end }} 82 | {{- end }} 83 | {{- end }} 84 | 85 | -------------------------------------------------------------------------------- /charts/streams-app/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.files }} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: {{ include "streams-app.fullname" . }} 6 | {{- include "streams-app.annotations" . }} 7 | labels: 8 | {{- include "streams-app.labels" . | nindent 4 }} 9 | {{- range $key, $value := .Values.labels }} 10 | {{ $key | quote }}: {{ $value | quote }} 11 | {{- end }} 12 | data: 13 | {{- range $key, $value := .Values.files }} 14 | {{ $key }}: {{ $value.content | quote }} 15 | {{- end }} 16 | {{ end }} 17 | -------------------------------------------------------------------------------- /charts/streams-app/templates/jmx-configmap.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.prometheus.jmx.enabled }} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: {{ include "streams-app.fullname" . }}-jmx 6 | {{- include "streams-app.annotations" . }} 7 | labels: 8 | {{- include "streams-app.labels" . | nindent 4 }} 9 | {{- range $key, $value := .Values.labels }} 10 | {{ $key | quote }}: {{ $value | quote }} 11 | {{- end }} 12 | data: 13 | jmx-kafka-streams-app-prometheus.yml: |+ 14 | jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:{{ .Values.jmx.port }}/jmxrmi 15 | lowercaseOutputName: true 16 | lowercaseOutputLabelNames: true 17 | ssl: false 18 | rules: {{ toYaml .Values.prometheus.jmx.metricRules | nindent 4 }} 19 | {{- end }} 20 | -------------------------------------------------------------------------------- /charts/streams-app/templates/secrets.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.secrets }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ include "streams-app.fullname" . }} 6 | {{- include "streams-app.annotations" . }} 7 | labels: 8 | {{- include "streams-app.labels" . | nindent 4 }} 9 | {{- range $key, $value := .Values.labels }} 10 | {{ $key | quote }}: {{ $value | quote }} 11 | {{- end }} 12 | type: Opaque 13 | data: 14 | {{- range $key, $value := .Values.secrets }} 15 | {{ $key }}: {{ $value | b64enc }} 16 | {{- end }} 17 | {{- end }} 18 | -------------------------------------------------------------------------------- /charts/streams-app/templates/service.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.service.enabled }} 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: {{ include "streams-app.fullname" . }} 6 | {{- include "streams-app.annotations" . }} 7 | labels: 8 | {{- include "streams-app.labels" . | nindent 4 }} 9 | {{- range $key, $value := .Values.service.labels }} 10 | {{ $key }}: {{ $value }} 11 | {{- end }} 12 | spec: 13 | selector: 14 | {{- include "streams-app.selectorLabels" . | nindent 4 }} 15 | ports: 16 | {{- range .Values.ports }} 17 | {{- if .servicePort }} 18 | - targetPort: {{ .containerPort }} 19 | port: {{ .servicePort }} 20 | name: {{ .name }} 21 | {{- if .protocol }} 22 | protocol: {{ .protocol }} 23 | {{- end }} 24 | {{- end }} 25 | {{- end }} 26 | type: {{ .Values.service.type }} 27 | {{- end }} 28 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | version=4.1.4-SNAPSHOT 2 | org.gradle.caching=true 3 | org.gradle.parallel=true 4 | org.gradle.jvmargs=-Xmx4096m 5 | -------------------------------------------------------------------------------- /gradle/libs.versions.toml: -------------------------------------------------------------------------------- 1 | [versions] 2 | junit = "5.12.2" 3 | mockito = "5.17.0" 4 | testcontainers = "1.21.0" 5 | 6 | [libraries] 7 | kafka-streams-utils = { group = "com.bakdata.kafka", name = "kafka-streams-utils", version = "1.1.1" } 8 | kafka-tools = { group = "org.apache.kafka", name = "kafka-tools" } 9 | kafka-streams = { group = "org.apache.kafka", name = "kafka-streams" } 10 | kafka-clients = { group = "org.apache.kafka", name = "kafka-clients" } 11 | kafka-schema-serializer = { group = "io.confluent", name = "kafka-schema-serializer" } 12 | kafka-schema-registry-client = { group = "io.confluent", name = "kafka-schema-registry-client" } 13 | kafka-streams-avro-serde = { group = "io.confluent", name = "kafka-streams-avro-serde" } 14 | largeMessage-bom = { group = "com.bakdata.kafka", name = "large-message-bom", version = "2.12.1" } 15 | largeMessage-core = { group = "com.bakdata.kafka", name = "large-message-core" } 16 | errorHandling-bom = { group = "com.bakdata.kafka", name = "error-handling-bom", version = "1.8.0" } 17 | errorHandling-core = { group = "com.bakdata.kafka", name = "error-handling-core" } 18 | picocli = { group = "info.picocli", name = "picocli", version = "4.7.7" } 19 | slf4j = { group = "org.slf4j", name = "slf4j-api", version = "2.0.17" } 20 | jool = { group = "org.jooq", name = "jool", version = "0.9.15" } 21 | resilience4j-retry = { group = "io.github.resilience4j", name = "resilience4j-retry", version = "1.7.1" } 22 | 23 | junit-platform-launcher = { group = "org.junit.platform", name = "junit-platform-launcher" } 24 | junit-jupiter = { group = "org.junit.jupiter", name = "junit-jupiter", version.ref = "junit" } 25 | junit-pioneer = { group = "org.junit-pioneer", name = "junit-pioneer", version = "2.3.0" } 26 | junit-systemExit = { group = "com.ginsberg", name = "junit5-system-exit", version = "1.1.2" } 27 | assertj = { group = "org.assertj", name = "assertj-core", version = "3.27.3" } 28 | mockito-core = { group = "org.mockito", name = "mockito-core", version.ref = "mockito" } 29 | mockito-junit = { group = "org.mockito", name = "mockito-junit-jupiter", version.ref = "mockito" } 30 | testcontainers-junit = { group = "org.testcontainers", name = "junit-jupiter", version.ref = "testcontainers" } 31 | testcontainers-kafka = { group = "org.testcontainers", name = "kafka", version.ref = "testcontainers" } 32 | fluentKafkaStreamsTests = { group = "com.bakdata.fluent-kafka-streams-tests", name = "fluent-kafka-streams-tests-junit5", version = "3.3.0" } 33 | log4j-slf4j2 = { group = "org.apache.logging.log4j", name = "log4j-slf4j2-impl", version = "2.24.3" } 34 | awaitility = { group = "org.awaitility", name = "awaitility", version = "4.3.0" } 35 | 36 | [plugins] 37 | release = { id = "com.bakdata.release", version = "1.9.1" } 38 | sonar = { id = "com.bakdata.sonar", version = "1.9.1" } 39 | sonatype = { id = "com.bakdata.sonatype", version = "1.9.1" } 40 | lombok = { id = "io.freefair.lombok", version = "8.13.1" } 41 | avro = { id = "com.github.davidmc24.gradle.plugin.avro", version = "1.9.1" } 42 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bakdata/streams-bootstrap/4182f5aca93d986638edff9a310091af3566740d/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-all.zip 4 | networkTimeout=10000 5 | validateDistributionUrl=true 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | @rem SPDX-License-Identifier: Apache-2.0 17 | @rem 18 | 19 | @if "%DEBUG%"=="" @echo off 20 | @rem ########################################################################## 21 | @rem 22 | @rem Gradle startup script for Windows 23 | @rem 24 | @rem ########################################################################## 25 | 26 | @rem Set local scope for the variables with windows NT shell 27 | if "%OS%"=="Windows_NT" setlocal 28 | 29 | set DIRNAME=%~dp0 30 | if "%DIRNAME%"=="" set DIRNAME=. 31 | @rem This is normally unused 32 | set APP_BASE_NAME=%~n0 33 | set APP_HOME=%DIRNAME% 34 | 35 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 36 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 37 | 38 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 39 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 40 | 41 | @rem Find java.exe 42 | if defined JAVA_HOME goto findJavaFromJavaHome 43 | 44 | set JAVA_EXE=java.exe 45 | %JAVA_EXE% -version >NUL 2>&1 46 | if %ERRORLEVEL% equ 0 goto execute 47 | 48 | echo. 1>&2 49 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 50 | echo. 1>&2 51 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 52 | echo location of your Java installation. 1>&2 53 | 54 | goto fail 55 | 56 | :findJavaFromJavaHome 57 | set JAVA_HOME=%JAVA_HOME:"=% 58 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 59 | 60 | if exist "%JAVA_EXE%" goto execute 61 | 62 | echo. 1>&2 63 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 64 | echo. 1>&2 65 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 66 | echo location of your Java installation. 1>&2 67 | 68 | goto fail 69 | 70 | :execute 71 | @rem Setup the command line 72 | 73 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 74 | 75 | 76 | @rem Execute Gradle 77 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 78 | 79 | :end 80 | @rem End local scope for the variables with windows NT shell 81 | if %ERRORLEVEL% equ 0 goto mainEnd 82 | 83 | :fail 84 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 85 | rem the _cmd.exe /c_ return code! 86 | set EXIT_CODE=%ERRORLEVEL% 87 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 88 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 89 | exit /b %EXIT_CODE% 90 | 91 | :mainEnd 92 | if "%OS%"=="Windows_NT" endlocal 93 | 94 | :omega 95 | -------------------------------------------------------------------------------- /monitoring/pod_monitor.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: monitoring.coreos.com/v1 2 | kind: PodMonitor 3 | metadata: 4 | name: kafka-streams 5 | spec: 6 | selector: 7 | matchExpressions: 8 | - key: "streams-bootstrap/kind" 9 | operator: In 10 | values: 11 | - streams-app 12 | podMetricsEndpoints: 13 | - path: /metrics 14 | targetPort: 5556 15 | -------------------------------------------------------------------------------- /settings.gradle.kts: -------------------------------------------------------------------------------- 1 | pluginManagement { 2 | repositories { 3 | gradlePluginPortal() 4 | } 5 | } 6 | 7 | rootProject.name = "streams-bootstrap" 8 | 9 | include( 10 | ":streams-bootstrap-core", 11 | ":streams-bootstrap-test", 12 | ":streams-bootstrap-large-messages", 13 | ":streams-bootstrap-cli", 14 | ":streams-bootstrap-bom", 15 | ) 16 | -------------------------------------------------------------------------------- /streams-bootstrap-bom/build.gradle.kts: -------------------------------------------------------------------------------- 1 | description = "BOM for streams-bootstrap." 2 | 3 | plugins { 4 | id("java-platform") 5 | } 6 | 7 | dependencies { 8 | constraints { 9 | api(project(":streams-bootstrap-core")) 10 | api(project(":streams-bootstrap-cli")) 11 | api(project(":streams-bootstrap-large-messages")) 12 | api(project(":streams-bootstrap-test")) 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /streams-bootstrap-cli/build.gradle.kts: -------------------------------------------------------------------------------- 1 | description = "Base classes to create standalone Java applications using picocli" 2 | 3 | plugins { 4 | id("java-library") 5 | alias(libs.plugins.avro) 6 | } 7 | 8 | dependencies { 9 | api(project(":streams-bootstrap-core")) 10 | api(libs.picocli) 11 | implementation(libs.slf4j) 12 | 13 | testRuntimeOnly(libs.junit.platform.launcher) 14 | testImplementation(libs.junit.jupiter) 15 | testImplementation(libs.assertj) 16 | testImplementation(libs.mockito.core) 17 | testImplementation(libs.mockito.junit) 18 | testImplementation(testFixtures(project(":streams-bootstrap-test"))) 19 | testImplementation(libs.junit.systemExit) 20 | testImplementation(libs.kafka.streams.avro.serde) { 21 | exclude(group = "org.apache.kafka", module = "kafka-clients") // force usage of OSS kafka-clients 22 | } 23 | testImplementation(libs.log4j.slf4j2) 24 | } 25 | -------------------------------------------------------------------------------- /streams-bootstrap-cli/src/main/java/com/bakdata/kafka/SimpleKafkaProducerApplication.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.function.Supplier; 28 | import lombok.NonNull; 29 | import lombok.RequiredArgsConstructor; 30 | 31 | /** 32 | * {@code KafkaProducerApplication} without any additional configuration options. 33 | * 34 | * @param type of {@link ProducerApp} created by this application 35 | */ 36 | @RequiredArgsConstructor 37 | public final class SimpleKafkaProducerApplication extends KafkaProducerApplication { 38 | private final @NonNull Supplier appFactory; 39 | 40 | @Override 41 | public T createApp() { 42 | return this.appFactory.get(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /streams-bootstrap-cli/src/main/java/com/bakdata/kafka/SimpleKafkaStreamsApplication.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.function.Supplier; 28 | import lombok.NonNull; 29 | import lombok.RequiredArgsConstructor; 30 | 31 | /** 32 | * {@code KafkaStreamsApplication} without any additional configuration options. 33 | * 34 | * @param type of {@link StreamsApp} created by this application 35 | */ 36 | @RequiredArgsConstructor 37 | public final class SimpleKafkaStreamsApplication extends KafkaStreamsApplication { 38 | private final @NonNull Supplier appFactory; 39 | 40 | @Override 41 | public T createApp() { 42 | return this.appFactory.get(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /streams-bootstrap-cli/src/main/java/com/bakdata/kafka/StringListConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.Arrays; 28 | import java.util.List; 29 | import java.util.stream.Collectors; 30 | import picocli.CommandLine.ITypeConverter; 31 | 32 | /** 33 | * Converter for lists inside collection type parsed by PicoCLI. List members need to be separated by {@code ;} 34 | */ 35 | public class StringListConverter implements ITypeConverter> { 36 | 37 | @Override 38 | public List convert(final String value) { 39 | final String[] split = value.split(";"); 40 | return Arrays.stream(split) 41 | .map(String::trim) 42 | .filter(s -> !s.isEmpty()) 43 | .collect(Collectors.toList()); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /streams-bootstrap-cli/src/test/avro/TestRecord.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "namespace": "com.bakdata.kafka", 4 | "name": "TestRecord", 5 | "fields": [ 6 | { 7 | "name": "content", 8 | "type": "string" 9 | } 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /streams-bootstrap-cli/src/test/java/com/bakdata/kafka/CloseFlagApp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.Getter; 28 | import lombok.NoArgsConstructor; 29 | import lombok.Setter; 30 | import org.apache.kafka.common.serialization.Serdes.StringSerde; 31 | 32 | @NoArgsConstructor 33 | @Getter 34 | @Setter 35 | public class CloseFlagApp extends KafkaStreamsApplication { 36 | 37 | private boolean closed = false; 38 | private boolean appClosed = false; 39 | 40 | @Override 41 | public void close() { 42 | super.close(); 43 | this.closed = true; 44 | } 45 | 46 | @Override 47 | public StreamsApp createApp() { 48 | return new StreamsApp() { 49 | @Override 50 | public void buildTopology(final StreamsBuilderX builder) { 51 | final KStreamX input = builder.streamInput(); 52 | input.toOutputTopic(); 53 | } 54 | 55 | @Override 56 | public String getUniqueAppId(final StreamsTopicConfig topics) { 57 | return CloseFlagApp.this.getClass().getSimpleName() + "-" + topics.getOutputTopic(); 58 | } 59 | 60 | @Override 61 | public SerdeConfig defaultSerializationConfig() { 62 | return new SerdeConfig(StringSerde.class, StringSerde.class); 63 | } 64 | 65 | @Override 66 | public void close() { 67 | CloseFlagApp.this.appClosed = true; 68 | } 69 | }; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /streams-bootstrap-cli/src/test/java/com/bakdata/kafka/StringListConverterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static org.assertj.core.api.Assertions.assertThat; 28 | 29 | import org.junit.jupiter.api.Test; 30 | 31 | class StringListConverterTest { 32 | 33 | @Test 34 | void shouldConvertSingleElement() { 35 | assertThat(new StringListConverter().convert("foo")) 36 | .hasSize(1) 37 | .containsExactly("foo"); 38 | } 39 | 40 | @Test 41 | void shouldConvertEmpty() { 42 | assertThat(new StringListConverter().convert("")).isEmpty(); 43 | } 44 | 45 | @Test 46 | void shouldSplit() { 47 | assertThat(new StringListConverter().convert("foo;bar")) 48 | .hasSize(2) 49 | .containsExactly("foo", "bar"); 50 | } 51 | 52 | @Test 53 | void shouldTrim() { 54 | assertThat(new StringListConverter().convert("foo ; bar")) 55 | .hasSize(2) 56 | .containsExactly("foo", "bar"); 57 | } 58 | 59 | @Test 60 | void shouldIgnoreEmpty() { 61 | assertThat(new StringListConverter().convert("foo;;bar;")) 62 | .hasSize(2) 63 | .containsExactly("foo", "bar"); 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/Mirror.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka.test_applications; 26 | 27 | import com.bakdata.kafka.KStreamX; 28 | import com.bakdata.kafka.SerdeConfig; 29 | import com.bakdata.kafka.StreamsApp; 30 | import com.bakdata.kafka.StreamsBuilderX; 31 | import com.bakdata.kafka.StreamsTopicConfig; 32 | import lombok.NoArgsConstructor; 33 | import org.apache.kafka.common.serialization.Serdes.StringSerde; 34 | 35 | @NoArgsConstructor 36 | public class Mirror implements StreamsApp { 37 | @Override 38 | public void buildTopology(final StreamsBuilderX builder) { 39 | final KStreamX input = builder.streamInput(); 40 | input.toOutputTopic(); 41 | } 42 | 43 | @Override 44 | public String getUniqueAppId(final StreamsTopicConfig topics) { 45 | return this.getClass().getSimpleName() + "-" + topics.getOutputTopic(); 46 | } 47 | 48 | @Override 49 | public SerdeConfig defaultSerializationConfig() { 50 | return new SerdeConfig(StringSerde.class, StringSerde.class); 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/WordCount.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka.test_applications; 26 | 27 | import com.bakdata.kafka.KStreamX; 28 | import com.bakdata.kafka.KTableX; 29 | import com.bakdata.kafka.SerdeConfig; 30 | import com.bakdata.kafka.StreamsApp; 31 | import com.bakdata.kafka.StreamsTopicConfig; 32 | import com.bakdata.kafka.StreamsBuilderX; 33 | import java.util.Arrays; 34 | import java.util.regex.Pattern; 35 | import lombok.NoArgsConstructor; 36 | import org.apache.kafka.common.serialization.Serdes; 37 | import org.apache.kafka.common.serialization.Serdes.StringSerde; 38 | import org.apache.kafka.streams.kstream.Materialized; 39 | import org.apache.kafka.streams.kstream.Produced; 40 | 41 | @NoArgsConstructor 42 | public class WordCount implements StreamsApp { 43 | 44 | @Override 45 | public void buildTopology(final StreamsBuilderX builder) { 46 | final KStreamX textLines = builder.streamInput(); 47 | 48 | final Pattern pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS); 49 | final KTableX wordCounts = textLines 50 | .flatMapValues(value -> Arrays.asList(pattern.split(value.toLowerCase()))) 51 | .groupBy((key, word) -> word) 52 | .count(Materialized.as("counts")); 53 | 54 | wordCounts.toStream().toOutputTopic(Produced.valueSerde(Serdes.Long())); 55 | } 56 | 57 | @Override 58 | public String getUniqueAppId(final StreamsTopicConfig topics) { 59 | return this.getClass().getSimpleName() + "-" + topics.getOutputTopic(); 60 | } 61 | 62 | @Override 63 | public SerdeConfig defaultSerializationConfig() { 64 | return new SerdeConfig(StringSerde.class, StringSerde.class); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /streams-bootstrap-cli/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /streams-bootstrap-core/build.gradle.kts: -------------------------------------------------------------------------------- 1 | description = "Collection of commonly used modules when writing a Kafka Streams Application" 2 | 3 | plugins { 4 | id("java-library") 5 | alias(libs.plugins.avro) 6 | } 7 | 8 | dependencies { 9 | api(libs.kafka.streams.utils) 10 | implementation(libs.kafka.tools) { 11 | exclude(group = "org.slf4j", module = "slf4j-reload4j") 12 | } 13 | 14 | api(libs.kafka.streams) 15 | api(libs.kafka.clients) 16 | implementation(libs.kafka.schema.serializer) { 17 | exclude(group = "org.apache.kafka", module = "kafka-clients") // force usage of OSS kafka-clients 18 | exclude(group = "org.slf4j", module = "slf4j-api") // Conflict with 2.x when used as dependency 19 | } 20 | api(libs.kafka.schema.registry.client) { 21 | exclude(group = "org.apache.kafka", module = "kafka-clients") // force usage of OSS kafka-clients 22 | exclude(group = "org.slf4j", module = "slf4j-api") // Conflict with 2.x when used as dependency 23 | } 24 | implementation(libs.slf4j) 25 | implementation(libs.jool) 26 | implementation(libs.resilience4j.retry) 27 | api(platform(libs.errorHandling.bom)) 28 | api(libs.errorHandling.core) 29 | 30 | testRuntimeOnly(libs.junit.platform.launcher) 31 | testImplementation(libs.junit.jupiter) 32 | testImplementation(libs.junit.pioneer) 33 | testImplementation(libs.assertj) 34 | testImplementation(libs.mockito.core) 35 | testImplementation(libs.mockito.junit) 36 | 37 | testImplementation(testFixtures(project(":streams-bootstrap-test"))) 38 | testImplementation(libs.kafka.streams.avro.serde) { 39 | exclude(group = "org.apache.kafka", module = "kafka-clients") // force usage of OSS kafka-clients 40 | } 41 | testImplementation(libs.log4j.slf4j2) 42 | } 43 | 44 | tasks.withType { 45 | jvmArgs( 46 | "--add-opens=java.base/java.lang=ALL-UNNAMED", 47 | "--add-opens=java.base/java.util=ALL-UNNAMED" 48 | ) 49 | maxHeapSize = "4g" 50 | } 51 | -------------------------------------------------------------------------------- /streams-bootstrap-core/lombok.config: -------------------------------------------------------------------------------- 1 | # This file is generated by the 'io.freefair.lombok' Gradle plugin 2 | config.stopBubbling = true 3 | lombok.addLombokGeneratedAnnotation = true 4 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/App.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static java.util.Collections.emptyMap; 28 | 29 | import java.util.Map; 30 | 31 | /** 32 | * Kafka application that defines necessary configurations 33 | * @param type of topic config 34 | * @param type of clean up config 35 | */ 36 | public interface App extends AutoCloseable { 37 | 38 | /** 39 | * Configure clean up behavior 40 | * @param configuration provides all runtime application configurations 41 | * @return clean up configuration 42 | */ 43 | C setupCleanUp(final EffectiveAppConfiguration configuration); 44 | 45 | @Override 46 | default void close() { 47 | // do nothing by default 48 | } 49 | 50 | /** 51 | * This method should give a default configuration to run your application with. 52 | * @return Returns a default Kafka configuration. Empty by default 53 | */ 54 | default Map createKafkaProperties() { 55 | return emptyMap(); 56 | } 57 | 58 | /** 59 | * Setup Kafka resources, such as topics, before running this app 60 | * @param configuration provides all runtime application configurations 61 | */ 62 | default void setup(final EffectiveAppConfiguration configuration) { 63 | // do nothing by default 64 | } 65 | 66 | /** 67 | * Configure default serialization behavior 68 | * @return {@code SerializationConfig} 69 | */ 70 | SerializationConfig defaultSerializationConfig(); 71 | } 72 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/AppConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static java.util.Collections.emptyMap; 28 | 29 | import java.util.Map; 30 | import lombok.EqualsAndHashCode; 31 | import lombok.NonNull; 32 | import lombok.RequiredArgsConstructor; 33 | import lombok.Value; 34 | 35 | /** 36 | * Configuration of an app. This includes topics and Kafka configuration 37 | * @param type of topic config 38 | */ 39 | @Value 40 | @RequiredArgsConstructor 41 | @EqualsAndHashCode 42 | public class AppConfiguration { 43 | @NonNull 44 | T topics; 45 | @NonNull 46 | Map kafkaConfig; 47 | 48 | /** 49 | * Create a new {@code AppConfiguration} with empty Kafka configuration 50 | * @param topics topics to use for app 51 | */ 52 | public AppConfiguration(final T topics) { 53 | this(topics, emptyMap()); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/CapturingStreamsUncaughtExceptionHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.NonNull; 28 | import lombok.RequiredArgsConstructor; 29 | import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler; 30 | 31 | @RequiredArgsConstructor 32 | class CapturingStreamsUncaughtExceptionHandler implements StreamsUncaughtExceptionHandler { 33 | 34 | private final @NonNull StreamsUncaughtExceptionHandler wrapped; 35 | private Throwable lastException; 36 | 37 | @Override 38 | public StreamThreadExceptionResponse handle(final Throwable exception) { 39 | final StreamThreadExceptionResponse response = this.wrapped.handle(exception); 40 | this.lastException = exception; 41 | return response; 42 | } 43 | 44 | void throwException() { 45 | if (this.lastException instanceof RuntimeException) { 46 | throw (RuntimeException) this.lastException; 47 | } 48 | throw new StreamsApplicationException("Kafka Streams has transitioned to error", this.lastException); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/CleanUpException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | /** 28 | * Exception thrown if running clean up was unsuccessful 29 | */ 30 | public class CleanUpException extends RuntimeException { 31 | public CleanUpException(final String message) { 32 | super(message); 33 | } 34 | 35 | public CleanUpException(final String message, final Throwable cause) { 36 | super(message, cause); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/CleanUpRunner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | /** 28 | * Cleans all resources associated with an application 29 | */ 30 | public interface CleanUpRunner extends AutoCloseable { 31 | 32 | @Override 33 | void close(); 34 | 35 | /** 36 | * Clean all resources associated with an application 37 | */ 38 | void clean(); 39 | } 40 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredApp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | /** 28 | * An application with a corresponding configuration 29 | * 30 | * @param type of executable app after configuring {@link KafkaEndpointConfig} 31 | */ 32 | public interface ConfiguredApp extends AutoCloseable { 33 | /** 34 | * Create an executable app using the provided {@code KafkaEndpointConfig} 35 | * @param endpointConfig endpoint to run app on 36 | * @return executable streams app 37 | */ 38 | E withEndpoint(KafkaEndpointConfig endpointConfig); 39 | 40 | @Override 41 | void close(); 42 | } 43 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/DefaultStreamsUncaughtExceptionHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler; 28 | 29 | /** 30 | * {@code StreamsUncaughtExceptionHandler} that does not handle the exception and responds with 31 | * {@link org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse#SHUTDOWN_CLIENT}. Mimics default behavior of {@link org.apache.kafka.streams.KafkaStreams} if no {@code StreamsUncaughtExceptionHandler} has been configured. 32 | * @see org.apache.kafka.streams.KafkaStreams#setUncaughtExceptionHandler(StreamsUncaughtExceptionHandler) 33 | */ 34 | class DefaultStreamsUncaughtExceptionHandler implements StreamsUncaughtExceptionHandler { 35 | @Override 36 | public StreamThreadExceptionResponse handle(final Throwable e) { 37 | return StreamThreadExceptionResponse.SHUTDOWN_CLIENT; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/EffectiveAppConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import com.bakdata.kafka.util.ImprovedAdminClient; 28 | import java.util.Map; 29 | import lombok.EqualsAndHashCode; 30 | import lombok.NonNull; 31 | import lombok.Value; 32 | 33 | /** 34 | * Configuration for setting up an app 35 | * @param type of topic config 36 | * @see StreamsApp#setup(EffectiveAppConfiguration) 37 | * @see StreamsApp#setupCleanUp(EffectiveAppConfiguration) 38 | * @see ProducerApp#setup(EffectiveAppConfiguration) 39 | * @see ProducerApp#setupCleanUp(EffectiveAppConfiguration) 40 | */ 41 | @Value 42 | @EqualsAndHashCode 43 | public class EffectiveAppConfiguration { 44 | @NonNull 45 | T topics; 46 | @NonNull 47 | Map kafkaProperties; 48 | 49 | /** 50 | * Create a new {@code ImprovedAdminClient} using {@link #kafkaProperties} 51 | * @return {@code ImprovedAdminClient} 52 | */ 53 | public ImprovedAdminClient createAdminClient() { 54 | return ImprovedAdminClient.create(this.kafkaProperties); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/EnvironmentKafkaConfigParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.Locale; 28 | import java.util.Map; 29 | import java.util.Map.Entry; 30 | import java.util.regex.Pattern; 31 | import java.util.stream.Collectors; 32 | 33 | /** 34 | * Parse configuration properties of a Kafka app from environment variables 35 | */ 36 | public final class EnvironmentKafkaConfigParser { 37 | 38 | static final String PREFIX = "KAFKA_"; 39 | private static final Pattern UNDERSCORE = Pattern.compile("_"); 40 | private static final Pattern PREFIX_PATTERN = Pattern.compile("^" + PREFIX); 41 | 42 | private EnvironmentKafkaConfigParser() { 43 | throw new UnsupportedOperationException("Utility class"); 44 | } 45 | 46 | /** 47 | * Parse a list of environment variables as a streams configuration. All variables starting with {@code KAFKA_} 48 | * prefix are converted. {@code _} are replaced by {@code .} 49 | * 50 | * @param environment map of environment variables 51 | * @return parsed streams configuration 52 | */ 53 | public static Map parseVariables(final Map environment) { 54 | return environment.entrySet().stream() 55 | .filter(e -> e.getKey().startsWith(PREFIX)) 56 | .collect(Collectors.toMap(EnvironmentKafkaConfigParser::convertEnvironmentVariable, Entry::getValue)); 57 | } 58 | 59 | private static String convertEnvironmentVariable(final Entry environmentEntry) { 60 | final String key = environmentEntry.getKey(); 61 | final String withoutPrefix = PREFIX_PATTERN.matcher(key).replaceAll(""); 62 | return UNDERSCORE.matcher(withoutPrefix).replaceAll(".") 63 | .toLowerCase(Locale.getDefault()); 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/ExecutableApp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | /** 28 | * An application with a corresponding topic and Kafka configuration 29 | * @param type returned by {@link #createRunner()} and {@link #createRunner(Object)} 30 | * @param type returned by {@link #createCleanUpRunner()} 31 | * @param type of options to create runner 32 | */ 33 | public interface ExecutableApp extends AutoCloseable { 34 | 35 | @Override 36 | void close(); 37 | 38 | /** 39 | * Create {@code Runner} in order to run application with default options 40 | * @return {@code Runner} 41 | */ 42 | R createRunner(); 43 | 44 | /** 45 | * Create {@code Runner} in order to run application 46 | * @param options options for creating runner 47 | * @return {@code Runner} 48 | */ 49 | R createRunner(O options); 50 | 51 | /** 52 | * Create {@code CleanUpRunner} in order to clean application 53 | * @return {@code CleanUpRunner} 54 | */ 55 | C createCleanUpRunner(); 56 | } 57 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/HasCleanHook.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | /** 28 | * Interface for performing actions when cleaning apps 29 | * @param self for chaining 30 | */ 31 | @FunctionalInterface 32 | public interface HasCleanHook { 33 | /** 34 | * Register a hook that is invoked when cleaning apps 35 | * @param hook factory to create hook from 36 | * @return self for chaining 37 | */ 38 | SELF registerCleanHook(Runnable hook); 39 | 40 | } 41 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/HasTopicHooks.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | /** 28 | * Interface for performing actions on topics 29 | * @param self for chaining 30 | */ 31 | @FunctionalInterface 32 | public interface HasTopicHooks { 33 | /** 34 | * Register a hook that is invoked when performing actions on topics 35 | * 36 | * @param hook Action to run. Topic is passed as parameter 37 | * @return self for chaining 38 | */ 39 | SELF registerTopicHook(TopicHook hook); 40 | 41 | /** 42 | * Hook for performing actions on topics 43 | */ 44 | interface TopicHook extends AutoCloseable { 45 | /** 46 | * Called when a topic is deleted 47 | * @param topic name of the topic 48 | */ 49 | default void deleted(final String topic) { 50 | // do nothing 51 | } 52 | 53 | @Override 54 | default void close() { 55 | // do nothing 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/KErrorStreamX.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import org.apache.kafka.streams.kstream.Named; 28 | 29 | /** 30 | * {@link KStreamX} that contains successfully processed records and errors of a previous operation 31 | * @param type of keys in the original {@link KStreamX} 32 | * @param type of values in the original {@link KStreamX} 33 | * @param type of keys in the processed {@link KStreamX} 34 | * @param type of values in the processed {@link KStreamX} 35 | */ 36 | public interface KErrorStreamX { 37 | 38 | /** 39 | * Get the stream of successfully processed values 40 | * @return stream of processed values 41 | */ 42 | KStreamX values(); 43 | 44 | /** 45 | * Get the stream of successfully processed values 46 | * @param named name of the processor 47 | * @return stream of processed values 48 | */ 49 | KStreamX values(Named named); 50 | 51 | /** 52 | * Get the stream of errors that occurred during processing 53 | * @return stream of errors 54 | */ 55 | KStreamX> errors(); 56 | 57 | /** 58 | * Get the stream of errors that occurred during processing 59 | * @param named name of the processor 60 | * @return stream of errors 61 | */ 62 | KStreamX> errors(Named named); 63 | } 64 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/KafkaEndpointConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; 28 | import java.util.Collections; 29 | import java.util.HashMap; 30 | import java.util.Map; 31 | import lombok.Builder; 32 | import lombok.NonNull; 33 | import org.apache.kafka.streams.StreamsConfig; 34 | 35 | /** 36 | * Configuration to connect to Kafka infrastructure, i.e., bootstrap servers and optionally schema registry. 37 | */ 38 | @Builder 39 | public class KafkaEndpointConfig { 40 | private final @NonNull String bootstrapServers; 41 | private final String schemaRegistryUrl; 42 | 43 | /** 44 | * Create Kafka properties to connect to infrastructure. 45 | * The following properties are configured: 46 | *
    47 | *
  • {@code bootstrap.servers}
    • 48 | *
  • {@code schema.registry.url}
    • 49 | *
50 | * @return properties used for connecting to Kafka 51 | */ 52 | public Map createKafkaProperties() { 53 | final Map kafkaConfig = new HashMap<>(); 54 | kafkaConfig.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, this.bootstrapServers); 55 | if (this.schemaRegistryUrl != null) { 56 | kafkaConfig.put(AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, this.schemaRegistryUrl); 57 | } 58 | return Collections.unmodifiableMap(kafkaConfig); 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/KeyValueKErrorStreamX.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.NonNull; 28 | import lombok.RequiredArgsConstructor; 29 | import org.apache.kafka.streams.kstream.Named; 30 | 31 | @RequiredArgsConstructor 32 | class KeyValueKErrorStreamX implements KErrorStreamX { 33 | private final @NonNull KStreamX> stream; 34 | 35 | @Override 36 | public KStreamX values() { 37 | return this.stream.flatMapValues(ProcessedKeyValue::getValues); 38 | } 39 | 40 | @Override 41 | public KStreamX values(final Named named) { 42 | return this.stream.flatMapValues(ProcessedKeyValue::getValues, named); 43 | } 44 | 45 | @Override 46 | public KStreamX> errors() { 47 | return this.stream.flatMap(ProcessedKeyValue::getErrors); 48 | } 49 | 50 | @Override 51 | public KStreamX> errors(final Named named) { 52 | return this.stream.flatMap(ProcessedKeyValue::getErrors, named); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/ModifierChain.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.function.BiFunction; 28 | import java.util.function.Function; 29 | import lombok.AccessLevel; 30 | import lombok.NonNull; 31 | import lombok.RequiredArgsConstructor; 32 | 33 | @RequiredArgsConstructor(access = AccessLevel.PROTECTED) 34 | abstract class ModifierChain { 35 | 36 | private final @NonNull Function initializer; 37 | 38 | protected abstract SELF newInstance(Function initializer); 39 | 40 | protected final SELF modify(final BiFunction modifier) { 41 | return this.newInstance(context -> this.modify(context, modifier)); 42 | } 43 | 44 | protected final SELF modify(final Function modifier) { 45 | return this.newInstance(context -> this.modify(context, modifier)); 46 | } 47 | 48 | final T configure(final C context) { 49 | return this.initializer.apply(context); 50 | } 51 | 52 | private T modify(final C context, final BiFunction modifier) { 53 | final T configure = this.configure(context); 54 | return modifier.apply(configure, context); 55 | } 56 | 57 | private T modify(final C context, final Function modifier) { 58 | final T configure = this.configure(context); 59 | return modifier.apply(configure); 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/NoOpStateListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import org.apache.kafka.streams.KafkaStreams.State; 28 | import org.apache.kafka.streams.KafkaStreams.StateListener; 29 | 30 | /** 31 | * {@code StateListener} that does nothing. 32 | * @see org.apache.kafka.streams.KafkaStreams#setStateListener(StateListener) 33 | */ 34 | class NoOpStateListener implements StateListener { 35 | @Override 36 | public void onChange(final State newState, final State oldState) { 37 | // do nothing 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/ProducerApp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | /** 28 | * Application that defines how to produce messages to Kafka and necessary configurations 29 | */ 30 | public interface ProducerApp extends App { 31 | 32 | /** 33 | * Create a runnable that produces Kafka messages 34 | * @param builder provides all runtime application configurations 35 | * @return {@code ProducerRunnable} 36 | */ 37 | ProducerRunnable buildRunnable(ProducerBuilder builder); 38 | 39 | /** 40 | * @return {@code ProducerCleanUpConfiguration} 41 | * @see ProducerCleanUpRunner 42 | */ 43 | @Override 44 | default ProducerCleanUpConfiguration setupCleanUp( 45 | final EffectiveAppConfiguration configuration) { 46 | return new ProducerCleanUpConfiguration(); 47 | } 48 | 49 | @Override 50 | SerializerConfig defaultSerializationConfig(); 51 | } 52 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/ProducerCleanUpConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.ArrayList; 28 | import java.util.Collection; 29 | import lombok.NonNull; 30 | 31 | /** 32 | * Provides configuration options for {@link ProducerCleanUpRunner} 33 | */ 34 | public class ProducerCleanUpConfiguration 35 | implements HasTopicHooks, HasCleanHook, 36 | AutoCloseable { 37 | private final @NonNull Collection topicHooks = new ArrayList<>(); 38 | private final @NonNull Collection cleanHooks = new ArrayList<>(); 39 | 40 | /** 41 | * Register a hook that is executed whenever a topic has been deleted by the cleanup runner. 42 | */ 43 | @Override 44 | public ProducerCleanUpConfiguration registerTopicHook(final TopicHook hook) { 45 | this.topicHooks.add(hook); 46 | return this; 47 | } 48 | 49 | /** 50 | * Register an action that is executed after {@link ProducerCleanUpRunner#clean()} has finished 51 | */ 52 | @Override 53 | public ProducerCleanUpConfiguration registerCleanHook(final Runnable hook) { 54 | this.cleanHooks.add(hook); 55 | return this; 56 | } 57 | 58 | @Override 59 | public void close() { 60 | this.topicHooks.forEach(TopicHook::close); 61 | } 62 | 63 | void runCleanHooks() { 64 | this.cleanHooks.forEach(Runnable::run); 65 | } 66 | 67 | void runTopicDeletionHooks(final String topic) { 68 | this.topicHooks.forEach(hook -> hook.deleted(topic)); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/ProducerExecutionOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.Builder; 28 | 29 | /** 30 | * Options to run a Kafka Producer app using {@link ProducerRunner} 31 | */ 32 | @Builder 33 | public final class ProducerExecutionOptions { 34 | } 35 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/ProducerRunnable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | /** 28 | * Produce messages to Kafka 29 | */ 30 | @FunctionalInterface 31 | public interface ProducerRunnable extends AutoCloseable { 32 | 33 | /** 34 | * Produce messages to Kafka 35 | */ 36 | void run(); 37 | 38 | @Override 39 | default void close() { 40 | // do nothing by default 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/ProducerRunner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.NonNull; 28 | import lombok.RequiredArgsConstructor; 29 | import lombok.extern.slf4j.Slf4j; 30 | 31 | /** 32 | * Runs a Kafka Producer application 33 | */ 34 | @RequiredArgsConstructor 35 | @Slf4j 36 | public class ProducerRunner implements Runner { 37 | 38 | private final @NonNull ProducerRunnable runnable; 39 | 40 | @Override 41 | public void close() { 42 | log.info("Closing producer"); 43 | this.runnable.close(); 44 | } 45 | 46 | @Override 47 | public void run() { 48 | log.info("Starting producer"); 49 | this.runnable.run(); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/ProducerTopicConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static java.util.Collections.emptyMap; 28 | 29 | import java.util.Map; 30 | import lombok.Builder; 31 | import lombok.EqualsAndHashCode; 32 | import lombok.NonNull; 33 | import lombok.Value; 34 | 35 | /** 36 | * Provides topic configuration for a {@link ProducerApp} 37 | */ 38 | @Builder 39 | @Value 40 | @EqualsAndHashCode 41 | public class ProducerTopicConfig { 42 | 43 | String outputTopic; 44 | /** 45 | * Output topics that are identified by a label 46 | */ 47 | @Builder.Default 48 | @NonNull 49 | Map labeledOutputTopics = emptyMap(); 50 | 51 | /** 52 | * Get output topic for a specified label 53 | * 54 | * @param label label of output topic 55 | * @return topic name 56 | */ 57 | public String getOutputTopic(final String label) { 58 | final String topic = this.labeledOutputTopics.get(label); 59 | if (topic == null) { 60 | throw new IllegalArgumentException(String.format("No output topic for label '%s' available", label)); 61 | } 62 | return topic; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/Runner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | /** 28 | * Interface for running applications 29 | */ 30 | public interface Runner extends AutoCloseable, Runnable { 31 | 32 | @Override 33 | void close(); 34 | 35 | /** 36 | * Run the application 37 | */ 38 | @Override 39 | void run(); 40 | } 41 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/RunningStreams.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import com.bakdata.kafka.StreamsExecutionOptions.StreamsExecutionOptionsBuilder; 28 | import java.util.function.Consumer; 29 | import lombok.Builder; 30 | import lombok.NonNull; 31 | import lombok.Value; 32 | import org.apache.kafka.streams.KafkaStreams; 33 | import org.apache.kafka.streams.StreamsConfig; 34 | import org.apache.kafka.streams.Topology; 35 | 36 | /** 37 | * A running {@link KafkaStreams} instance along with its {@link StreamsConfig} and 38 | * {@link org.apache.kafka.streams.Topology} 39 | * 40 | * @see StreamsExecutionOptionsBuilder#onStart(Consumer) 41 | */ 42 | @Builder 43 | @Value 44 | public class RunningStreams { 45 | 46 | @NonNull 47 | ImprovedStreamsConfig config; 48 | @NonNull 49 | Topology topology; 50 | @NonNull 51 | KafkaStreams streams; 52 | } 53 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/SerdeConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.Map; 28 | import lombok.NonNull; 29 | import lombok.RequiredArgsConstructor; 30 | import lombok.With; 31 | import org.apache.kafka.common.serialization.Serde; 32 | import org.apache.kafka.streams.StreamsConfig; 33 | 34 | /** 35 | * Defines how to (de-)serialize the data in a Kafka Streams app 36 | */ 37 | @RequiredArgsConstructor 38 | @With 39 | public class SerdeConfig implements SerializationConfig { 40 | 41 | private final @NonNull Class keySerde; 42 | private final @NonNull Class valueSerde; 43 | 44 | @Override 45 | public Map createProperties() { 46 | return Map.of( 47 | StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, this.keySerde, 48 | StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, this.valueSerde 49 | ); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/SerializationConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.Map; 28 | 29 | /** 30 | * Defines how to (de-)serialize the data in a Kafka client 31 | */ 32 | @FunctionalInterface 33 | public interface SerializationConfig { 34 | 35 | /** 36 | * Create properties from this {@code SerializationConfig} 37 | * @return Map of serialization configurations 38 | */ 39 | Map createProperties(); 40 | } 41 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/SerializerConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.Map; 28 | import lombok.NonNull; 29 | import lombok.RequiredArgsConstructor; 30 | import lombok.With; 31 | import org.apache.kafka.clients.producer.ProducerConfig; 32 | import org.apache.kafka.common.serialization.Serializer; 33 | 34 | /** 35 | * Defines how to serialize the data in a Kafka producer 36 | */ 37 | @RequiredArgsConstructor 38 | @With 39 | public class SerializerConfig implements SerializationConfig { 40 | 41 | private final @NonNull Class keySerializer; 42 | private final @NonNull Class valueSerializer; 43 | 44 | @Override 45 | public Map createProperties() { 46 | return Map.of( 47 | ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, this.keySerializer, 48 | ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, this.valueSerializer 49 | ); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsApp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | /** 28 | * Application that defines a Kafka Streams {@link org.apache.kafka.streams.Topology} and necessary configurations 29 | */ 30 | public interface StreamsApp extends App { 31 | 32 | /** 33 | * Build the Kafka Streams {@link org.apache.kafka.streams.Topology} to be run by the app. 34 | * 35 | * @param builder provides all runtime application configurations and supports building the 36 | * {@link org.apache.kafka.streams.Topology} 37 | */ 38 | void buildTopology(StreamsBuilderX builder); 39 | 40 | /** 41 | * This must be set to a unique value for every application interacting with your Kafka cluster to ensure internal 42 | * state encapsulation. Could be set to: className-outputTopic 43 | * 44 | * @param topics provides runtime topic configuration 45 | * @return unique application identifier 46 | */ 47 | String getUniqueAppId(StreamsTopicConfig topics); 48 | 49 | /** 50 | * @return {@code StreamsCleanUpConfiguration} 51 | * @see StreamsCleanUpRunner 52 | */ 53 | @Override 54 | default StreamsCleanUpConfiguration setupCleanUp( 55 | final EffectiveAppConfiguration configuration) { 56 | return new StreamsCleanUpConfiguration(); 57 | } 58 | 59 | @Override 60 | SerdeConfig defaultSerializationConfig(); 61 | } 62 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsApplicationException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | /** 28 | * Exception thrown if running streams application was unsuccessful 29 | */ 30 | public class StreamsApplicationException extends RuntimeException { 31 | public StreamsApplicationException(final String message) { 32 | super(message); 33 | } 34 | 35 | public StreamsApplicationException(final String message, final Throwable cause) { 36 | super(message, cause); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsCleanUpConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.ArrayList; 28 | import java.util.Collection; 29 | import lombok.NonNull; 30 | 31 | /** 32 | * Provides configuration options for {@link StreamsCleanUpRunner} 33 | */ 34 | public class StreamsCleanUpConfiguration 35 | implements HasTopicHooks, HasCleanHook, 36 | AutoCloseable { 37 | private final @NonNull Collection topicHooks = new ArrayList<>(); 38 | private final @NonNull Collection cleanHooks = new ArrayList<>(); 39 | private final @NonNull Collection resetHooks = new ArrayList<>(); 40 | 41 | /** 42 | * Register a hook that is executed whenever a topic has been deleted by the cleanup runner. 43 | */ 44 | @Override 45 | public StreamsCleanUpConfiguration registerTopicHook(final TopicHook hook) { 46 | this.topicHooks.add(hook); 47 | return this; 48 | } 49 | 50 | /** 51 | * Register a hook that is executed after {@link StreamsCleanUpRunner#clean()} has finished 52 | */ 53 | @Override 54 | public StreamsCleanUpConfiguration registerCleanHook(final Runnable hook) { 55 | this.cleanHooks.add(hook); 56 | return this; 57 | } 58 | 59 | /** 60 | * Register a hook that is executed after {@link StreamsCleanUpRunner#reset()} has finished 61 | * @param hook factory to create hook from 62 | * @return self for chaining 63 | */ 64 | public StreamsCleanUpConfiguration registerResetHook(final Runnable hook) { 65 | this.resetHooks.add(hook); 66 | return this; 67 | } 68 | 69 | @Override 70 | public void close() { 71 | this.topicHooks.forEach(TopicHook::close); 72 | } 73 | 74 | void runCleanHooks() { 75 | this.cleanHooks.forEach(Runnable::run); 76 | } 77 | 78 | void runResetHooks() { 79 | this.resetHooks.forEach(Runnable::run); 80 | } 81 | 82 | void runTopicDeletionHooks(final String topic) { 83 | this.topicHooks.forEach(hook -> hook.deleted(topic)); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsShutdownStateListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import java.util.concurrent.CountDownLatch; 28 | import lombok.NonNull; 29 | import lombok.RequiredArgsConstructor; 30 | import org.apache.kafka.streams.KafkaStreams.State; 31 | import org.apache.kafka.streams.KafkaStreams.StateListener; 32 | 33 | @RequiredArgsConstructor 34 | class StreamsShutdownStateListener implements StateListener { 35 | 36 | private final CountDownLatch streamsShutdown = new CountDownLatch(1); 37 | private @NonNull StateListener wrapped; 38 | 39 | @Override 40 | public void onChange(final State newState, final State oldState) { 41 | this.wrapped.onChange(newState, oldState); 42 | if (newState.hasCompletedShutdown()) { 43 | this.streamsShutdown.countDown(); 44 | } 45 | } 46 | 47 | void await() throws InterruptedException { 48 | this.streamsShutdown.await(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamX.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import org.apache.kafka.common.utils.Bytes; 28 | import org.apache.kafka.streams.kstream.Initializer; 29 | import org.apache.kafka.streams.kstream.Materialized; 30 | import org.apache.kafka.streams.kstream.Named; 31 | import org.apache.kafka.streams.kstream.TimeWindowedCogroupedKStream; 32 | import org.apache.kafka.streams.kstream.Windowed; 33 | import org.apache.kafka.streams.state.WindowStore; 34 | 35 | /** 36 | * Extends the {@link TimeWindowedCogroupedKStream} interface by adding methods to simplify Serde configuration, 37 | * error handling, and topic access 38 | * @param type of keys 39 | * @param type of values 40 | */ 41 | public interface TimeWindowedCogroupedKStreamX extends TimeWindowedCogroupedKStream { 42 | 43 | @Override 44 | KTableX, VOut> aggregate(Initializer initializer); 45 | 46 | @Override 47 | KTableX, VOut> aggregate(Initializer initializer, Named named); 48 | 49 | @Override 50 | KTableX, VOut> aggregate(Initializer initializer, 51 | Materialized> materialized); 52 | 53 | /** 54 | * @see #aggregate(Initializer, Materialized) 55 | */ 56 | KTableX, VOut> aggregate(Initializer initializer, 57 | MaterializedX> materialized); 58 | 59 | @Override 60 | KTableX, VOut> aggregate(Initializer initializer, Named named, 61 | Materialized> materialized); 62 | 63 | /** 64 | * @see #aggregate(Initializer, Named, Materialized) 65 | */ 66 | KTableX, VOut> aggregate(Initializer initializer, Named named, 67 | MaterializedX> materialized); 68 | } 69 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/ValueKErrorStreamX.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import lombok.NonNull; 28 | import lombok.RequiredArgsConstructor; 29 | import org.apache.kafka.streams.kstream.Named; 30 | 31 | @RequiredArgsConstructor 32 | class ValueKErrorStreamX implements KErrorStreamX { 33 | private final @NonNull KStreamX> stream; 34 | 35 | @Override 36 | public KStreamX values() { 37 | return this.stream.flatMapValues(ProcessedValue::getValues); 38 | } 39 | 40 | @Override 41 | public KStreamX values(final Named named) { 42 | return this.stream.flatMapValues(ProcessedValue::getValues, named); 43 | } 44 | 45 | @Override 46 | public KStreamX> errors() { 47 | return this.stream.flatMapValues(ProcessedValue::getErrors); 48 | } 49 | 50 | @Override 51 | public KStreamX> errors(final Named named) { 52 | return this.stream.flatMapValues(ProcessedValue::getErrors, named); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/KafkaAdminException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka.util; 26 | 27 | /** 28 | * Exception thrown by {@link TopicClient}. 29 | */ 30 | public class KafkaAdminException extends RuntimeException { 31 | 32 | KafkaAdminException(final String message, final Throwable cause) { 33 | super(message, cause); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopicSettings.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka.util; 26 | 27 | import lombok.Builder; 28 | import lombok.Value; 29 | 30 | /** 31 | * Contains information about a topic, including number of partitions and replication factor. 32 | */ 33 | @Value 34 | @Builder 35 | public class TopicSettings { 36 | int partitions; 37 | short replicationFactor; 38 | } 39 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/avro/TestRecord.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "namespace": "com.bakdata.kafka", 4 | "name": "TestRecord", 5 | "fields": [ 6 | { 7 | "name": "content", 8 | "type": "string" 9 | } 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/AvroMirrorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import com.bakdata.fluent_kafka_streams_tests.junit5.TestTopologyExtension; 28 | import com.bakdata.kafka.test_applications.MirrorWithNonDefaultSerde; 29 | import java.util.List; 30 | import org.junit.jupiter.api.Test; 31 | import org.junit.jupiter.api.extension.RegisterExtension; 32 | 33 | class AvroMirrorTest { 34 | private final ConfiguredStreamsApp app = createApp(); 35 | @RegisterExtension 36 | final TestTopologyExtension testTopology = 37 | TestTopologyFactory.withSchemaRegistry().createTopologyExtension(this.app); 38 | 39 | private static ConfiguredStreamsApp createApp() { 40 | final AppConfiguration configuration = new AppConfiguration<>(StreamsTopicConfig.builder() 41 | .inputTopics(List.of("input")) 42 | .outputTopic("output") 43 | .build()); 44 | return new ConfiguredStreamsApp<>(new MirrorWithNonDefaultSerde(), configuration); 45 | } 46 | 47 | @Test 48 | void shouldMirror() { 49 | final TestRecord testRecord = TestRecord.newBuilder() 50 | .setContent("bar") 51 | .build(); 52 | this.testTopology.input() 53 | .configureWithKeySerde(MirrorWithNonDefaultSerde.newKeySerde()) 54 | .configureWithValueSerde(MirrorWithNonDefaultSerde.newValueSerde()) 55 | .add(testRecord, testRecord); 56 | 57 | this.testTopology.streamOutput() 58 | .configureWithKeySerde(MirrorWithNonDefaultSerde.newKeySerde()) 59 | .configureWithValueSerde(MirrorWithNonDefaultSerde.newValueSerde()) 60 | .expectNextRecord().hasKey(testRecord).hasValue(testRecord) 61 | .expectNoMoreRecord(); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/DoubleApp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import com.bakdata.fluent_kafka_streams_tests.TestTopology; 28 | import org.apache.kafka.common.serialization.Serdes.DoubleSerde; 29 | 30 | abstract class DoubleApp implements StreamsApp { 31 | 32 | @Override 33 | public String getUniqueAppId(final StreamsTopicConfig topics) { 34 | return "my-app"; 35 | } 36 | 37 | @Override 38 | public SerdeConfig defaultSerializationConfig() { 39 | return new SerdeConfig(DoubleSerde.class, DoubleSerde.class); 40 | } 41 | 42 | TestTopology startApp(final StreamsTopicConfig topicConfig) { 43 | final ConfiguredStreamsApp configuredApp = TestHelper.configureApp(this, topicConfig); 44 | return TestHelper.startApp(configuredApp); 45 | } 46 | 47 | TestTopology startApp() { 48 | return this.startApp(StreamsTopicConfig.builder().build()); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/EnvironmentKafkaConfigParserTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static org.assertj.core.api.Assertions.assertThat; 28 | 29 | import java.util.Map; 30 | import org.junit.jupiter.api.Test; 31 | 32 | class EnvironmentKafkaConfigParserTest { 33 | 34 | @Test 35 | void shouldParseStreamsConfig() { 36 | assertThat(EnvironmentKafkaConfigParser.parseVariables(Map.of( 37 | "KAFKA_FOO", "bar", 38 | "KAFKA_BAZ", "qux" 39 | ))) 40 | .hasSize(2) 41 | .containsEntry("foo", "bar") 42 | .containsEntry("baz", "qux"); 43 | } 44 | 45 | @Test 46 | void shouldIgnoreVariablesWithoutPrefix() { 47 | assertThat(EnvironmentKafkaConfigParser.parseVariables(Map.of( 48 | "APP_FOO", "bar" 49 | ))).isEmpty(); 50 | } 51 | 52 | @Test 53 | void shouldConvertUnderscores() { 54 | assertThat(EnvironmentKafkaConfigParser.parseVariables(Map.of( 55 | "KAFKA_FOO_BAR", "baz" 56 | ))) 57 | .hasSize(1) 58 | .containsEntry("foo.bar", "baz"); 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleFixedKeyProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import org.apache.kafka.streams.processor.StateStore; 28 | import org.apache.kafka.streams.processor.api.FixedKeyProcessor; 29 | import org.apache.kafka.streams.processor.api.FixedKeyProcessorContext; 30 | import org.apache.kafka.streams.processor.api.FixedKeyRecord; 31 | 32 | abstract class SimpleFixedKeyProcessor implements FixedKeyProcessor { 33 | private FixedKeyProcessorContext context = null; 34 | 35 | @Override 36 | public void init(final FixedKeyProcessorContext context) { 37 | this.context = context; 38 | } 39 | 40 | protected void forward(final FixedKeyRecord outputRecord) { 41 | this.context.forward(outputRecord); 42 | } 43 | 44 | protected S getStateStore(final String name) { 45 | return this.context.getStateStore(name); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleLegacyProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import org.apache.kafka.streams.processor.Processor; 28 | import org.apache.kafka.streams.processor.ProcessorContext; 29 | import org.apache.kafka.streams.processor.StateStore; 30 | 31 | abstract class SimpleLegacyProcessor implements Processor { 32 | private ProcessorContext context = null; 33 | 34 | @Override 35 | public void init(final ProcessorContext context) { 36 | this.context = context; 37 | } 38 | 39 | @Override 40 | public void close() { 41 | // do nothing 42 | } 43 | 44 | protected S getStateStore(final String name) { 45 | return this.context.getStateStore(name); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import org.apache.kafka.streams.processor.StateStore; 28 | import org.apache.kafka.streams.processor.api.Processor; 29 | import org.apache.kafka.streams.processor.api.ProcessorContext; 30 | import org.apache.kafka.streams.processor.api.Record; 31 | 32 | abstract class SimpleProcessor implements Processor { 33 | private ProcessorContext context = null; 34 | 35 | @Override 36 | public void init(final ProcessorContext context) { 37 | this.context = context; 38 | } 39 | 40 | protected void forward(final Record outputRecord) { 41 | this.context.forward(outputRecord); 42 | } 43 | 44 | protected S getStateStore(final String name) { 45 | return this.context.getStateStore(name); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleTransformer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import org.apache.kafka.streams.kstream.Transformer; 28 | import org.apache.kafka.streams.processor.ProcessorContext; 29 | import org.apache.kafka.streams.processor.StateStore; 30 | 31 | abstract class SimpleTransformer implements Transformer { 32 | private ProcessorContext context = null; 33 | 34 | @Override 35 | public void init(final ProcessorContext context) { 36 | this.context = context; 37 | } 38 | 39 | @Override 40 | public void close() { 41 | // do nothing 42 | } 43 | 44 | protected S getStateStore(final String name) { 45 | return this.context.getStateStore(name); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import org.apache.kafka.streams.kstream.ValueTransformer; 28 | import org.apache.kafka.streams.processor.ProcessorContext; 29 | import org.apache.kafka.streams.processor.StateStore; 30 | 31 | abstract class SimpleValueTransformer implements ValueTransformer { 32 | private ProcessorContext context = null; 33 | 34 | @Override 35 | public void init(final ProcessorContext context) { 36 | this.context = context; 37 | } 38 | 39 | @Override 40 | public void close() { 41 | // do nothing 42 | } 43 | 44 | protected S getStateStore(final String name) { 45 | return this.context.getStateStore(name); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformerWithKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import org.apache.kafka.streams.kstream.ValueTransformerWithKey; 28 | import org.apache.kafka.streams.processor.ProcessorContext; 29 | import org.apache.kafka.streams.processor.StateStore; 30 | 31 | abstract class SimpleValueTransformerWithKey implements ValueTransformerWithKey { 32 | private ProcessorContext context = null; 33 | 34 | @Override 35 | public void init(final ProcessorContext context) { 36 | this.context = context; 37 | } 38 | 39 | @Override 40 | public void close() { 41 | // do nothing 42 | } 43 | 44 | protected S getStateStore(final String name) { 45 | return this.context.getStateStore(name); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamsCleanUpRunnerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static org.assertj.core.api.Assertions.assertThat; 28 | 29 | import java.io.File; 30 | import java.io.FileInputStream; 31 | import java.io.IOException; 32 | import java.util.Map; 33 | import java.util.Properties; 34 | import org.apache.kafka.common.serialization.StringSerializer; 35 | import org.junit.jupiter.api.Test; 36 | 37 | class StreamsCleanUpRunnerTest { 38 | 39 | @Test 40 | void createTemporaryPropertiesFile() throws IOException { 41 | final Map config = Map.of( 42 | "foo", "bar", 43 | "baz", StringSerializer.class 44 | ); 45 | final File file = StreamsCleanUpRunner.createTemporaryPropertiesFile("appId", config); 46 | 47 | assertThat(file).exists(); 48 | 49 | final Properties properties = new Properties(); 50 | try (final FileInputStream inStream = new FileInputStream(file)) { 51 | properties.load(inStream); 52 | } 53 | 54 | final Properties expected = StreamsCleanUpRunner.toStringBasedProperties(config); 55 | assertThat(properties).containsAllEntriesOf(expected); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamsExecutionOptionsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static java.util.Collections.emptyMap; 28 | import static org.assertj.core.api.Assertions.assertThat; 29 | 30 | import java.util.Map; 31 | import org.apache.kafka.clients.consumer.ConsumerConfig; 32 | import org.junit.jupiter.api.Test; 33 | 34 | class StreamsExecutionOptionsTest { 35 | 36 | @Test 37 | void shouldLeaveGroup() { 38 | final StreamsExecutionOptions options = StreamsExecutionOptions.builder() 39 | .build(); 40 | assertThat(options.shouldLeaveGroup(emptyMap())).isTrue(); 41 | } 42 | 43 | @Test 44 | void shouldNotLeaveGroup() { 45 | final StreamsExecutionOptions options = StreamsExecutionOptions.builder() 46 | .volatileGroupInstanceId(false) 47 | .build(); 48 | assertThat(options.shouldLeaveGroup(Map.of( 49 | ConsumerConfig.GROUP_INSTANCE_ID_CONFIG, "foo" 50 | ))).isFalse(); 51 | } 52 | 53 | @Test 54 | void shouldLeaveGroupWithVolatileGroupId() { 55 | final StreamsExecutionOptions options = StreamsExecutionOptions.builder() 56 | .volatileGroupInstanceId(true) 57 | .build(); 58 | assertThat(options.shouldLeaveGroup(Map.of( 59 | ConsumerConfig.GROUP_INSTANCE_ID_CONFIG, "foo" 60 | ))).isTrue(); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/StringApp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import com.bakdata.fluent_kafka_streams_tests.TestTopology; 28 | import java.util.Map; 29 | import org.apache.kafka.common.serialization.Serdes.StringSerde; 30 | 31 | abstract class StringApp implements StreamsApp { 32 | 33 | @Override 34 | public String getUniqueAppId(final StreamsTopicConfig topics) { 35 | return "my-app"; 36 | } 37 | 38 | @Override 39 | public SerdeConfig defaultSerializationConfig() { 40 | return new SerdeConfig(StringSerde.class, StringSerde.class); 41 | } 42 | 43 | TestTopology startApp(final StreamsTopicConfig topicConfig) { 44 | final ConfiguredStreamsApp configuredApp = TestHelper.configureApp(this, topicConfig); 45 | return TestHelper.startApp(configuredApp); 46 | } 47 | 48 | TestTopology startApp() { 49 | return this.startApp(StreamsTopicConfig.builder().build()); 50 | } 51 | 52 | ConfiguredStreamsApp configureApp(final Map config) { 53 | return TestHelper.configureApp(this, StreamsTopicConfig.builder().build(), config); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/TestHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import static java.util.Collections.emptyMap; 28 | 29 | import com.bakdata.fluent_kafka_streams_tests.TestTopology; 30 | import java.lang.Thread.UncaughtExceptionHandler; 31 | import java.util.Map; 32 | import lombok.Getter; 33 | import lombok.experimental.UtilityClass; 34 | 35 | @UtilityClass 36 | public class TestHelper { 37 | public static Thread run(final StreamsRunner runner) { 38 | // run in Thread because the application blocks indefinitely 39 | final Thread thread = new Thread(runner); 40 | final UncaughtExceptionHandler handler = new CapturingUncaughtExceptionHandler(); 41 | thread.setUncaughtExceptionHandler(handler); 42 | thread.start(); 43 | return thread; 44 | } 45 | 46 | static TestTopology startApp(final ConfiguredStreamsApp app) { 47 | final TestTopology topology = 48 | TestTopologyFactory.withoutSchemaRegistry().createTopology(app); 49 | topology.start(); 50 | return topology; 51 | } 52 | 53 | static ConfiguredStreamsApp configureApp(final StreamsApp app, 54 | final StreamsTopicConfig topicConfig) { 55 | return configureApp(app, topicConfig, emptyMap()); 56 | } 57 | 58 | static ConfiguredStreamsApp configureApp(final StreamsApp app, 59 | final StreamsTopicConfig topicConfig, 60 | final Map config) { 61 | return new ConfiguredStreamsApp<>(app, new AppConfiguration<>(topicConfig, config)); 62 | } 63 | 64 | @Getter 65 | public static class CapturingUncaughtExceptionHandler implements UncaughtExceptionHandler { 66 | private Throwable lastException; 67 | 68 | @Override 69 | public void uncaughtException(final Thread t, final Throwable e) { 70 | this.lastException = e; 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/AvroKeyProducer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka.test_applications; 26 | 27 | import com.bakdata.kafka.ProducerApp; 28 | import com.bakdata.kafka.ProducerBuilder; 29 | import com.bakdata.kafka.ProducerRunnable; 30 | import com.bakdata.kafka.SerializerConfig; 31 | import com.bakdata.kafka.TestRecord; 32 | import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerializer; 33 | import org.apache.kafka.clients.producer.Producer; 34 | import org.apache.kafka.clients.producer.ProducerRecord; 35 | import org.apache.kafka.common.serialization.StringSerializer; 36 | 37 | public class AvroKeyProducer implements ProducerApp { 38 | @Override 39 | public ProducerRunnable buildRunnable(final ProducerBuilder builder) { 40 | return () -> { 41 | try (final Producer producer = builder.createProducer()) { 42 | producer.send(new ProducerRecord<>(builder.getTopics().getOutputTopic(), 43 | TestRecord.newBuilder().setContent("key").build(), "value")); 44 | } 45 | }; 46 | } 47 | 48 | @Override 49 | public SerializerConfig defaultSerializationConfig() { 50 | return new SerializerConfig(SpecificAvroSerializer.class, StringSerializer.class); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/AvroValueProducer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka.test_applications; 26 | 27 | import com.bakdata.kafka.ProducerApp; 28 | import com.bakdata.kafka.ProducerBuilder; 29 | import com.bakdata.kafka.ProducerRunnable; 30 | import com.bakdata.kafka.SerializerConfig; 31 | import com.bakdata.kafka.TestRecord; 32 | import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerializer; 33 | import org.apache.kafka.clients.producer.Producer; 34 | import org.apache.kafka.clients.producer.ProducerRecord; 35 | import org.apache.kafka.common.serialization.StringSerializer; 36 | 37 | public class AvroValueProducer implements ProducerApp { 38 | @Override 39 | public ProducerRunnable buildRunnable(final ProducerBuilder builder) { 40 | return () -> { 41 | try (final Producer producer = builder.createProducer()) { 42 | producer.send(new ProducerRecord<>(builder.getTopics().getOutputTopic(), "key", 43 | TestRecord.newBuilder().setContent("value").build())); 44 | } 45 | }; 46 | } 47 | 48 | @Override 49 | public SerializerConfig defaultSerializationConfig() { 50 | return new SerializerConfig(StringSerializer.class, SpecificAvroSerializer.class); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/LabeledInputTopics.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka.test_applications; 26 | 27 | import com.bakdata.kafka.KStreamX; 28 | import com.bakdata.kafka.SerdeConfig; 29 | import com.bakdata.kafka.StreamsApp; 30 | import com.bakdata.kafka.StreamsTopicConfig; 31 | import com.bakdata.kafka.StreamsBuilderX; 32 | import lombok.NoArgsConstructor; 33 | import org.apache.kafka.common.serialization.Serdes.StringSerde; 34 | 35 | @NoArgsConstructor 36 | public class LabeledInputTopics implements StreamsApp { 37 | @Override 38 | public void buildTopology(final StreamsBuilderX builder) { 39 | final KStreamX input = builder.streamInput("label"); 40 | input.toOutputTopic(); 41 | } 42 | 43 | @Override 44 | public String getUniqueAppId(final StreamsTopicConfig topics) { 45 | return this.getClass().getSimpleName() + "-" + topics.getOutputTopic(); 46 | } 47 | 48 | @Override 49 | public SerdeConfig defaultSerializationConfig() { 50 | return new SerdeConfig(StringSerde.class, StringSerde.class); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/Mirror.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka.test_applications; 26 | 27 | import com.bakdata.kafka.KStreamX; 28 | import com.bakdata.kafka.SerdeConfig; 29 | import com.bakdata.kafka.StreamsApp; 30 | import com.bakdata.kafka.StreamsBuilderX; 31 | import com.bakdata.kafka.StreamsTopicConfig; 32 | import lombok.NoArgsConstructor; 33 | import org.apache.kafka.common.serialization.Serdes.StringSerde; 34 | 35 | @NoArgsConstructor 36 | public class Mirror implements StreamsApp { 37 | @Override 38 | public void buildTopology(final StreamsBuilderX builder) { 39 | final KStreamX input = builder.streamInput(); 40 | input.toOutputTopic(); 41 | } 42 | 43 | @Override 44 | public String getUniqueAppId(final StreamsTopicConfig topics) { 45 | return this.getClass().getSimpleName() + "-" + topics.getOutputTopic(); 46 | } 47 | 48 | @Override 49 | public SerdeConfig defaultSerializationConfig() { 50 | return new SerdeConfig(StringSerde.class, StringSerde.class); 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorKeyWithAvro.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka.test_applications; 26 | 27 | import com.bakdata.kafka.KStreamX; 28 | import com.bakdata.kafka.SerdeConfig; 29 | import com.bakdata.kafka.StreamsApp; 30 | import com.bakdata.kafka.StreamsTopicConfig; 31 | import com.bakdata.kafka.TestRecord; 32 | import com.bakdata.kafka.StreamsBuilderX; 33 | import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde; 34 | import lombok.NoArgsConstructor; 35 | import org.apache.kafka.common.serialization.Serdes.StringSerde; 36 | 37 | @NoArgsConstructor 38 | public class MirrorKeyWithAvro implements StreamsApp { 39 | @Override 40 | public void buildTopology(final StreamsBuilderX builder) { 41 | final KStreamX input = builder.streamInput(); 42 | input.toOutputTopic(); 43 | } 44 | 45 | @Override 46 | public String getUniqueAppId(final StreamsTopicConfig topics) { 47 | return this.getClass().getSimpleName() + "-" + topics.getOutputTopic(); 48 | } 49 | 50 | @Override 51 | public SerdeConfig defaultSerializationConfig() { 52 | return new SerdeConfig(SpecificAvroSerde.class, StringSerde.class); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorValueWithAvro.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka.test_applications; 26 | 27 | import com.bakdata.kafka.KStreamX; 28 | import com.bakdata.kafka.SerdeConfig; 29 | import com.bakdata.kafka.StreamsApp; 30 | import com.bakdata.kafka.StreamsTopicConfig; 31 | import com.bakdata.kafka.TestRecord; 32 | import com.bakdata.kafka.StreamsBuilderX; 33 | import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde; 34 | import lombok.NoArgsConstructor; 35 | import org.apache.kafka.common.serialization.Serdes.StringSerde; 36 | 37 | @NoArgsConstructor 38 | public class MirrorValueWithAvro implements StreamsApp { 39 | @Override 40 | public void buildTopology(final StreamsBuilderX builder) { 41 | final KStreamX input = builder.streamInput(); 42 | input.toOutputTopic(); 43 | } 44 | 45 | @Override 46 | public String getUniqueAppId(final StreamsTopicConfig topics) { 47 | return this.getClass().getSimpleName() + "-" + topics.getOutputTopic(); 48 | } 49 | 50 | @Override 51 | public SerdeConfig defaultSerializationConfig() { 52 | return new SerdeConfig(StringSerde.class, SpecificAvroSerde.class); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka.test_applications; 26 | 27 | import com.bakdata.kafka.ConsumedX; 28 | import com.bakdata.kafka.KStreamX; 29 | import com.bakdata.kafka.Preconfigured; 30 | import com.bakdata.kafka.ProducedX; 31 | import com.bakdata.kafka.SerdeConfig; 32 | import com.bakdata.kafka.StreamsApp; 33 | import com.bakdata.kafka.StreamsBuilderX; 34 | import com.bakdata.kafka.StreamsTopicConfig; 35 | import com.bakdata.kafka.TestRecord; 36 | import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde; 37 | import lombok.NoArgsConstructor; 38 | import org.apache.kafka.common.serialization.Serde; 39 | import org.apache.kafka.common.serialization.Serdes.StringSerde; 40 | 41 | @NoArgsConstructor 42 | public class MirrorWithNonDefaultSerde implements StreamsApp { 43 | 44 | public static Preconfigured> newKeySerde() { 45 | return Preconfigured.create(new SpecificAvroSerde<>()); 46 | } 47 | 48 | public static Preconfigured> newValueSerde() { 49 | return Preconfigured.create(new SpecificAvroSerde<>()); 50 | } 51 | 52 | @Override 53 | public void buildTopology(final StreamsBuilderX builder) { 54 | final Preconfigured> valueSerde = newValueSerde(); 55 | final Preconfigured> keySerde = newKeySerde(); 56 | final KStreamX input = 57 | builder.streamInput(ConsumedX.with(keySerde, valueSerde)); 58 | input.toOutputTopic(ProducedX.with(keySerde, valueSerde)); 59 | } 60 | 61 | @Override 62 | public String getUniqueAppId(final StreamsTopicConfig topics) { 63 | return this.getClass().getSimpleName() + "-" + topics.getOutputTopic(); 64 | } 65 | 66 | @Override 67 | public SerdeConfig defaultSerializationConfig() { 68 | return new SerdeConfig(StringSerde.class, StringSerde.class); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/StringProducer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka.test_applications; 26 | 27 | import com.bakdata.kafka.ProducerApp; 28 | import com.bakdata.kafka.ProducerBuilder; 29 | import com.bakdata.kafka.ProducerRunnable; 30 | import com.bakdata.kafka.SerializerConfig; 31 | import org.apache.kafka.clients.producer.Producer; 32 | import org.apache.kafka.clients.producer.ProducerRecord; 33 | import org.apache.kafka.common.serialization.StringSerializer; 34 | 35 | public class StringProducer implements ProducerApp { 36 | @Override 37 | public ProducerRunnable buildRunnable(final ProducerBuilder builder) { 38 | return () -> { 39 | try (final Producer producer = builder.createProducer()) { 40 | producer.send(new ProducerRecord<>(builder.getTopics().getOutputTopic(), "foo", "bar")); 41 | } 42 | }; 43 | } 44 | 45 | @Override 46 | public SerializerConfig defaultSerializationConfig() { 47 | return new SerializerConfig(StringSerializer.class, StringSerializer.class); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCount.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka.test_applications; 26 | 27 | import com.bakdata.kafka.KStreamX; 28 | import com.bakdata.kafka.KTableX; 29 | import com.bakdata.kafka.SerdeConfig; 30 | import com.bakdata.kafka.StreamsApp; 31 | import com.bakdata.kafka.StreamsBuilderX; 32 | import com.bakdata.kafka.StreamsTopicConfig; 33 | import java.util.Arrays; 34 | import java.util.regex.Pattern; 35 | import lombok.NoArgsConstructor; 36 | import org.apache.kafka.common.serialization.Serdes; 37 | import org.apache.kafka.common.serialization.Serdes.StringSerde; 38 | import org.apache.kafka.streams.kstream.Materialized; 39 | import org.apache.kafka.streams.kstream.Produced; 40 | 41 | @NoArgsConstructor 42 | public class WordCount implements StreamsApp { 43 | 44 | @Override 45 | public void buildTopology(final StreamsBuilderX builder) { 46 | final KStreamX textLines = builder.streamInput(); 47 | 48 | final Pattern pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS); 49 | final KTableX wordCounts = textLines 50 | .flatMapValues(value -> Arrays.asList(pattern.split(value.toLowerCase()))) 51 | .groupBy((key, word) -> word) 52 | .count(Materialized.as("counts")); 53 | 54 | wordCounts.toStream().toOutputTopic(Produced.valueSerde(Serdes.Long())); 55 | } 56 | 57 | @Override 58 | public String getUniqueAppId(final StreamsTopicConfig topics) { 59 | return this.getClass().getSimpleName() + "-" + topics.getOutputTopic(); 60 | } 61 | 62 | @Override 63 | public SerdeConfig defaultSerializationConfig() { 64 | return new SerdeConfig(StringSerde.class, StringSerde.class); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCountPattern.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka.test_applications; 26 | 27 | import com.bakdata.kafka.KStreamX; 28 | import com.bakdata.kafka.KTableX; 29 | import com.bakdata.kafka.SerdeConfig; 30 | import com.bakdata.kafka.StreamsApp; 31 | import com.bakdata.kafka.StreamsTopicConfig; 32 | import com.bakdata.kafka.StreamsBuilderX; 33 | import java.util.Arrays; 34 | import java.util.regex.Pattern; 35 | import lombok.NoArgsConstructor; 36 | import org.apache.kafka.common.serialization.Serde; 37 | import org.apache.kafka.common.serialization.Serdes; 38 | import org.apache.kafka.common.serialization.Serdes.StringSerde; 39 | import org.apache.kafka.streams.kstream.Materialized; 40 | import org.apache.kafka.streams.kstream.Produced; 41 | 42 | @NoArgsConstructor 43 | public class WordCountPattern implements StreamsApp { 44 | 45 | @Override 46 | public void buildTopology(final StreamsBuilderX builder) { 47 | final KStreamX textLines = builder.streamInputPattern(); 48 | 49 | final Pattern pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS); 50 | final KTableX wordCounts = textLines 51 | .flatMapValues(value -> Arrays.asList(pattern.split(value.toLowerCase()))) 52 | .groupBy((key, word) -> word) 53 | .count(Materialized.as("counts")); 54 | 55 | final Serde longValueSerde = Serdes.Long(); 56 | wordCounts.toStream().toOutputTopic(Produced.valueSerde(longValueSerde)); 57 | } 58 | 59 | @Override 60 | public String getUniqueAppId(final StreamsTopicConfig topics) { 61 | return this.getClass().getSimpleName() + "-" + topics.getOutputTopic(); 62 | } 63 | 64 | @Override 65 | public SerdeConfig defaultSerializationConfig() { 66 | return new SerdeConfig(StringSerde.class, StringSerde.class); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /streams-bootstrap-core/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /streams-bootstrap-large-messages/build.gradle.kts: -------------------------------------------------------------------------------- 1 | description = "Utils for using Large Message SerDe with your Kafka Streams Application" 2 | 3 | plugins { 4 | id("java-library") 5 | } 6 | 7 | dependencies { 8 | api(project(":streams-bootstrap-core")) 9 | api(platform(libs.largeMessage.bom)) 10 | implementation(libs.largeMessage.core) 11 | } 12 | -------------------------------------------------------------------------------- /streams-bootstrap-large-messages/lombok.config: -------------------------------------------------------------------------------- 1 | # This file is generated by the 'io.freefair.lombok' Gradle plugin 2 | config.stopBubbling = true 3 | lombok.addLombokGeneratedAnnotation = true 4 | -------------------------------------------------------------------------------- /streams-bootstrap-large-messages/src/main/java/com/bakdata/kafka/LargeMessageProducerApp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | /** 28 | * {@link ProducerApp} that automatically removes files associated with {@code LargeMessageSerializer} 29 | */ 30 | public interface LargeMessageProducerApp extends ProducerApp { 31 | 32 | @Override 33 | default ProducerCleanUpConfiguration setupCleanUp( 34 | final EffectiveAppConfiguration configuration) { 35 | final ProducerCleanUpConfiguration cleanUpConfiguration = ProducerApp.super.setupCleanUp(configuration); 36 | return LargeMessageAppUtils.registerTopicHook(cleanUpConfiguration, configuration); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /streams-bootstrap-large-messages/src/main/java/com/bakdata/kafka/LargeMessageStreamsApp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2024 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | /** 28 | * {@link StreamsApp} that automatically removes files associated with {@code LargeMessageSerde} 29 | */ 30 | public interface LargeMessageStreamsApp extends StreamsApp { 31 | 32 | @Override 33 | default StreamsCleanUpConfiguration setupCleanUp( 34 | final EffectiveAppConfiguration configuration) { 35 | final StreamsCleanUpConfiguration cleanUpConfiguration = StreamsApp.super.setupCleanUp(configuration); 36 | return LargeMessageAppUtils.registerTopicHook(cleanUpConfiguration, configuration); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /streams-bootstrap-test/build.gradle.kts: -------------------------------------------------------------------------------- 1 | description = "Utils for testing your Kafka Streams Application" 2 | 3 | plugins { 4 | id("java-library") 5 | } 6 | 7 | dependencies { 8 | api(project(":streams-bootstrap-core")) 9 | api(libs.fluentKafkaStreamsTests) 10 | implementation(libs.slf4j) 11 | 12 | testRuntimeOnly(libs.junit.platform.launcher) 13 | testImplementation(libs.junit.jupiter) 14 | testImplementation(libs.assertj) 15 | testFixturesApi(libs.testcontainers.junit) 16 | testFixturesApi(libs.testcontainers.kafka) 17 | testImplementation(libs.log4j.slf4j2) 18 | testFixturesApi(libs.awaitility) 19 | } 20 | -------------------------------------------------------------------------------- /streams-bootstrap-test/lombok.config: -------------------------------------------------------------------------------- 1 | # This file is generated by the 'io.freefair.lombok' Gradle plugin 2 | config.stopBubbling = true 3 | lombok.addLombokGeneratedAnnotation = true 4 | -------------------------------------------------------------------------------- /streams-bootstrap-test/src/test/java/com/bakdata/kafka/SimpleStreamsApp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import org.apache.kafka.common.serialization.Serdes.StringSerde; 28 | 29 | class SimpleStreamsApp implements StreamsApp { 30 | 31 | @Override 32 | public void buildTopology(final StreamsBuilderX builder) { 33 | builder.streamInput().toOutputTopic(); 34 | } 35 | 36 | @Override 37 | public String getUniqueAppId(final StreamsTopicConfig topics) { 38 | return "group"; 39 | } 40 | 41 | @Override 42 | public SerdeConfig defaultSerializationConfig() { 43 | return new SerdeConfig(StringSerde.class, StringSerde.class); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /streams-bootstrap-test/src/test/java/com/bakdata/kafka/TestTopologyFactoryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * MIT License 3 | * 4 | * Copyright (c) 2025 bakdata 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in all 14 | * copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | * SOFTWARE. 23 | */ 24 | 25 | package com.bakdata.kafka; 26 | 27 | import com.bakdata.fluent_kafka_streams_tests.junit5.TestTopologyExtension; 28 | import java.util.List; 29 | import org.junit.jupiter.api.Test; 30 | import org.junit.jupiter.api.extension.RegisterExtension; 31 | 32 | class TestTopologyFactoryTest extends KafkaTest { 33 | 34 | @RegisterExtension 35 | private final TestTopologyExtension testTopology = TestTopologyFactory.withoutSchemaRegistry() 36 | .createTopologyExtension(createApp()); 37 | 38 | private static ConfiguredStreamsApp createApp() { 39 | final StreamsApp app = new SimpleStreamsApp(); 40 | return new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() 41 | .inputTopics(List.of("input")) 42 | .outputTopic("output") 43 | .build())); 44 | } 45 | 46 | @Test 47 | void shouldVerify() { 48 | this.testTopology.input() 49 | .add("foo", "bar"); 50 | this.testTopology.streamOutput() 51 | .expectNextRecord() 52 | .hasKey("foo") 53 | .hasValue("bar") 54 | .expectNoMoreRecord(); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /streams-bootstrap-test/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | --------------------------------------------------------------------------------