├── .github └── ISSUE_TEMPLATE │ └── bug_report.md ├── .gitignore ├── .jdk8 ├── .mvn ├── jvm.config ├── maven.config └── wrapper │ ├── MavenWrapperDownloader.java │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── .settings.xml ├── LICENSE ├── README.adoc ├── docs ├── .jdk8 ├── pom.xml └── src │ └── main │ ├── asciidoc │ ├── .gitignore │ ├── Guardfile │ ├── README.adoc │ ├── _configprops.adoc │ ├── appendix.adoc │ ├── building.adoc │ ├── contributing.adoc │ ├── dlq.adoc │ ├── ghpages.sh │ ├── images │ │ ├── kafka-binder.png │ │ ├── kafka-streams-initializr.png │ │ └── spring-initializr-kafka-streams.png │ ├── index-docinfo.xml │ ├── kafka-streams.adoc │ ├── overview.adoc │ ├── partitions.adoc │ ├── spring-cloud-stream-binder-kafka-aggregate.adoc │ ├── spring-cloud-stream-binder-kafka.adoc │ └── tips.adoc │ └── ruby │ └── generate_readme.sh ├── mvnw ├── mvnw.cmd ├── pom.xml ├── spring-cloud-starter-stream-kafka ├── .jdk8 └── pom.xml ├── spring-cloud-stream-binder-kafka-core ├── .jdk8 ├── .settings │ └── org.eclipse.jdt.ui.prefs ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── springframework │ │ └── cloud │ │ └── stream │ │ └── binder │ │ └── kafka │ │ ├── properties │ │ ├── JaasLoginModuleConfiguration.java │ │ ├── KafkaBinderConfigurationProperties.java │ │ ├── KafkaBindingProperties.java │ │ ├── KafkaConsumerProperties.java │ │ ├── KafkaExtendedBindingProperties.java │ │ ├── KafkaProducerProperties.java │ │ └── KafkaTopicProperties.java │ │ ├── provisioning │ │ ├── AdminClientConfigCustomizer.java │ │ └── KafkaTopicProvisioner.java │ │ └── utils │ │ ├── DlqDestinationResolver.java │ │ ├── DlqPartitionFunction.java │ │ └── KafkaTopicUtils.java │ └── test │ ├── java │ └── org │ │ └── springframework │ │ └── cloud │ │ └── stream │ │ └── binder │ │ └── kafka │ │ ├── properties │ │ └── KafkaBinderConfigurationPropertiesTest.java │ │ └── provisioning │ │ └── KafkaTopicProvisionerTests.java │ └── resources │ ├── test.truststore.ks │ ├── testclient.keystore │ └── testclient.truststore ├── spring-cloud-stream-binder-kafka-streams ├── .jdk8 ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── springframework │ │ │ └── cloud │ │ │ └── stream │ │ │ └── binder │ │ │ └── kafka │ │ │ └── streams │ │ │ ├── AbstractKafkaStreamsBinderProcessor.java │ │ │ ├── DeserializationExceptionHandler.java │ │ │ ├── EncodingDecodingBindAdviceHandler.java │ │ │ ├── ExtendedBindingHandlerMappingsProviderAutoConfiguration.java │ │ │ ├── GlobalKTableBinder.java │ │ │ ├── GlobalKTableBinderConfiguration.java │ │ │ ├── GlobalKTableBoundElementFactory.java │ │ │ ├── InteractiveQueryService.java │ │ │ ├── KStreamBinder.java │ │ │ ├── KStreamBinderConfiguration.java │ │ │ ├── KStreamBoundElementFactory.java │ │ │ ├── KTableBinder.java │ │ │ ├── KTableBinderConfiguration.java │ │ │ ├── KTableBoundElementFactory.java │ │ │ ├── KafkaStreamsBinderHealthIndicator.java │ │ │ ├── KafkaStreamsBinderHealthIndicatorConfiguration.java │ │ │ ├── KafkaStreamsBinderMetrics.java │ │ │ ├── KafkaStreamsBinderSupportAutoConfiguration.java │ │ │ ├── KafkaStreamsBinderUtils.java │ │ │ ├── KafkaStreamsBindingInformationCatalogue.java │ │ │ ├── KafkaStreamsFunctionProcessor.java │ │ │ ├── KafkaStreamsJaasConfiguration.java │ │ │ ├── KafkaStreamsMessageConversionDelegate.java │ │ │ ├── KafkaStreamsRegistry.java │ │ │ ├── KeyValueSerdeResolver.java │ │ │ ├── MultiBinderPropertiesConfiguration.java │ │ │ ├── SendToDlqAndContinue.java │ │ │ ├── SkipAndContinueExceptionHandler.java │ │ │ ├── StreamsBuilderFactoryManager.java │ │ │ ├── endpoint │ │ │ ├── KafkaStreamsTopologyEndpoint.java │ │ │ └── KafkaStreamsTopologyEndpointAutoConfiguration.java │ │ │ ├── function │ │ │ ├── FunctionDetectorCondition.java │ │ │ ├── KafkaStreamsBindableProxyFactory.java │ │ │ ├── KafkaStreamsFunctionAutoConfiguration.java │ │ │ ├── KafkaStreamsFunctionBeanPostProcessor.java │ │ │ └── KafkaStreamsFunctionProcessorInvoker.java │ │ │ ├── properties │ │ │ ├── KafkaStreamsBinderConfigurationProperties.java │ │ │ ├── KafkaStreamsBindingProperties.java │ │ │ ├── KafkaStreamsConsumerProperties.java │ │ │ ├── KafkaStreamsExtendedBindingProperties.java │ │ │ └── KafkaStreamsProducerProperties.java │ │ │ └── serde │ │ │ └── CollectionSerde.java │ └── resources │ │ └── META-INF │ │ ├── spring.binders │ │ └── spring.factories │ └── test │ ├── java │ └── org │ │ └── springframework │ │ └── cloud │ │ └── stream │ │ └── binder │ │ └── kafka │ │ └── streams │ │ ├── ExtendedBindingHandlerMappingsProviderAutoConfigurationTests.java │ │ ├── KafkaStreamsEventTypeRoutingTests.java │ │ ├── KafkaStreamsFunctionCompositionTests.java │ │ ├── KafkaStreamsInteractiveQueryIntegrationTests.java │ │ ├── MultipleFunctionsInSameAppTests.java │ │ ├── bootstrap │ │ ├── KafkaStreamsBinderBootstrapTest.java │ │ └── KafkaStreamsBinderJaasInitTests.java │ │ ├── function │ │ ├── KafkaStreamsBinderWordCountBranchesFunctionTests.java │ │ ├── KafkaStreamsBinderWordCountFunctionTests.java │ │ ├── KafkaStreamsComponentBeansTests.java │ │ ├── KafkaStreamsFunctionStateStoreTests.java │ │ ├── KafkaStreamsRetryTests.java │ │ ├── SerdesProvidedAsBeansTests.java │ │ ├── StreamToGlobalKTableFunctionTests.java │ │ └── StreamToTableJoinFunctionTests.java │ │ ├── integration │ │ ├── DlqDestinationResolverTests.java │ │ ├── KafkaStreamsBinderDestinationIsPatternTests.java │ │ ├── KafkaStreamsBinderHealthIndicatorTests.java │ │ ├── KafkaStreamsBinderMultipleInputTopicsTest.java │ │ ├── KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests.java │ │ ├── KafkaStreamsBinderTombstoneTests.java │ │ ├── KafkaStreamsStateStoreIntegrationTests.java │ │ └── KafkastreamsBinderPojoInputStringOutputIntegrationTests.java │ │ └── serde │ │ └── CollectionSerdeTest.java │ └── resources │ └── logback.xml ├── spring-cloud-stream-binder-kafka ├── .jdk8 ├── .settings │ └── org.eclipse.jdt.ui.prefs ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── springframework │ │ │ └── cloud │ │ │ └── stream │ │ │ └── binder │ │ │ └── kafka │ │ │ ├── BinderHeaderMapper.java │ │ │ ├── KafkaBinderEnvironmentPostProcessor.java │ │ │ ├── KafkaBinderHealth.java │ │ │ ├── KafkaBinderHealthIndicator.java │ │ │ ├── KafkaBinderMetrics.java │ │ │ ├── KafkaBindingRebalanceListener.java │ │ │ ├── KafkaExpressionEvaluatingInterceptor.java │ │ │ ├── KafkaMessageChannelBinder.java │ │ │ ├── KafkaNullConverter.java │ │ │ ├── ListenerContainerWithDlqAndRetryCustomizer.java │ │ │ └── config │ │ │ ├── ClientFactoryCustomizer.java │ │ │ ├── ConsumerConfigCustomizer.java │ │ │ ├── ExtendedBindingHandlerMappingsProviderConfiguration.java │ │ │ ├── KafkaBinderConfiguration.java │ │ │ ├── KafkaBinderHealthIndicatorConfiguration.java │ │ │ └── ProducerConfigCustomizer.java │ └── resources │ │ └── META-INF │ │ ├── spring.binders │ │ └── spring.factories │ └── test │ ├── java │ └── org │ │ └── springframework │ │ └── cloud │ │ └── stream │ │ └── binder │ │ └── kafka │ │ ├── AbstractKafkaTestBinder.java │ │ ├── AdminConfigTests.java │ │ ├── AutoCreateTopicDisabledTests.java │ │ ├── KafkaBinderAutoConfigurationPropertiesTest.java │ │ ├── KafkaBinderConfigurationPropertiesTest.java │ │ ├── KafkaBinderConfigurationTest.java │ │ ├── KafkaBinderHealthIndicatorTest.java │ │ ├── KafkaBinderJaasInitializerListenerTest.java │ │ ├── KafkaBinderMetricsTest.java │ │ ├── KafkaBinderTests.java │ │ ├── KafkaBinderUnitTests.java │ │ ├── KafkaTestBinder.java │ │ ├── KafkaTransactionTests.java │ │ ├── RawKafkaPartitionTestSupport.java │ │ ├── TestKafkaProperties.java │ │ ├── bootstrap │ │ ├── KafkaBinderBootstrapTest.java │ │ ├── KafkaBinderCustomHealthCheckTests.java │ │ └── KafkaBinderMeterRegistryTest.java │ │ ├── integration │ │ ├── KafkaBinderActuatorTests.java │ │ ├── KafkaBinderExtendedPropertiesTest.java │ │ ├── KafkaConfigCustomizationTests.java │ │ ├── KafkaNullConverterTest.java │ │ ├── KafkaRetryDlqBinderOrContainerTests.java │ │ ├── ProducerOnlyTransactionTests.java │ │ └── topic │ │ │ └── configs │ │ │ ├── BaseKafkaBinderTopicPropertiesUpdateTest.java │ │ │ ├── DisabledKafkaBinderTopicPropertiesUpdateTest.java │ │ │ └── KafkaBinderTopicPropertiesUpdateTest.java │ │ └── integration2 │ │ └── ConsumerProducerTransactionTests.java │ └── resources │ ├── binder-config-autoconfig.properties │ ├── binder-config.properties │ ├── jaas-sample-kafka-only.conf │ ├── jaas-sample-with-zk.conf │ └── logback.xml └── update-version.sh /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: Please create new issues in https://github.com/spring-cloud/spring-cloud-stream/issues 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | Please create all new issues in https://github.com/spring-cloud/spring-cloud-stream/issues. The Kafka binder repository has been relocated to the core Spring Cloud Stream repo. 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /application.yml 2 | /application.properties 3 | asciidoctor.css 4 | *~ 5 | .#* 6 | *# 7 | target/ 8 | build/ 9 | bin/ 10 | _site/ 11 | .classpath 12 | .project 13 | .settings 14 | .springBeans 15 | .DS_Store 16 | *.sw* 17 | *.iml 18 | *.ipr 19 | *.iws 20 | .idea/* 21 | */.idea 22 | .factorypath 23 | dump.rdb 24 | .apt_generated 25 | artifacts 26 | .sts4-cache 27 | -------------------------------------------------------------------------------- /.jdk8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-stream-binder-kafka/31ea10683422d38d8185c6b236b8954200562b77/.jdk8 -------------------------------------------------------------------------------- /.mvn/jvm.config: -------------------------------------------------------------------------------- 1 | -Xmx1024m -XX:CICompilerCount=1 -XX:TieredStopAtLevel=1 -Djava.security.egd=file:/dev/./urandom -------------------------------------------------------------------------------- /.mvn/maven.config: -------------------------------------------------------------------------------- 1 | -DaltSnapshotDeploymentRepository=repo.spring.io::default::https://repo.spring.io/libs-snapshot-local -P spring 2 | -------------------------------------------------------------------------------- /.mvn/wrapper/MavenWrapperDownloader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2007-present the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | import java.net.*; 17 | import java.io.*; 18 | import java.nio.channels.*; 19 | import java.util.Properties; 20 | 21 | public class MavenWrapperDownloader { 22 | 23 | private static final String WRAPPER_VERSION = "0.5.6"; 24 | /** 25 | * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. 26 | */ 27 | private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/" 28 | + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar"; 29 | 30 | /** 31 | * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to 32 | * use instead of the default one. 33 | */ 34 | private static final String MAVEN_WRAPPER_PROPERTIES_PATH = 35 | ".mvn/wrapper/maven-wrapper.properties"; 36 | 37 | /** 38 | * Path where the maven-wrapper.jar will be saved to. 39 | */ 40 | private static final String MAVEN_WRAPPER_JAR_PATH = 41 | ".mvn/wrapper/maven-wrapper.jar"; 42 | 43 | /** 44 | * Name of the property which should be used to override the default download url for the wrapper. 45 | */ 46 | private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; 47 | 48 | public static void main(String args[]) { 49 | System.out.println("- Downloader started"); 50 | File baseDirectory = new File(args[0]); 51 | System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); 52 | 53 | // If the maven-wrapper.properties exists, read it and check if it contains a custom 54 | // wrapperUrl parameter. 55 | File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); 56 | String url = DEFAULT_DOWNLOAD_URL; 57 | if(mavenWrapperPropertyFile.exists()) { 58 | FileInputStream mavenWrapperPropertyFileInputStream = null; 59 | try { 60 | mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); 61 | Properties mavenWrapperProperties = new Properties(); 62 | mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); 63 | url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); 64 | } catch (IOException e) { 65 | System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); 66 | } finally { 67 | try { 68 | if(mavenWrapperPropertyFileInputStream != null) { 69 | mavenWrapperPropertyFileInputStream.close(); 70 | } 71 | } catch (IOException e) { 72 | // Ignore ... 73 | } 74 | } 75 | } 76 | System.out.println("- Downloading from: " + url); 77 | 78 | File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); 79 | if(!outputFile.getParentFile().exists()) { 80 | if(!outputFile.getParentFile().mkdirs()) { 81 | System.out.println( 82 | "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'"); 83 | } 84 | } 85 | System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); 86 | try { 87 | downloadFileFromURL(url, outputFile); 88 | System.out.println("Done"); 89 | System.exit(0); 90 | } catch (Throwable e) { 91 | System.out.println("- Error downloading"); 92 | e.printStackTrace(); 93 | System.exit(1); 94 | } 95 | } 96 | 97 | private static void downloadFileFromURL(String urlString, File destination) throws Exception { 98 | if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) { 99 | String username = System.getenv("MVNW_USERNAME"); 100 | char[] password = System.getenv("MVNW_PASSWORD").toCharArray(); 101 | Authenticator.setDefault(new Authenticator() { 102 | @Override 103 | protected PasswordAuthentication getPasswordAuthentication() { 104 | return new PasswordAuthentication(username, password); 105 | } 106 | }); 107 | } 108 | URL website = new URL(urlString); 109 | ReadableByteChannel rbc; 110 | rbc = Channels.newChannel(website.openStream()); 111 | FileOutputStream fos = new FileOutputStream(destination); 112 | fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); 113 | fos.close(); 114 | rbc.close(); 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-stream-binder-kafka/31ea10683422d38d8185c6b236b8954200562b77/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip 2 | wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar 3 | -------------------------------------------------------------------------------- /.settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | repo.spring.io 6 | ${env.CI_DEPLOY_USERNAME} 7 | ${env.CI_DEPLOY_PASSWORD} 8 | 9 | 10 | 11 | 12 | 18 | spring 19 | true 20 | 21 | 22 | spring-snapshots 23 | Spring Snapshots 24 | https://repo.spring.io/libs-snapshot-local 25 | 26 | true 27 | 28 | 29 | 30 | spring-milestones 31 | Spring Milestones 32 | https://repo.spring.io/libs-milestone-local 33 | 34 | false 35 | 36 | 37 | 38 | spring-releases 39 | Spring Releases 40 | https://repo.spring.io/release 41 | 42 | false 43 | 44 | 45 | 46 | 47 | 48 | spring-snapshots 49 | Spring Snapshots 50 | https://repo.spring.io/libs-snapshot-local 51 | 52 | true 53 | 54 | 55 | 56 | spring-milestones 57 | Spring Milestones 58 | https://repo.spring.io/libs-milestone-local 59 | 60 | false 61 | 62 | 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /docs/.jdk8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-stream-binder-kafka/31ea10683422d38d8185c6b236b8954200562b77/docs/.jdk8 -------------------------------------------------------------------------------- /docs/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | spring-cloud-stream-binder-kafka-docs 7 | 8 | org.springframework.cloud 9 | spring-cloud-stream-binder-kafka-parent 10 | 4.0.0-SNAPSHOT 11 | 12 | jar 13 | spring-cloud-stream-binder-kafka-docs 14 | Spring Cloud Stream Kafka Binder Docs 15 | 16 | spring-cloud-stream-binder-kafka 17 | ${basedir}/.. 18 | 3.4 19 | .*stream.* 20 | deploy 21 | 22 | 23 | 24 | ${project.groupId} 25 | spring-cloud-starter-stream-kafka 26 | ${project.version} 27 | 28 | 29 | 30 | src/main/asciidoc 31 | 32 | 33 | 34 | docs 35 | 36 | 37 | 38 | pl.project13.maven 39 | git-commit-id-plugin 40 | 41 | 42 | maven-dependency-plugin 43 | 44 | 45 | maven-resources-plugin 46 | 47 | 48 | org.codehaus.mojo 49 | exec-maven-plugin 50 | 51 | 52 | org.asciidoctor 53 | asciidoctor-maven-plugin 54 | 55 | 56 | maven-antrun-plugin 57 | 58 | 59 | maven-deploy-plugin 60 | 61 | 62 | 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /docs/src/main/asciidoc/.gitignore: -------------------------------------------------------------------------------- 1 | *.html 2 | *.css 3 | -------------------------------------------------------------------------------- /docs/src/main/asciidoc/Guardfile: -------------------------------------------------------------------------------- 1 | require 'asciidoctor' 2 | require 'erb' 3 | 4 | guard 'shell' do 5 | watch(/.*\.adoc$/) {|m| 6 | Asciidoctor.render_file('index.adoc', \ 7 | :in_place => true, \ 8 | :safe => Asciidoctor::SafeMode::UNSAFE, \ 9 | :attributes=> { \ 10 | 'source-highlighter' => 'prettify', \ 11 | 'icons' => 'font', \ 12 | 'linkcss'=> 'true', \ 13 | 'copycss' => 'true', \ 14 | 'doctype' => 'book'}) 15 | } 16 | end 17 | 18 | guard 'livereload' do 19 | watch(%r{^.+\.(css|js|html)$}) 20 | end 21 | -------------------------------------------------------------------------------- /docs/src/main/asciidoc/README.adoc: -------------------------------------------------------------------------------- 1 | :jdkversion: 1.8 2 | :github-tag: master 3 | :github-repo: spring-cloud/spring-cloud-stream-binder-kafka 4 | 5 | :github-raw: https://raw.githubusercontent.com/{github-repo}/{github-tag} 6 | :github-code: https://github.com/{github-repo}/tree/{github-tag} 7 | 8 | image::https://circleci.com/gh/spring-cloud/spring-cloud-stream-binder-kafka.svg?style=svg["CircleCI", link="https://circleci.com/gh/spring-cloud/spring-cloud-stream-binder-kafka"] 9 | image::https://codecov.io/gh/spring-cloud/spring-cloud-stream-binder-kafka/branch/{github-tag}/graph/badge.svg["codecov", link="https://codecov.io/gh/spring-cloud/spring-cloud-stream-binder-kafka"] 10 | image::https://badges.gitter.im/spring-cloud/spring-cloud-stream-binder-kafka.svg[Gitter, link="https://gitter.im/spring-cloud/spring-cloud-stream-binder-kafka?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge"] 11 | 12 | // ====================================================================================== 13 | 14 | == Apache Kafka Binder 15 | 16 | === Usage 17 | 18 | To use Apache Kafka binder, you need to add `spring-cloud-stream-binder-kafka` as a dependency to your Spring Cloud Stream application, as shown in the following example for Maven: 19 | 20 | [source,xml] 21 | ---- 22 | 23 | org.springframework.cloud 24 | spring-cloud-stream-binder-kafka 25 | 26 | ---- 27 | 28 | Alternatively, you can also use the Spring Cloud Stream Kafka Starter, as shown in the following example for Maven: 29 | 30 | [source,xml] 31 | ---- 32 | 33 | org.springframework.cloud 34 | spring-cloud-starter-stream-kafka 35 | 36 | ---- 37 | 38 | == Apache Kafka Streams Binder 39 | 40 | === Usage 41 | 42 | To use Apache Kafka Streams binder, you need to add `spring-cloud-stream-binder-kafka-streams` as a dependency to your Spring Cloud Stream application, as shown in the following example for Maven: 43 | 44 | [source,xml] 45 | ---- 46 | 47 | org.springframework.cloud 48 | spring-cloud-stream-binder-kafka-streams 49 | 50 | ---- 51 | 52 | = Appendices 53 | [appendix] 54 | include::building.adoc[] 55 | include::contributing.adoc[] 56 | 57 | // ====================================================================================== 58 | -------------------------------------------------------------------------------- /docs/src/main/asciidoc/appendix.adoc: -------------------------------------------------------------------------------- 1 | [[appendix]] 2 | = Appendices 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /docs/src/main/asciidoc/building.adoc: -------------------------------------------------------------------------------- 1 | [[building]] 2 | == Building 3 | 4 | :jdkversion: 1.7 5 | 6 | === Basic Compile and Test 7 | 8 | To build the source you will need to install JDK {jdkversion}. 9 | 10 | The build uses the Maven wrapper so you don't have to install a specific 11 | version of Maven. To enable the tests, you should have Kafka server 0.9 or above running 12 | before building. See below for more information on running the servers. 13 | 14 | The main build command is 15 | 16 | ---- 17 | $ ./mvnw clean install 18 | ---- 19 | 20 | You can also add '-DskipTests' if you like, to avoid running the tests. 21 | 22 | NOTE: You can also install Maven (>=3.3.3) yourself and run the `mvn` command 23 | in place of `./mvnw` in the examples below. If you do that you also 24 | might need to add `-P spring` if your local Maven settings do not 25 | contain repository declarations for spring pre-release artifacts. 26 | 27 | NOTE: Be aware that you might need to increase the amount of memory 28 | available to Maven by setting a `MAVEN_OPTS` environment variable with 29 | a value like `-Xmx512m -XX:MaxPermSize=128m`. We try to cover this in 30 | the `.mvn` configuration, so if you find you have to do it to make a 31 | build succeed, please raise a ticket to get the settings added to 32 | source control. 33 | 34 | 35 | The projects that require middleware generally include a 36 | `docker-compose.yml`, so consider using 37 | https://compose.docker.io/[Docker Compose] to run the middeware servers 38 | in Docker containers. 39 | 40 | === Documentation 41 | 42 | There is a "full" profile that will generate documentation. 43 | 44 | === Working with the code 45 | If you don't have an IDE preference we would recommend that you use 46 | https://www.springsource.com/developer/sts[Spring Tools Suite] or 47 | https://eclipse.org[Eclipse] when working with the code. We use the 48 | https://eclipse.org/m2e/[m2eclipe] eclipse plugin for maven support. Other IDEs and tools 49 | should also work without issue. 50 | 51 | ==== Importing into eclipse with m2eclipse 52 | We recommend the https://eclipse.org/m2e/[m2eclipe] eclipse plugin when working with 53 | eclipse. If you don't already have m2eclipse installed it is available from the "eclipse 54 | marketplace". 55 | 56 | Unfortunately m2e does not yet support Maven 3.3, so once the projects 57 | are imported into Eclipse you will also need to tell m2eclipse to use 58 | the `.settings.xml` file for the projects. If you do not do this you 59 | may see many different errors related to the POMs in the 60 | projects. Open your Eclipse preferences, expand the Maven 61 | preferences, and select User Settings. In the User Settings field 62 | click Browse and navigate to the Spring Cloud project you imported 63 | selecting the `.settings.xml` file in that project. Click Apply and 64 | then OK to save the preference changes. 65 | 66 | NOTE: Alternatively you can copy the repository settings from https://github.com/spring-cloud/spring-cloud-build/blob/master/.settings.xml[`.settings.xml`] into your own `~/.m2/settings.xml`. 67 | 68 | ==== Importing into eclipse without m2eclipse 69 | If you prefer not to use m2eclipse you can generate eclipse project metadata using the 70 | following command: 71 | 72 | [indent=0] 73 | ---- 74 | $ ./mvnw eclipse:eclipse 75 | ---- 76 | 77 | The generated eclipse projects can be imported by selecting `import existing projects` 78 | from the `file` menu. 79 | -------------------------------------------------------------------------------- /docs/src/main/asciidoc/contributing.adoc: -------------------------------------------------------------------------------- 1 | [[contributing] 2 | == Contributing 3 | 4 | Spring Cloud is released under the non-restrictive Apache 2.0 license, 5 | and follows a very standard Github development process, using Github 6 | tracker for issues and merging pull requests into master. If you want 7 | to contribute even something trivial please do not hesitate, but 8 | follow the guidelines below. 9 | 10 | === Sign the Contributor License Agreement 11 | Before we accept a non-trivial patch or pull request we will need you to sign the 12 | https://support.springsource.com/spring_committer_signup[contributor's agreement]. 13 | Signing the contributor's agreement does not grant anyone commit rights to the main 14 | repository, but it does mean that we can accept your contributions, and you will get an 15 | author credit if we do. Active contributors might be asked to join the core team, and 16 | given the ability to merge pull requests. 17 | 18 | === Code Conventions and Housekeeping 19 | None of these is essential for a pull request, but they will all help. They can also be 20 | added after the original pull request but before a merge. 21 | 22 | * Use the Spring Framework code format conventions. If you use Eclipse 23 | you can import formatter settings using the 24 | `eclipse-code-formatter.xml` file from the 25 | https://github.com/spring-cloud/build/tree/master/eclipse-coding-conventions.xml[Spring 26 | Cloud Build] project. If using IntelliJ, you can use the 27 | https://plugins.jetbrains.com/plugin/6546[Eclipse Code Formatter 28 | Plugin] to import the same file. 29 | * Make sure all new `.java` files to have a simple Javadoc class comment with at least an 30 | `@author` tag identifying you, and preferably at least a paragraph on what the class is 31 | for. 32 | * Add the ASF license header comment to all new `.java` files (copy from existing files 33 | in the project) 34 | * Add yourself as an `@author` to the .java files that you modify substantially (more 35 | than cosmetic changes). 36 | * Add some Javadocs and, if you change the namespace, some XSD doc elements. 37 | * A few unit tests would help a lot as well -- someone has to do it. 38 | * If no-one else is using your branch, please rebase it against the current master (or 39 | other target branch in the main project). 40 | * When writing a commit message please follow https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html[these conventions], 41 | if you are fixing an existing issue please add `Fixes gh-XXXX` at the end of the commit 42 | message (where XXXX is the issue number). -------------------------------------------------------------------------------- /docs/src/main/asciidoc/images/kafka-binder.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-stream-binder-kafka/31ea10683422d38d8185c6b236b8954200562b77/docs/src/main/asciidoc/images/kafka-binder.png -------------------------------------------------------------------------------- /docs/src/main/asciidoc/images/kafka-streams-initializr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-stream-binder-kafka/31ea10683422d38d8185c6b236b8954200562b77/docs/src/main/asciidoc/images/kafka-streams-initializr.png -------------------------------------------------------------------------------- /docs/src/main/asciidoc/images/spring-initializr-kafka-streams.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-stream-binder-kafka/31ea10683422d38d8185c6b236b8954200562b77/docs/src/main/asciidoc/images/spring-initializr-kafka-streams.png -------------------------------------------------------------------------------- /docs/src/main/asciidoc/index-docinfo.xml: -------------------------------------------------------------------------------- 1 | Spring Cloud Stream Kafka Binder 2 | {spring-cloud-stream-binder-kafka-version} 3 | 4 | 2013-2016 5 | Pivotal Software, Inc. 6 | 7 | 8 | 9 | Copies of this document may be made for your own use and for distribution to 10 | others, provided that you do not charge any fee for such copies and further 11 | provided that each copy contains this Copyright Notice, whether distributed in 12 | print or electronically. 13 | 14 | 15 | -------------------------------------------------------------------------------- /docs/src/main/asciidoc/partitions.adoc: -------------------------------------------------------------------------------- 1 | === Partitioning with the Kafka Binder 2 | 3 | Apache Kafka supports topic partitioning natively. 4 | 5 | Sometimes it is advantageous to send data to specific partitions -- for example, when you want to strictly order message processing (all messages for a particular customer should go to the same partition). 6 | 7 | The following example shows how to configure the producer and consumer side: 8 | 9 | [source, java] 10 | ---- 11 | @SpringBootApplication 12 | @EnableBinding(Source.class) 13 | public class KafkaPartitionProducerApplication { 14 | 15 | private static final Random RANDOM = new Random(System.currentTimeMillis()); 16 | 17 | private static final String[] data = new String[] { 18 | "foo1", "bar1", "qux1", 19 | "foo2", "bar2", "qux2", 20 | "foo3", "bar3", "qux3", 21 | "foo4", "bar4", "qux4", 22 | }; 23 | 24 | public static void main(String[] args) { 25 | new SpringApplicationBuilder(KafkaPartitionProducerApplication.class) 26 | .web(false) 27 | .run(args); 28 | } 29 | 30 | @InboundChannelAdapter(channel = Source.OUTPUT, poller = @Poller(fixedRate = "5000")) 31 | public Message generate() { 32 | String value = data[RANDOM.nextInt(data.length)]; 33 | System.out.println("Sending: " + value); 34 | return MessageBuilder.withPayload(value) 35 | .setHeader("partitionKey", value) 36 | .build(); 37 | } 38 | 39 | } 40 | ---- 41 | 42 | .application.yml 43 | [source, yaml] 44 | ---- 45 | spring: 46 | cloud: 47 | stream: 48 | bindings: 49 | output: 50 | destination: partitioned.topic 51 | producer: 52 | partition-key-expression: headers['partitionKey'] 53 | partition-count: 12 54 | ---- 55 | 56 | IMPORTANT: The topic must be provisioned to have enough partitions to achieve the desired concurrency for all consumer groups. 57 | The above configuration supports up to 12 consumer instances (6 if their `concurrency` is 2, 4 if their concurrency is 3, and so on). 58 | It is generally best to "`over-provision`" the partitions to allow for future increases in consumers or concurrency. 59 | 60 | NOTE: The preceding configuration uses the default partitioning (`key.hashCode() % partitionCount`). 61 | This may or may not provide a suitably balanced algorithm, depending on the key values. 62 | You can override this default by using the `partitionSelectorExpression` or `partitionSelectorClass` properties. 63 | 64 | Since partitions are natively handled by Kafka, no special configuration is needed on the consumer side. 65 | Kafka allocates partitions across the instances. 66 | 67 | The following Spring Boot application listens to a Kafka stream and prints (to the console) the partition ID to which each message goes: 68 | 69 | [source, java] 70 | ---- 71 | @SpringBootApplication 72 | @EnableBinding(Sink.class) 73 | public class KafkaPartitionConsumerApplication { 74 | 75 | public static void main(String[] args) { 76 | new SpringApplicationBuilder(KafkaPartitionConsumerApplication.class) 77 | .web(false) 78 | .run(args); 79 | } 80 | 81 | @StreamListener(Sink.INPUT) 82 | public void listen(@Payload String in, @Header(KafkaHeaders.RECEIVED_PARTITION_ID) int partition) { 83 | System.out.println(in + " received from partition " + partition); 84 | } 85 | 86 | } 87 | ---- 88 | 89 | .application.yml 90 | [source, yaml] 91 | ---- 92 | spring: 93 | cloud: 94 | stream: 95 | bindings: 96 | input: 97 | destination: partitioned.topic 98 | group: myGroup 99 | ---- 100 | 101 | You can add instances as needed. 102 | Kafka rebalances the partition allocations. 103 | If the instance count (or `instance count * concurrency`) exceeds the number of partitions, some consumers are idle. 104 | -------------------------------------------------------------------------------- /docs/src/main/asciidoc/spring-cloud-stream-binder-kafka-aggregate.adoc: -------------------------------------------------------------------------------- 1 | include::overview.adoc[leveloffset=+1] 2 | include::dlq.adoc[leveloffset=+1] 3 | include::partitions.adoc[leveloffset=+1] 4 | -------------------------------------------------------------------------------- /docs/src/main/asciidoc/spring-cloud-stream-binder-kafka.adoc: -------------------------------------------------------------------------------- 1 | :github-tag: master 2 | :github-repo: spring-cloud/spring-cloud-stream-binder-kafka 3 | :github-raw: https://raw.githubusercontent.com/{github-repo}/{github-tag} 4 | :github-code: https://github.com/{github-repo}/tree/{github-tag} 5 | :toc: left 6 | :toclevels: 8 7 | :nofooter: 8 | :sectlinks: true 9 | 10 | 11 | [[spring-cloud-stream-binder-kafka-reference]] 12 | = Spring Cloud Stream Kafka Binder Reference Guide 13 | Sabby Anandan, Marius Bogoevici, Eric Bottard, Mark Fisher, Ilayaperumal Gopinathan, Gunnar Hillert, Mark Pollack, Patrick Peralta, Glenn Renfro, Thomas Risberg, Dave Syer, David Turanski, Janne Valkealahti, Benjamin Klein, Henryk Konsek, Gary Russell, Arnaud Jardiné, Soby Chacko 14 | :doctype: book 15 | :toc: 16 | :toclevels: 4 17 | :source-highlighter: prettify 18 | :numbered: 19 | :icons: font 20 | :hide-uri-scheme: 21 | :spring-cloud-stream-binder-kafka-repo: snapshot 22 | :github-tag: master 23 | :spring-cloud-stream-binder-kafka-docs-version: current 24 | :spring-cloud-stream-binder-kafka-docs: https://docs.spring.io/spring-cloud-stream-binder-kafka/docs/{spring-cloud-stream-binder-kafka-docs-version}/reference 25 | :spring-cloud-stream-binder-kafka-docs-current: https://docs.spring.io/spring-cloud-stream-binder-kafka/docs/current-SNAPSHOT/reference/html/ 26 | :github-repo: spring-cloud/spring-cloud-stream-binder-kafka 27 | :github-raw: https://raw.github.com/{github-repo}/{github-tag} 28 | :github-code: https://github.com/{github-repo}/tree/{github-tag} 29 | :github-wiki: https://github.com/{github-repo}/wiki 30 | :github-master-code: https://github.com/{github-repo}/tree/master 31 | :sc-ext: java 32 | // ====================================================================================== 33 | 34 | 35 | *{project-version}* 36 | 37 | 38 | = Reference Guide 39 | include::overview.adoc[] 40 | 41 | include::dlq.adoc[] 42 | 43 | include::partitions.adoc[] 44 | 45 | include::kafka-streams.adoc[] 46 | 47 | include::tips.adoc[] 48 | 49 | = Appendices 50 | [appendix] 51 | include::building.adoc[] 52 | 53 | include::contributing.adoc[] 54 | 55 | // ====================================================================================== 56 | -------------------------------------------------------------------------------- /docs/src/main/ruby/generate_readme.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | base_dir = File.join(File.dirname(__FILE__),'../../..') 4 | src_dir = File.join(base_dir, "/src/main/asciidoc") 5 | require 'asciidoctor' 6 | require 'optparse' 7 | 8 | options = {} 9 | file = "#{src_dir}/README.adoc" 10 | 11 | OptionParser.new do |o| 12 | o.on('-o OUTPUT_FILE', 'Output file (default is stdout)') { |file| options[:to_file] = file unless file=='-' } 13 | o.on('-h', '--help') { puts o; exit } 14 | o.parse! 15 | end 16 | 17 | file = ARGV[0] if ARGV.length>0 18 | 19 | # Copied from https://github.com/asciidoctor/asciidoctor-extensions-lab/blob/master/scripts/asciidoc-coalescer.rb 20 | doc = Asciidoctor.load_file file, safe: :unsafe, header_only: true, attributes: options[:attributes] 21 | header_attr_names = (doc.instance_variable_get :@attributes_modified).to_a 22 | header_attr_names.each {|k| doc.attributes[%(#{k}!)] = '' unless doc.attr? k } 23 | attrs = doc.attributes 24 | attrs['allow-uri-read'] = true 25 | puts attrs 26 | 27 | out = "// Do not edit this file (e.g. go instead to src/main/asciidoc)\n\n" 28 | doc = Asciidoctor.load_file file, safe: :unsafe, parse: false, attributes: attrs 29 | out << doc.reader.read 30 | 31 | unless options[:to_file] 32 | puts out 33 | else 34 | File.open(options[:to_file],'w+') do |file| 35 | file.write(out) 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /spring-cloud-starter-stream-kafka/.jdk8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-stream-binder-kafka/31ea10683422d38d8185c6b236b8954200562b77/spring-cloud-starter-stream-kafka/.jdk8 -------------------------------------------------------------------------------- /spring-cloud-starter-stream-kafka/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | org.springframework.cloud 6 | spring-cloud-stream-binder-kafka-parent 7 | 4.0.0-SNAPSHOT 8 | 9 | spring-cloud-starter-stream-kafka 10 | Spring Cloud Starter Stream Kafka 11 | https://projects.spring.io/spring-cloud 12 | 13 | Pivotal Software, Inc. 14 | https://www.spring.io 15 | 16 | 17 | ${basedir}/../.. 18 | 19 | 20 | 21 | org.springframework.cloud 22 | spring-cloud-stream-binder-kafka 23 | ${project.version} 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-core/.jdk8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-stream-binder-kafka/31ea10683422d38d8185c6b236b8954200562b77/spring-cloud-stream-binder-kafka-core/.jdk8 -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-core/.settings/org.eclipse.jdt.ui.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.ui.ignorelowercasenames=true 3 | org.eclipse.jdt.ui.importorder=java;javax;com;org;org.springframework;ch.qos;\#; 4 | org.eclipse.jdt.ui.ondemandthreshold=99 5 | org.eclipse.jdt.ui.staticondemandthreshold=99 6 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-core/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework.cloud 7 | spring-cloud-stream-binder-kafka-parent 8 | 4.0.0-SNAPSHOT 9 | 10 | spring-cloud-stream-binder-kafka-core 11 | Spring Cloud Stream Kafka Binder Core 12 | https://projects.spring.io/spring-cloud 13 | 14 | Pivotal Software, Inc. 15 | https://www.spring.io 16 | 17 | 18 | 19 | 20 | org.springframework.cloud 21 | spring-cloud-stream 22 | 23 | 24 | org.springframework.integration 25 | spring-integration-kafka 26 | 27 | 28 | org.springframework.boot 29 | spring-boot-configuration-processor 30 | true 31 | 32 | 33 | org.springframework.boot 34 | spring-boot-test 35 | test 36 | 37 | 38 | org.springframework.kafka 39 | spring-kafka-test 40 | test 41 | 42 | 43 | org.springframework.cloud 44 | spring-cloud-stream-binder-test 45 | test 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/JaasLoginModuleConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016-2018 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.properties; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | import javax.security.auth.login.AppConfigurationEntry; 23 | 24 | import org.springframework.kafka.security.jaas.KafkaJaasLoginModuleInitializer; 25 | import org.springframework.util.Assert; 26 | 27 | /** 28 | * Contains properties for setting up an {@link AppConfigurationEntry} that can be used 29 | * for the Kafka or Zookeeper client. 30 | * 31 | * @author Marius Bogoevici 32 | * @author Soby Chacko 33 | */ 34 | public class JaasLoginModuleConfiguration { 35 | 36 | private String loginModule = "com.sun.security.auth.module.Krb5LoginModule"; 37 | 38 | private KafkaJaasLoginModuleInitializer.ControlFlag controlFlag = KafkaJaasLoginModuleInitializer.ControlFlag.REQUIRED; 39 | 40 | private Map options = new HashMap<>(); 41 | 42 | public String getLoginModule() { 43 | return this.loginModule; 44 | } 45 | 46 | public void setLoginModule(String loginModule) { 47 | Assert.notNull(loginModule, "cannot be null"); 48 | this.loginModule = loginModule; 49 | } 50 | 51 | public KafkaJaasLoginModuleInitializer.ControlFlag getControlFlag() { 52 | return this.controlFlag; 53 | } 54 | 55 | public void setControlFlag(String controlFlag) { 56 | Assert.notNull(controlFlag, "cannot be null"); 57 | this.controlFlag = KafkaJaasLoginModuleInitializer.ControlFlag 58 | .valueOf(controlFlag.toUpperCase()); 59 | } 60 | 61 | public Map getOptions() { 62 | return this.options; 63 | } 64 | 65 | public void setOptions(Map options) { 66 | this.options = options; 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaBindingProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016-2018 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.properties; 18 | 19 | import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider; 20 | 21 | /** 22 | * Container object for Kafka specific extended producer and consumer binding properties. 23 | * 24 | * @author Marius Bogoevici 25 | * @author Oleg Zhurakousky 26 | */ 27 | public class KafkaBindingProperties implements BinderSpecificPropertiesProvider { 28 | 29 | /** 30 | * Consumer specific binding properties. @see {@link KafkaConsumerProperties}. 31 | */ 32 | private KafkaConsumerProperties consumer = new KafkaConsumerProperties(); 33 | 34 | /** 35 | * Producer specific binding properties. @see {@link KafkaProducerProperties}. 36 | */ 37 | private KafkaProducerProperties producer = new KafkaProducerProperties(); 38 | 39 | /** 40 | * @return {@link KafkaConsumerProperties} 41 | * Consumer specific binding properties. @see {@link KafkaConsumerProperties}. 42 | */ 43 | public KafkaConsumerProperties getConsumer() { 44 | return this.consumer; 45 | } 46 | 47 | public void setConsumer(KafkaConsumerProperties consumer) { 48 | this.consumer = consumer; 49 | } 50 | 51 | /** 52 | * @return {@link KafkaProducerProperties} 53 | * Producer specific binding properties. @see {@link KafkaProducerProperties}. 54 | */ 55 | public KafkaProducerProperties getProducer() { 56 | return this.producer; 57 | } 58 | 59 | public void setProducer(KafkaProducerProperties producer) { 60 | this.producer = producer; 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaExtendedBindingProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016-2018 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.properties; 18 | 19 | import java.util.Map; 20 | 21 | import org.springframework.boot.context.properties.ConfigurationProperties; 22 | import org.springframework.cloud.stream.binder.AbstractExtendedBindingProperties; 23 | import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider; 24 | 25 | /** 26 | * Kafka specific extended binding properties class that extends from 27 | * {@link AbstractExtendedBindingProperties}. 28 | * 29 | * @author Marius Bogoevici 30 | * @author Gary Russell 31 | * @author Soby Chacko 32 | * @author Oleg Zhurakousky 33 | */ 34 | @ConfigurationProperties("spring.cloud.stream.kafka") 35 | public class KafkaExtendedBindingProperties extends 36 | AbstractExtendedBindingProperties { 37 | 38 | private static final String DEFAULTS_PREFIX = "spring.cloud.stream.kafka.default"; 39 | 40 | @Override 41 | public String getDefaultsPrefix() { 42 | return DEFAULTS_PREFIX; 43 | } 44 | 45 | @Override 46 | public Map getBindings() { 47 | return this.doGetBindings(); 48 | } 49 | 50 | @Override 51 | public Class getExtendedPropertiesEntryClass() { 52 | return KafkaBindingProperties.class; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/properties/KafkaTopicProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.properties; 18 | 19 | import java.util.HashMap; 20 | import java.util.List; 21 | import java.util.Map; 22 | 23 | /** 24 | * Properties for configuring topics. 25 | * 26 | * @author Aldo Sinanaj 27 | * @since 2.2 28 | * 29 | */ 30 | public class KafkaTopicProperties { 31 | 32 | private Short replicationFactor; 33 | 34 | private Map> replicasAssignments = new HashMap<>(); 35 | 36 | private Map properties = new HashMap<>(); 37 | 38 | public Short getReplicationFactor() { 39 | return replicationFactor; 40 | } 41 | 42 | public void setReplicationFactor(Short replicationFactor) { 43 | this.replicationFactor = replicationFactor; 44 | } 45 | 46 | public Map> getReplicasAssignments() { 47 | return replicasAssignments; 48 | } 49 | 50 | public void setReplicasAssignments(Map> replicasAssignments) { 51 | this.replicasAssignments = replicasAssignments; 52 | } 53 | 54 | public Map getProperties() { 55 | return properties; 56 | } 57 | 58 | public void setProperties(Map properties) { 59 | this.properties = properties; 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/provisioning/AdminClientConfigCustomizer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021-2021 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.provisioning; 18 | 19 | import java.util.Map; 20 | 21 | /** 22 | * Customizer for configuring AdminClient. 23 | * 24 | * @author Soby Chacko 25 | * @since 3.1.2 26 | */ 27 | @FunctionalInterface 28 | public interface AdminClientConfigCustomizer { 29 | 30 | void configure(Map adminClientProperties); 31 | } 32 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/utils/DlqDestinationResolver.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2020 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.utils; 18 | 19 | import java.util.function.BiFunction; 20 | 21 | import org.apache.kafka.clients.consumer.ConsumerRecord; 22 | 23 | /** 24 | * A {@link BiFunction} extension for defining DLQ destination resolvers. 25 | * 26 | * The BiFunction takes the {@link ConsumerRecord} and the exception as inputs 27 | * and returns a topic name as the DLQ. 28 | * 29 | * @author Soby Chacko 30 | * @since 3.0.9 31 | */ 32 | @FunctionalInterface 33 | public interface DlqDestinationResolver extends BiFunction, Exception, String> { 34 | 35 | } 36 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/utils/DlqPartitionFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.utils; 18 | 19 | import org.apache.commons.logging.Log; 20 | import org.apache.kafka.clients.consumer.ConsumerRecord; 21 | 22 | import org.springframework.lang.Nullable; 23 | 24 | /** 25 | * A TriFunction that takes a consumer group, consumer record, and throwable and returns 26 | * which partition to publish to the dead letter topic. Returning {@code null} means Kafka 27 | * will choose the partition. 28 | * 29 | * @author Gary Russell 30 | * @since 3.0 31 | * 32 | */ 33 | @FunctionalInterface 34 | public interface DlqPartitionFunction { 35 | 36 | /** 37 | * Returns the same partition as the original recor. 38 | */ 39 | DlqPartitionFunction ORIGINAL_PARTITION = (group, rec, ex) -> rec.partition(); 40 | 41 | /** 42 | * Returns 0. 43 | */ 44 | DlqPartitionFunction PARTITION_ZERO = (group, rec, ex) -> 0; 45 | 46 | /** 47 | * Apply the function. 48 | * @param group the consumer group. 49 | * @param record the consumer record. 50 | * @param throwable the exception. 51 | * @return the DLQ partition, or null. 52 | */ 53 | @Nullable 54 | Integer apply(String group, ConsumerRecord record, Throwable throwable); 55 | 56 | /** 57 | * Determine the fallback function to use based on the dlq partition count if no 58 | * {@link DlqPartitionFunction} bean is provided. 59 | * @param dlqPartitions the partition count. 60 | * @param logger the logger. 61 | * @return the fallback. 62 | */ 63 | static DlqPartitionFunction determineFallbackFunction(@Nullable Integer dlqPartitions, Log logger) { 64 | if (dlqPartitions == null) { 65 | return ORIGINAL_PARTITION; 66 | } 67 | else if (dlqPartitions > 1) { 68 | logger.error("'dlqPartitions' is > 1 but a custom DlqPartitionFunction bean is not provided"); 69 | return ORIGINAL_PARTITION; 70 | } 71 | else { 72 | return PARTITION_ZERO; 73 | } 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-core/src/main/java/org/springframework/cloud/stream/binder/kafka/utils/KafkaTopicUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.utils; 18 | 19 | import java.io.UnsupportedEncodingException; 20 | 21 | /** 22 | * Utility methods releated to Kafka topics. 23 | * 24 | * @author Soby Chacko 25 | */ 26 | public final class KafkaTopicUtils { 27 | 28 | private KafkaTopicUtils() { 29 | 30 | } 31 | 32 | /** 33 | * Validate topic name. Allowed chars are ASCII alphanumerics, '.', '_' and '-'. 34 | * @param topicName name of the topic 35 | */ 36 | public static void validateTopicName(String topicName) { 37 | try { 38 | byte[] utf8 = topicName.getBytes("UTF-8"); 39 | for (byte b : utf8) { 40 | if (!((b >= 'a') && (b <= 'z') || (b >= 'A') && (b <= 'Z') 41 | || (b >= '0') && (b <= '9') || (b == '.') || (b == '-') 42 | || (b == '_'))) { 43 | throw new IllegalArgumentException( 44 | "Topic name can only have ASCII alphanumerics, '.', '_' and '-', but was: '" 45 | + topicName + "'"); 46 | } 47 | } 48 | } 49 | catch (UnsupportedEncodingException ex) { 50 | throw new AssertionError(ex); // Can't happen 51 | } 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-core/src/test/resources/test.truststore.ks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-stream-binder-kafka/31ea10683422d38d8185c6b236b8954200562b77/spring-cloud-stream-binder-kafka-core/src/test/resources/test.truststore.ks -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-core/src/test/resources/testclient.keystore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-stream-binder-kafka/31ea10683422d38d8185c6b236b8954200562b77/spring-cloud-stream-binder-kafka-core/src/test/resources/testclient.keystore -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-core/src/test/resources/testclient.truststore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-stream-binder-kafka/31ea10683422d38d8185c6b236b8954200562b77/spring-cloud-stream-binder-kafka-core/src/test/resources/testclient.truststore -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/.jdk8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-stream-binder-kafka/31ea10683422d38d8185c6b236b8954200562b77/spring-cloud-stream-binder-kafka-streams/.jdk8 -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | spring-cloud-stream-binder-kafka-streams 6 | jar 7 | spring-cloud-stream-binder-kafka-streams 8 | Kafka Streams Binder Implementation 9 | 10 | 11 | org.springframework.cloud 12 | spring-cloud-stream-binder-kafka-parent 13 | 4.0.0-SNAPSHOT 14 | 15 | 16 | 17 | 1.8.2 18 | 19 | 20 | 21 | 22 | org.springframework.cloud 23 | spring-cloud-stream-binder-kafka-core 24 | 25 | 26 | org.springframework.boot 27 | spring-boot-starter-actuator 28 | true 29 | 30 | 31 | org.springframework.boot 32 | spring-boot-configuration-processor 33 | true 34 | 35 | 36 | org.springframework.boot 37 | spring-boot-autoconfigure 38 | true 39 | 40 | 41 | org.apache.kafka 42 | kafka-streams 43 | 44 | 45 | org.springframework.kafka 46 | spring-kafka 47 | 48 | 49 | org.springframework.boot 50 | spring-boot-test 51 | test 52 | 53 | 54 | org.springframework.kafka 55 | spring-kafka-test 56 | 57 | 58 | org.springframework.cloud 59 | spring-cloud-stream-binder-test 60 | test 61 | 62 | 63 | org.springframework.boot 64 | spring-boot-autoconfigure-processor 65 | true 66 | 67 | 68 | org.apache.kafka 69 | kafka_2.13 70 | 71 | 72 | org.apache.kafka 73 | kafka_2.13 74 | test 75 | 76 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/DeserializationExceptionHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams; 18 | 19 | /** 20 | * Enumeration for various {@link org.apache.kafka.streams.errors.DeserializationExceptionHandler} types. 21 | * 22 | * @author Soby Chacko 23 | * @since 3.0.0 24 | */ 25 | public enum DeserializationExceptionHandler { 26 | 27 | /** 28 | * Deserialization error handler with log and continue. 29 | * See {@link org.apache.kafka.streams.errors.LogAndContinueExceptionHandler} 30 | */ 31 | logAndContinue, 32 | /** 33 | * Deserialization error handler with log and fail. 34 | * See {@link org.apache.kafka.streams.errors.LogAndFailExceptionHandler} 35 | */ 36 | logAndFail, 37 | /** 38 | * Deserialization error handler with DLQ send. 39 | * See {@link org.springframework.kafka.streams.RecoveringDeserializationExceptionHandler} 40 | */ 41 | sendToDlq, 42 | /** 43 | * Deserialization error handler that silently skips the error and continue. 44 | */ 45 | skipAndContinue; 46 | 47 | } 48 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/EncodingDecodingBindAdviceHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams; 18 | 19 | import org.springframework.boot.context.properties.ConfigurationPropertiesBindHandlerAdvisor; 20 | import org.springframework.boot.context.properties.bind.AbstractBindHandler; 21 | import org.springframework.boot.context.properties.bind.BindContext; 22 | import org.springframework.boot.context.properties.bind.BindHandler; 23 | import org.springframework.boot.context.properties.bind.BindResult; 24 | import org.springframework.boot.context.properties.bind.Bindable; 25 | import org.springframework.boot.context.properties.source.ConfigurationPropertyName; 26 | 27 | /** 28 | * {@link ConfigurationPropertiesBindHandlerAdvisor} to detect nativeEncoding/Decoding settings 29 | * provided by the application explicitly. 30 | * 31 | * @author Soby Chacko 32 | * @since 3.0.0 33 | */ 34 | public class EncodingDecodingBindAdviceHandler implements ConfigurationPropertiesBindHandlerAdvisor { 35 | 36 | private boolean encodingSettingProvided; 37 | private boolean decodingSettingProvided; 38 | 39 | public boolean isDecodingSettingProvided() { 40 | return decodingSettingProvided; 41 | } 42 | 43 | public boolean isEncodingSettingProvided() { 44 | return this.encodingSettingProvided; 45 | } 46 | 47 | @Override 48 | public BindHandler apply(BindHandler bindHandler) { 49 | BindHandler handler = new AbstractBindHandler(bindHandler) { 50 | @Override 51 | public Bindable onStart(ConfigurationPropertyName name, 52 | Bindable target, BindContext context) { 53 | final String configName = name.toString(); 54 | if (configName.contains("use") && configName.contains("native") && 55 | (configName.contains("encoding") || configName.contains("decoding"))) { 56 | BindResult result = context.getBinder().bind(name, target); 57 | if (result.isBound()) { 58 | if (configName.contains("encoding")) { 59 | EncodingDecodingBindAdviceHandler.this.encodingSettingProvided = true; 60 | } 61 | else { 62 | EncodingDecodingBindAdviceHandler.this.decodingSettingProvided = true; 63 | } 64 | return target.withExistingValue(result.get()); 65 | } 66 | } 67 | return bindHandler.onStart(name, target, context); 68 | } 69 | }; 70 | return handler; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/ExtendedBindingHandlerMappingsProviderAutoConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2021 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | import org.springframework.boot.autoconfigure.EnableAutoConfiguration; 23 | import org.springframework.boot.context.properties.source.ConfigurationPropertyName; 24 | import org.springframework.cloud.stream.config.BindingHandlerAdvise.MappingsProvider; 25 | import org.springframework.context.annotation.Bean; 26 | import org.springframework.context.annotation.Configuration; 27 | 28 | /** 29 | * {@link EnableAutoConfiguration Auto-configuration} for extended binding metadata for Kafka Streams. 30 | * 31 | * @author Chris Bono 32 | * @since 3.2 33 | */ 34 | @Configuration(proxyBeanMethods = false) 35 | public class ExtendedBindingHandlerMappingsProviderAutoConfiguration { 36 | 37 | @Bean 38 | public MappingsProvider kafkaStreamsExtendedPropertiesDefaultMappingsProvider() { 39 | return () -> { 40 | Map mappings = new HashMap<>(); 41 | mappings.put( 42 | ConfigurationPropertyName.of("spring.cloud.stream.kafka.streams"), 43 | ConfigurationPropertyName.of("spring.cloud.stream.kafka.streams.default")); 44 | mappings.put( 45 | ConfigurationPropertyName.of("spring.cloud.stream.kafka.streams.bindings"), 46 | ConfigurationPropertyName.of("spring.cloud.stream.kafka.streams.default")); 47 | return mappings; 48 | }; 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/GlobalKTableBinderConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2021 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams; 18 | 19 | import java.util.Map; 20 | 21 | import org.springframework.beans.factory.ObjectProvider; 22 | import org.springframework.beans.factory.annotation.Qualifier; 23 | import org.springframework.beans.factory.config.BeanFactoryPostProcessor; 24 | import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; 25 | import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; 26 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 27 | import org.springframework.cloud.stream.binder.kafka.provisioning.AdminClientConfigCustomizer; 28 | import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; 29 | import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; 30 | import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; 31 | import org.springframework.context.ApplicationContext; 32 | import org.springframework.context.annotation.Bean; 33 | import org.springframework.context.annotation.Configuration; 34 | import org.springframework.context.annotation.Import; 35 | 36 | /** 37 | * Configuration for GlobalKTable binder. 38 | * 39 | * @author Soby Chacko 40 | * @since 2.1.0 41 | */ 42 | @Configuration 43 | @Import({ KafkaAutoConfiguration.class, 44 | MultiBinderPropertiesConfiguration.class, 45 | KafkaStreamsBinderHealthIndicatorConfiguration.class, 46 | KafkaStreamsJaasConfiguration.class}) 47 | public class GlobalKTableBinderConfiguration { 48 | 49 | @Bean 50 | public KafkaTopicProvisioner provisioningProvider( 51 | KafkaStreamsBinderConfigurationProperties binderConfigurationProperties, 52 | KafkaProperties kafkaProperties, ObjectProvider adminClientConfigCustomizer) { 53 | return new KafkaTopicProvisioner(binderConfigurationProperties, kafkaProperties, adminClientConfigCustomizer.getIfUnique()); 54 | } 55 | 56 | @Bean 57 | public GlobalKTableBinder GlobalKTableBinder( 58 | KafkaStreamsBinderConfigurationProperties binderConfigurationProperties, 59 | KafkaTopicProvisioner kafkaTopicProvisioner, 60 | KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties, 61 | KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue, 62 | @Qualifier("streamConfigGlobalProperties") Map streamConfigGlobalProperties, 63 | KafkaStreamsRegistry kafkaStreamsRegistry) { 64 | 65 | GlobalKTableBinder globalKTableBinder = new GlobalKTableBinder(binderConfigurationProperties, 66 | kafkaTopicProvisioner, kafkaStreamsBindingInformationCatalogue, kafkaStreamsRegistry); 67 | globalKTableBinder.setKafkaStreamsExtendedBindingProperties( 68 | kafkaStreamsExtendedBindingProperties); 69 | return globalKTableBinder; 70 | } 71 | 72 | @Bean 73 | @ConditionalOnBean(name = "outerContext") 74 | public static BeanFactoryPostProcessor outerContextBeanFactoryPostProcessor() { 75 | return beanFactory -> { 76 | 77 | // It is safe to call getBean("outerContext") here, because this bean is 78 | // registered as first 79 | // and as independent from the parent context. 80 | ApplicationContext outerContext = (ApplicationContext) beanFactory 81 | .getBean("outerContext"); 82 | beanFactory.registerSingleton( 83 | KafkaStreamsExtendedBindingProperties.class.getSimpleName(), 84 | outerContext.getBean(KafkaStreamsExtendedBindingProperties.class)); 85 | beanFactory.registerSingleton( 86 | KafkaStreamsBindingInformationCatalogue.class.getSimpleName(), 87 | outerContext.getBean(KafkaStreamsBindingInformationCatalogue.class)); 88 | beanFactory.registerSingleton( 89 | KafkaStreamsRegistry.class.getSimpleName(), 90 | outerContext.getBean(KafkaStreamsRegistry.class)); 91 | }; 92 | } 93 | 94 | } 95 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/GlobalKTableBoundElementFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams; 18 | 19 | import org.aopalliance.intercept.MethodInterceptor; 20 | import org.aopalliance.intercept.MethodInvocation; 21 | import org.apache.kafka.streams.kstream.GlobalKTable; 22 | 23 | import org.springframework.aop.framework.ProxyFactory; 24 | import org.springframework.cloud.stream.binder.ConsumerProperties; 25 | import org.springframework.cloud.stream.binding.AbstractBindingTargetFactory; 26 | import org.springframework.cloud.stream.config.BindingProperties; 27 | import org.springframework.cloud.stream.config.BindingServiceProperties; 28 | import org.springframework.util.Assert; 29 | 30 | /** 31 | * {@link org.springframework.cloud.stream.binding.BindingTargetFactory} for 32 | * {@link GlobalKTable} 33 | * 34 | * Input bindings are only created as output bindings on GlobalKTable are not allowed. 35 | * 36 | * @author Soby Chacko 37 | * @since 2.1.0 38 | */ 39 | public class GlobalKTableBoundElementFactory 40 | extends AbstractBindingTargetFactory { 41 | 42 | private final BindingServiceProperties bindingServiceProperties; 43 | private final EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler; 44 | private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue; 45 | 46 | GlobalKTableBoundElementFactory(BindingServiceProperties bindingServiceProperties, 47 | EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler, 48 | KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue) { 49 | super(GlobalKTable.class); 50 | this.bindingServiceProperties = bindingServiceProperties; 51 | this.encodingDecodingBindAdviceHandler = encodingDecodingBindAdviceHandler; 52 | this.kafkaStreamsBindingInformationCatalogue = kafkaStreamsBindingInformationCatalogue; 53 | } 54 | 55 | @Override 56 | public GlobalKTable createInput(String name) { 57 | BindingProperties bindingProperties = this.bindingServiceProperties.getBindingProperties(name); 58 | ConsumerProperties consumerProperties = bindingProperties.getConsumer(); 59 | if (consumerProperties == null) { 60 | consumerProperties = this.bindingServiceProperties.getConsumerProperties(name); 61 | consumerProperties.setUseNativeDecoding(true); 62 | } 63 | else { 64 | if (!encodingDecodingBindAdviceHandler.isDecodingSettingProvided()) { 65 | consumerProperties.setUseNativeDecoding(true); 66 | } 67 | } 68 | // Always set multiplex to true in the kafka streams binder 69 | consumerProperties.setMultiplex(true); 70 | 71 | // @checkstyle:off 72 | GlobalKTableBoundElementFactory.GlobalKTableWrapperHandler wrapper = new GlobalKTableBoundElementFactory.GlobalKTableWrapperHandler(); 73 | // @checkstyle:on 74 | ProxyFactory proxyFactory = new ProxyFactory( 75 | GlobalKTableBoundElementFactory.GlobalKTableWrapper.class, 76 | GlobalKTable.class); 77 | proxyFactory.addAdvice(wrapper); 78 | 79 | final GlobalKTable proxy = (GlobalKTable) proxyFactory.getProxy(); 80 | this.kafkaStreamsBindingInformationCatalogue.addBindingNamePerTarget(proxy, name); 81 | return proxy; 82 | } 83 | 84 | @Override 85 | public GlobalKTable createOutput(String name) { 86 | throw new UnsupportedOperationException( 87 | "Outbound operations are not allowed on target type GlobalKTable"); 88 | } 89 | 90 | /** 91 | * Wrapper for GlobalKTable proxy. 92 | */ 93 | public interface GlobalKTableWrapper { 94 | 95 | void wrap(GlobalKTable delegate); 96 | 97 | } 98 | 99 | private static class GlobalKTableWrapperHandler implements 100 | GlobalKTableBoundElementFactory.GlobalKTableWrapper, MethodInterceptor { 101 | 102 | private GlobalKTable delegate; 103 | 104 | public void wrap(GlobalKTable delegate) { 105 | Assert.notNull(delegate, "delegate cannot be null"); 106 | if (this.delegate == null) { 107 | this.delegate = delegate; 108 | } 109 | } 110 | 111 | @Override 112 | public Object invoke(MethodInvocation methodInvocation) throws Throwable { 113 | if (methodInvocation.getMethod().getDeclaringClass() 114 | .equals(GlobalKTable.class)) { 115 | Assert.notNull(this.delegate, 116 | "Trying to prepareConsumerBinding " + methodInvocation.getMethod() 117 | + " but no delegate has been set."); 118 | return methodInvocation.getMethod().invoke(this.delegate, 119 | methodInvocation.getArguments()); 120 | } 121 | else if (methodInvocation.getMethod().getDeclaringClass() 122 | .equals(GlobalKTableBoundElementFactory.GlobalKTableWrapper.class)) { 123 | return methodInvocation.getMethod().invoke(this, 124 | methodInvocation.getArguments()); 125 | } 126 | else { 127 | throw new IllegalStateException( 128 | "Only GlobalKTable method invocations are permitted"); 129 | } 130 | } 131 | 132 | } 133 | 134 | } 135 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KStreamBinderConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017-2021 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams; 18 | 19 | import org.springframework.beans.factory.ObjectProvider; 20 | import org.springframework.beans.factory.config.BeanFactoryPostProcessor; 21 | import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; 22 | import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; 23 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 24 | import org.springframework.cloud.stream.binder.kafka.provisioning.AdminClientConfigCustomizer; 25 | import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; 26 | import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; 27 | import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; 28 | import org.springframework.context.ApplicationContext; 29 | import org.springframework.context.annotation.Bean; 30 | import org.springframework.context.annotation.Configuration; 31 | import org.springframework.context.annotation.Import; 32 | 33 | /** 34 | * Configuration for KStream binder. 35 | * 36 | * @author Marius Bogoevici 37 | * @author Gary Russell 38 | * @author Soby Chacko 39 | */ 40 | @Configuration 41 | @Import({ KafkaAutoConfiguration.class, 42 | MultiBinderPropertiesConfiguration.class, 43 | KafkaStreamsBinderHealthIndicatorConfiguration.class, 44 | KafkaStreamsJaasConfiguration.class}) 45 | public class KStreamBinderConfiguration { 46 | 47 | @Bean 48 | public KafkaTopicProvisioner provisioningProvider( 49 | KafkaStreamsBinderConfigurationProperties kafkaStreamsBinderConfigurationProperties, 50 | KafkaProperties kafkaProperties, ObjectProvider adminClientConfigCustomizer) { 51 | return new KafkaTopicProvisioner(kafkaStreamsBinderConfigurationProperties, 52 | kafkaProperties, adminClientConfigCustomizer.getIfUnique()); 53 | } 54 | 55 | @Bean 56 | public KStreamBinder kStreamBinder( 57 | KafkaStreamsBinderConfigurationProperties binderConfigurationProperties, 58 | KafkaTopicProvisioner kafkaTopicProvisioner, 59 | KafkaStreamsMessageConversionDelegate KafkaStreamsMessageConversionDelegate, 60 | KafkaStreamsBindingInformationCatalogue KafkaStreamsBindingInformationCatalogue, 61 | KeyValueSerdeResolver keyValueSerdeResolver, 62 | KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties, KafkaStreamsRegistry kafkaStreamsRegistry) { 63 | KStreamBinder kStreamBinder = new KStreamBinder(binderConfigurationProperties, 64 | kafkaTopicProvisioner, KafkaStreamsMessageConversionDelegate, 65 | KafkaStreamsBindingInformationCatalogue, keyValueSerdeResolver, kafkaStreamsRegistry); 66 | kStreamBinder.setKafkaStreamsExtendedBindingProperties( 67 | kafkaStreamsExtendedBindingProperties); 68 | return kStreamBinder; 69 | } 70 | 71 | @Bean 72 | @ConditionalOnBean(name = "outerContext") 73 | public static BeanFactoryPostProcessor outerContextBeanFactoryPostProcessor() { 74 | return beanFactory -> { 75 | 76 | // It is safe to call getBean("outerContext") here, because this bean is 77 | // registered as first 78 | // and as independent from the parent context. 79 | ApplicationContext outerContext = (ApplicationContext) beanFactory 80 | .getBean("outerContext"); 81 | beanFactory.registerSingleton( 82 | KafkaStreamsMessageConversionDelegate.class.getSimpleName(), 83 | outerContext.getBean(KafkaStreamsMessageConversionDelegate.class)); 84 | beanFactory.registerSingleton( 85 | KafkaStreamsBindingInformationCatalogue.class.getSimpleName(), 86 | outerContext.getBean(KafkaStreamsBindingInformationCatalogue.class)); 87 | beanFactory.registerSingleton(KeyValueSerdeResolver.class.getSimpleName(), 88 | outerContext.getBean(KeyValueSerdeResolver.class)); 89 | beanFactory.registerSingleton( 90 | KafkaStreamsExtendedBindingProperties.class.getSimpleName(), 91 | outerContext.getBean(KafkaStreamsExtendedBindingProperties.class)); 92 | beanFactory.registerSingleton( 93 | KafkaStreamsRegistry.class.getSimpleName(), 94 | outerContext.getBean(KafkaStreamsRegistry.class)); 95 | }; 96 | } 97 | 98 | } 99 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KTableBinderConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2021 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams; 18 | 19 | import java.util.Map; 20 | 21 | import org.springframework.beans.factory.ObjectProvider; 22 | import org.springframework.beans.factory.annotation.Qualifier; 23 | import org.springframework.beans.factory.config.BeanFactoryPostProcessor; 24 | import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; 25 | import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; 26 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 27 | import org.springframework.cloud.stream.binder.kafka.provisioning.AdminClientConfigCustomizer; 28 | import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; 29 | import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; 30 | import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; 31 | import org.springframework.context.ApplicationContext; 32 | import org.springframework.context.annotation.Bean; 33 | import org.springframework.context.annotation.Configuration; 34 | import org.springframework.context.annotation.Import; 35 | 36 | /** 37 | * Configuration for KTable binder. 38 | * 39 | * @author Soby Chacko 40 | */ 41 | @SuppressWarnings("ALL") 42 | @Configuration 43 | @Import({ KafkaAutoConfiguration.class, 44 | MultiBinderPropertiesConfiguration.class, 45 | KafkaStreamsBinderHealthIndicatorConfiguration.class, 46 | KafkaStreamsJaasConfiguration.class}) 47 | public class KTableBinderConfiguration { 48 | 49 | @Bean 50 | public KafkaTopicProvisioner provisioningProvider( 51 | KafkaStreamsBinderConfigurationProperties binderConfigurationProperties, 52 | KafkaProperties kafkaProperties, ObjectProvider adminClientConfigCustomizer) { 53 | return new KafkaTopicProvisioner(binderConfigurationProperties, kafkaProperties, adminClientConfigCustomizer.getIfUnique()); 54 | } 55 | 56 | @Bean 57 | public KTableBinder kTableBinder( 58 | KafkaStreamsBinderConfigurationProperties binderConfigurationProperties, 59 | KafkaTopicProvisioner kafkaTopicProvisioner, 60 | KafkaStreamsExtendedBindingProperties kafkaStreamsExtendedBindingProperties, 61 | KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue, 62 | @Qualifier("streamConfigGlobalProperties") Map streamConfigGlobalProperties, 63 | KafkaStreamsRegistry kafkaStreamsRegistry) { 64 | KTableBinder kTableBinder = new KTableBinder(binderConfigurationProperties, 65 | kafkaTopicProvisioner, kafkaStreamsBindingInformationCatalogue, kafkaStreamsRegistry); 66 | kTableBinder.setKafkaStreamsExtendedBindingProperties(kafkaStreamsExtendedBindingProperties); 67 | return kTableBinder; 68 | } 69 | 70 | @Bean 71 | @ConditionalOnBean(name = "outerContext") 72 | public static BeanFactoryPostProcessor outerContextBeanFactoryPostProcessor() { 73 | return beanFactory -> { 74 | 75 | // It is safe to call getBean("outerContext") here, because this bean is 76 | // registered as first 77 | // and as independent from the parent context. 78 | ApplicationContext outerContext = (ApplicationContext) beanFactory 79 | .getBean("outerContext"); 80 | beanFactory.registerSingleton( 81 | KafkaStreamsExtendedBindingProperties.class.getSimpleName(), 82 | outerContext.getBean(KafkaStreamsExtendedBindingProperties.class)); 83 | beanFactory.registerSingleton( 84 | KafkaStreamsBindingInformationCatalogue.class.getSimpleName(), 85 | outerContext.getBean(KafkaStreamsBindingInformationCatalogue.class)); 86 | beanFactory.registerSingleton( 87 | KafkaStreamsRegistry.class.getSimpleName(), 88 | outerContext.getBean(KafkaStreamsRegistry.class)); 89 | }; 90 | } 91 | 92 | 93 | } 94 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KTableBoundElementFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams; 18 | 19 | import org.aopalliance.intercept.MethodInterceptor; 20 | import org.aopalliance.intercept.MethodInvocation; 21 | import org.apache.kafka.streams.kstream.KTable; 22 | 23 | import org.springframework.aop.framework.ProxyFactory; 24 | import org.springframework.cloud.stream.binder.ConsumerProperties; 25 | import org.springframework.cloud.stream.binding.AbstractBindingTargetFactory; 26 | import org.springframework.cloud.stream.config.BindingProperties; 27 | import org.springframework.cloud.stream.config.BindingServiceProperties; 28 | import org.springframework.util.Assert; 29 | 30 | /** 31 | * {@link org.springframework.cloud.stream.binding.BindingTargetFactory} for 32 | * {@link KTable} 33 | * 34 | * Input bindings are only created as output bindings on KTable are not allowed. 35 | * 36 | * @author Soby Chacko 37 | */ 38 | class KTableBoundElementFactory extends AbstractBindingTargetFactory { 39 | 40 | private final BindingServiceProperties bindingServiceProperties; 41 | private final EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler; 42 | private final KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue; 43 | 44 | KTableBoundElementFactory(BindingServiceProperties bindingServiceProperties, 45 | EncodingDecodingBindAdviceHandler encodingDecodingBindAdviceHandler, 46 | KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue) { 47 | super(KTable.class); 48 | this.bindingServiceProperties = bindingServiceProperties; 49 | this.encodingDecodingBindAdviceHandler = encodingDecodingBindAdviceHandler; 50 | this.kafkaStreamsBindingInformationCatalogue = kafkaStreamsBindingInformationCatalogue; 51 | } 52 | 53 | @Override 54 | public KTable createInput(String name) { 55 | BindingProperties bindingProperties = this.bindingServiceProperties.getBindingProperties(name); 56 | ConsumerProperties consumerProperties = bindingProperties.getConsumer(); 57 | if (consumerProperties == null) { 58 | consumerProperties = this.bindingServiceProperties.getConsumerProperties(name); 59 | consumerProperties.setUseNativeDecoding(true); 60 | } 61 | else { 62 | if (!encodingDecodingBindAdviceHandler.isDecodingSettingProvided()) { 63 | consumerProperties.setUseNativeDecoding(true); 64 | } 65 | } 66 | // Always set multiplex to true in the kafka streams binder 67 | consumerProperties.setMultiplex(true); 68 | 69 | KTableBoundElementFactory.KTableWrapperHandler wrapper = new KTableBoundElementFactory.KTableWrapperHandler(); 70 | ProxyFactory proxyFactory = new ProxyFactory( 71 | KTableBoundElementFactory.KTableWrapper.class, KTable.class); 72 | proxyFactory.addAdvice(wrapper); 73 | 74 | final KTable proxy = (KTable) proxyFactory.getProxy(); 75 | this.kafkaStreamsBindingInformationCatalogue.addBindingNamePerTarget(proxy, name); 76 | return proxy; 77 | } 78 | 79 | @Override 80 | @SuppressWarnings("unchecked") 81 | public KTable createOutput(final String name) { 82 | throw new UnsupportedOperationException( 83 | "Outbound operations are not allowed on target type KTable"); 84 | } 85 | 86 | /** 87 | * Wrapper for KTable proxy. 88 | */ 89 | public interface KTableWrapper { 90 | 91 | void wrap(KTable delegate); 92 | 93 | } 94 | 95 | private static class KTableWrapperHandler 96 | implements KTableBoundElementFactory.KTableWrapper, MethodInterceptor { 97 | 98 | private KTable delegate; 99 | 100 | public void wrap(KTable delegate) { 101 | Assert.notNull(delegate, "delegate cannot be null"); 102 | if (this.delegate == null) { 103 | this.delegate = delegate; 104 | } 105 | } 106 | 107 | @Override 108 | public Object invoke(MethodInvocation methodInvocation) throws Throwable { 109 | if (methodInvocation.getMethod().getDeclaringClass().equals(KTable.class)) { 110 | Assert.notNull(this.delegate, 111 | "Trying to prepareConsumerBinding " + methodInvocation.getMethod() 112 | + " but no delegate has been set."); 113 | return methodInvocation.getMethod().invoke(this.delegate, 114 | methodInvocation.getArguments()); 115 | } 116 | else if (methodInvocation.getMethod().getDeclaringClass() 117 | .equals(KTableBoundElementFactory.KTableWrapper.class)) { 118 | return methodInvocation.getMethod().invoke(this, 119 | methodInvocation.getArguments()); 120 | } 121 | else { 122 | throw new IllegalStateException( 123 | "Only KTable method invocations are permitted"); 124 | } 125 | } 126 | 127 | } 128 | 129 | } 130 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsBinderHealthIndicatorConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams; 18 | 19 | import org.springframework.beans.factory.ObjectProvider; 20 | import org.springframework.beans.factory.annotation.Qualifier; 21 | import org.springframework.boot.actuate.autoconfigure.health.ConditionalOnEnabledHealthIndicator; 22 | import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; 23 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 24 | import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; 25 | import org.springframework.context.annotation.Bean; 26 | import org.springframework.context.annotation.Configuration; 27 | 28 | /** 29 | * Configuration class for Kafka-streams binder health indicator beans. 30 | * 31 | * @author Arnaud Jardiné 32 | */ 33 | @Configuration 34 | @ConditionalOnClass(name = "org.springframework.boot.actuate.health.HealthIndicator") 35 | @ConditionalOnEnabledHealthIndicator("binders") 36 | public class KafkaStreamsBinderHealthIndicatorConfiguration { 37 | 38 | @Bean 39 | public KafkaStreamsBinderHealthIndicator kafkaStreamsBinderHealthIndicator( 40 | ObjectProvider kafkaStreamsRegistry, 41 | @Qualifier("binderConfigurationProperties")KafkaStreamsBinderConfigurationProperties kafkaStreamsBinderConfigurationProperties, 42 | KafkaProperties kafkaProperties, KafkaStreamsBindingInformationCatalogue kafkaStreamsBindingInformationCatalogue) { 43 | if (kafkaStreamsRegistry.getIfUnique() != null) { 44 | return new KafkaStreamsBinderHealthIndicator(kafkaStreamsRegistry.getIfUnique(), kafkaStreamsBinderConfigurationProperties, 45 | kafkaProperties, kafkaStreamsBindingInformationCatalogue); 46 | } 47 | return null; 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsJaasConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams; 18 | 19 | import java.io.IOException; 20 | 21 | import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; 22 | import org.springframework.cloud.stream.binder.kafka.properties.JaasLoginModuleConfiguration; 23 | import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; 24 | import org.springframework.context.annotation.Bean; 25 | import org.springframework.context.annotation.Configuration; 26 | import org.springframework.kafka.security.jaas.KafkaJaasLoginModuleInitializer; 27 | 28 | /** 29 | * Jaas configuration bean for Kafka Streams binder types. 30 | * 31 | * @author Soby Chacko 32 | * @since 3.1.4 33 | */ 34 | @Configuration 35 | public class KafkaStreamsJaasConfiguration { 36 | 37 | @Bean 38 | @ConditionalOnMissingBean(KafkaJaasLoginModuleInitializer.class) 39 | public KafkaJaasLoginModuleInitializer jaasInitializer( 40 | KafkaBinderConfigurationProperties configurationProperties) 41 | throws IOException { 42 | KafkaJaasLoginModuleInitializer kafkaJaasLoginModuleInitializer = new KafkaJaasLoginModuleInitializer(); 43 | JaasLoginModuleConfiguration jaas = configurationProperties.getJaas(); 44 | if (jaas != null) { 45 | kafkaJaasLoginModuleInitializer.setLoginModule(jaas.getLoginModule()); 46 | 47 | KafkaJaasLoginModuleInitializer.ControlFlag controlFlag = jaas 48 | .getControlFlag(); 49 | 50 | if (controlFlag != null) { 51 | kafkaJaasLoginModuleInitializer.setControlFlag(controlFlag); 52 | } 53 | kafkaJaasLoginModuleInitializer.setOptions(jaas.getOptions()); 54 | } 55 | return kafkaJaasLoginModuleInitializer; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/KafkaStreamsRegistry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams; 18 | 19 | import java.util.ArrayList; 20 | import java.util.HashSet; 21 | import java.util.List; 22 | import java.util.Map; 23 | import java.util.Optional; 24 | import java.util.Set; 25 | import java.util.concurrent.ConcurrentHashMap; 26 | 27 | import org.apache.kafka.streams.KafkaStreams; 28 | import org.apache.kafka.streams.StreamsConfig; 29 | 30 | import org.springframework.kafka.config.StreamsBuilderFactoryBean; 31 | 32 | /** 33 | * An internal registry for holding {@link KafkaStreams} objects maintained through 34 | * {@link StreamsBuilderFactoryManager}. 35 | * 36 | * @author Soby Chacko 37 | */ 38 | public class KafkaStreamsRegistry { 39 | 40 | private final Map streamsBuilderFactoryBeanMap = new ConcurrentHashMap<>(); 41 | 42 | private final Set kafkaStreams = ConcurrentHashMap.newKeySet(); 43 | 44 | Set getKafkaStreams() { 45 | Set currentlyRunningKafkaStreams = new HashSet<>(); 46 | for (KafkaStreams ks : this.kafkaStreams) { 47 | final StreamsBuilderFactoryBean streamsBuilderFactoryBean = streamsBuilderFactoryBeanMap.get(ks); 48 | if (streamsBuilderFactoryBean.isRunning()) { 49 | currentlyRunningKafkaStreams.add(ks); 50 | } 51 | } 52 | return currentlyRunningKafkaStreams; 53 | } 54 | 55 | /** 56 | * Register the {@link KafkaStreams} object created in the application. 57 | * @param streamsBuilderFactoryBean {@link StreamsBuilderFactoryBean} 58 | */ 59 | void registerKafkaStreams(StreamsBuilderFactoryBean streamsBuilderFactoryBean) { 60 | final KafkaStreams kafkaStreams = streamsBuilderFactoryBean.getKafkaStreams(); 61 | this.kafkaStreams.add(kafkaStreams); 62 | this.streamsBuilderFactoryBeanMap.put(kafkaStreams, streamsBuilderFactoryBean); 63 | } 64 | 65 | void unregisterKafkaStreams(KafkaStreams kafkaStreams) { 66 | this.kafkaStreams.remove(kafkaStreams); 67 | this.streamsBuilderFactoryBeanMap.remove(kafkaStreams); 68 | } 69 | 70 | /** 71 | * 72 | * @param kafkaStreams {@link KafkaStreams} object 73 | * @return Corresponding {@link StreamsBuilderFactoryBean}. 74 | */ 75 | StreamsBuilderFactoryBean streamBuilderFactoryBean(KafkaStreams kafkaStreams) { 76 | return this.streamsBuilderFactoryBeanMap.get(kafkaStreams); 77 | } 78 | 79 | public StreamsBuilderFactoryBean streamsBuilderFactoryBean(String applicationId) { 80 | final Optional first = this.streamsBuilderFactoryBeanMap.values() 81 | .stream() 82 | .filter(streamsBuilderFactoryBean -> streamsBuilderFactoryBean.isRunning() && streamsBuilderFactoryBean 83 | .getStreamsConfiguration().getProperty(StreamsConfig.APPLICATION_ID_CONFIG) 84 | .equals(applicationId)) 85 | .findFirst(); 86 | return first.orElse(null); 87 | } 88 | 89 | public List streamsBuilderFactoryBeans() { 90 | return new ArrayList<>(this.streamsBuilderFactoryBeanMap.values()); 91 | } 92 | 93 | } 94 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/MultiBinderPropertiesConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams; 18 | 19 | import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; 20 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 21 | import org.springframework.boot.context.properties.ConfigurationProperties; 22 | import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; 23 | import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsBinderConfigurationProperties; 24 | import org.springframework.context.annotation.Bean; 25 | import org.springframework.context.annotation.Configuration; 26 | 27 | /** 28 | * @author Soby Chacko 29 | * @since 3.0.2 30 | */ 31 | @Configuration 32 | public class MultiBinderPropertiesConfiguration { 33 | 34 | @Bean 35 | @ConfigurationProperties(prefix = "spring.cloud.stream.kafka.streams.binder") 36 | @ConditionalOnBean(name = "outerContext") 37 | public KafkaBinderConfigurationProperties binderConfigurationProperties(KafkaProperties kafkaProperties) { 38 | return new KafkaStreamsBinderConfigurationProperties(kafkaProperties); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/SendToDlqAndContinue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | import org.apache.kafka.clients.consumer.ConsumerRecord; 23 | 24 | import org.springframework.kafka.listener.ConsumerRecordRecoverer; 25 | import org.springframework.kafka.listener.DeadLetterPublishingRecoverer; 26 | 27 | /** 28 | * Custom implementation for {@link ConsumerRecordRecoverer} that keeps a collection of 29 | * recoverer objects per input topics. These topics might be per input binding or multiplexed 30 | * topics in a single binding. 31 | * 32 | * @author Soby Chacko 33 | * @since 2.0.0 34 | */ 35 | public class SendToDlqAndContinue implements ConsumerRecordRecoverer { 36 | 37 | /** 38 | * DLQ dispatcher per topic in the application context. The key here is not the actual 39 | * DLQ topic but the incoming topic that caused the error. 40 | */ 41 | private Map dlqDispatchers = new HashMap<>(); 42 | 43 | /** 44 | * For a given topic, send the key/value record to DLQ topic. 45 | * 46 | * @param consumerRecord consumer record 47 | * @param exception exception 48 | */ 49 | public void sendToDlq(ConsumerRecord consumerRecord, Exception exception) { 50 | DeadLetterPublishingRecoverer kafkaStreamsDlqDispatch = this.dlqDispatchers.get(consumerRecord.topic()); 51 | kafkaStreamsDlqDispatch.accept(consumerRecord, exception); 52 | } 53 | 54 | void addKStreamDlqDispatch(String topic, 55 | DeadLetterPublishingRecoverer kafkaStreamsDlqDispatch) { 56 | this.dlqDispatchers.put(topic, kafkaStreamsDlqDispatch); 57 | } 58 | 59 | @Override 60 | public void accept(ConsumerRecord consumerRecord, Exception e) { 61 | this.dlqDispatchers.get(consumerRecord.topic()).accept(consumerRecord, e); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/SkipAndContinueExceptionHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021-2021 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams; 18 | 19 | import java.util.Map; 20 | 21 | import org.apache.kafka.clients.consumer.ConsumerRecord; 22 | import org.apache.kafka.streams.errors.DeserializationExceptionHandler; 23 | import org.apache.kafka.streams.processor.ProcessorContext; 24 | 25 | /** 26 | * 27 | * {@link DeserializationExceptionHandler} that allows to silently skip 28 | * deserialization exceptions and continue processing. 29 | * 30 | * @author Soby Chakco 31 | * @since 3.1.2 32 | */ 33 | public class SkipAndContinueExceptionHandler implements DeserializationExceptionHandler { 34 | 35 | @Override 36 | public DeserializationExceptionHandler.DeserializationHandlerResponse handle(final ProcessorContext context, 37 | final ConsumerRecord record, 38 | final Exception exception) { 39 | return DeserializationExceptionHandler.DeserializationHandlerResponse.CONTINUE; 40 | } 41 | 42 | @Override 43 | public void configure(final Map configs) { 44 | // ignore 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/endpoint/KafkaStreamsTopologyEndpoint.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2020 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams.endpoint; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | import org.springframework.boot.actuate.endpoint.annotation.Endpoint; 23 | import org.springframework.boot.actuate.endpoint.annotation.ReadOperation; 24 | import org.springframework.boot.actuate.endpoint.annotation.Selector; 25 | import org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsRegistry; 26 | import org.springframework.kafka.config.StreamsBuilderFactoryBean; 27 | import org.springframework.util.StringUtils; 28 | 29 | /** 30 | * Actuator endpoint for topology description. 31 | * 32 | * @author Soby Chacko 33 | * @since 3.0.4 34 | */ 35 | @Endpoint(id = "kafkastreamstopology") 36 | public class KafkaStreamsTopologyEndpoint { 37 | 38 | /** 39 | * Topology not found message. 40 | */ 41 | public static final String NO_TOPOLOGY_FOUND_MSG = "No topology found for the given application ID"; 42 | 43 | private final KafkaStreamsRegistry kafkaStreamsRegistry; 44 | 45 | public KafkaStreamsTopologyEndpoint(KafkaStreamsRegistry kafkaStreamsRegistry) { 46 | this.kafkaStreamsRegistry = kafkaStreamsRegistry; 47 | } 48 | 49 | @ReadOperation 50 | public List kafkaStreamsTopologies() { 51 | final List streamsBuilderFactoryBeans = this.kafkaStreamsRegistry.streamsBuilderFactoryBeans(); 52 | final StringBuilder topologyDescription = new StringBuilder(); 53 | final List descs = new ArrayList<>(); 54 | streamsBuilderFactoryBeans.stream() 55 | .forEach(streamsBuilderFactoryBean -> 56 | descs.add(streamsBuilderFactoryBean.getTopology().describe().toString())); 57 | return descs; 58 | } 59 | 60 | @ReadOperation 61 | public String kafkaStreamsTopology(@Selector String applicationId) { 62 | if (!StringUtils.isEmpty(applicationId)) { 63 | final StreamsBuilderFactoryBean streamsBuilderFactoryBean = this.kafkaStreamsRegistry.streamsBuilderFactoryBean(applicationId); 64 | if (streamsBuilderFactoryBean != null) { 65 | return streamsBuilderFactoryBean.getTopology().describe().toString(); 66 | } 67 | else { 68 | return NO_TOPOLOGY_FOUND_MSG; 69 | } 70 | } 71 | return NO_TOPOLOGY_FOUND_MSG; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/endpoint/KafkaStreamsTopologyEndpointAutoConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2020 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams.endpoint; 18 | 19 | import org.springframework.boot.actuate.autoconfigure.endpoint.EndpointAutoConfiguration; 20 | import org.springframework.boot.actuate.autoconfigure.endpoint.condition.ConditionalOnAvailableEndpoint; 21 | import org.springframework.boot.autoconfigure.AutoConfigureAfter; 22 | import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; 23 | import org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsBinderSupportAutoConfiguration; 24 | import org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsRegistry; 25 | import org.springframework.context.annotation.Bean; 26 | import org.springframework.context.annotation.Configuration; 27 | 28 | /** 29 | * @author Soby Chacko 30 | * @since 3.0.4 31 | */ 32 | @Configuration 33 | @ConditionalOnClass(name = { 34 | "org.springframework.boot.actuate.endpoint.annotation.Endpoint" }) 35 | @AutoConfigureAfter({EndpointAutoConfiguration.class, KafkaStreamsBinderSupportAutoConfiguration.class}) 36 | public class KafkaStreamsTopologyEndpointAutoConfiguration { 37 | 38 | @Bean 39 | @ConditionalOnAvailableEndpoint 40 | public KafkaStreamsTopologyEndpoint topologyEndpoint(KafkaStreamsRegistry kafkaStreamsRegistry) { 41 | return new KafkaStreamsTopologyEndpoint(kafkaStreamsRegistry); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsFunctionAutoConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams.function; 18 | 19 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 20 | import org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsFunctionProcessor; 21 | import org.springframework.cloud.stream.function.StreamFunctionProperties; 22 | import org.springframework.context.annotation.Bean; 23 | import org.springframework.context.annotation.Conditional; 24 | import org.springframework.context.annotation.Configuration; 25 | 26 | /** 27 | * @author Soby Chacko 28 | * @since 2.2.0 29 | */ 30 | @Configuration 31 | @EnableConfigurationProperties(StreamFunctionProperties.class) 32 | public class KafkaStreamsFunctionAutoConfiguration { 33 | 34 | @Bean 35 | @Conditional(FunctionDetectorCondition.class) 36 | public KafkaStreamsFunctionProcessorInvoker kafkaStreamsFunctionProcessorInvoker( 37 | KafkaStreamsFunctionBeanPostProcessor kafkaStreamsFunctionBeanPostProcessor, 38 | KafkaStreamsFunctionProcessor kafkaStreamsFunctionProcessor, 39 | KafkaStreamsBindableProxyFactory[] kafkaStreamsBindableProxyFactories, 40 | StreamFunctionProperties streamFunctionProperties) { 41 | return new KafkaStreamsFunctionProcessorInvoker(kafkaStreamsFunctionBeanPostProcessor.getResolvableTypes(), 42 | kafkaStreamsFunctionProcessor, kafkaStreamsBindableProxyFactories, kafkaStreamsFunctionBeanPostProcessor.getMethods(), 43 | streamFunctionProperties); 44 | } 45 | 46 | @Bean 47 | @Conditional(FunctionDetectorCondition.class) 48 | public KafkaStreamsFunctionBeanPostProcessor kafkaStreamsFunctionBeanPostProcessor(StreamFunctionProperties streamFunctionProperties) { 49 | return new KafkaStreamsFunctionBeanPostProcessor(streamFunctionProperties); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/function/KafkaStreamsFunctionProcessorInvoker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams.function; 18 | 19 | import java.lang.reflect.Method; 20 | import java.util.Arrays; 21 | import java.util.Map; 22 | import java.util.Optional; 23 | 24 | import javax.annotation.PostConstruct; 25 | 26 | import org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsFunctionProcessor; 27 | import org.springframework.cloud.stream.function.StreamFunctionProperties; 28 | import org.springframework.core.ResolvableType; 29 | import org.springframework.util.StringUtils; 30 | 31 | /** 32 | * @author Soby Chacko 33 | * @since 2.1.0 34 | */ 35 | public class KafkaStreamsFunctionProcessorInvoker { 36 | 37 | private final KafkaStreamsFunctionProcessor kafkaStreamsFunctionProcessor; 38 | private final Map resolvableTypeMap; 39 | private final KafkaStreamsBindableProxyFactory[] kafkaStreamsBindableProxyFactories; 40 | private final Map methods; 41 | private final StreamFunctionProperties streamFunctionProperties; 42 | 43 | public KafkaStreamsFunctionProcessorInvoker(Map resolvableTypeMap, 44 | KafkaStreamsFunctionProcessor kafkaStreamsFunctionProcessor, 45 | KafkaStreamsBindableProxyFactory[] kafkaStreamsBindableProxyFactories, 46 | Map methods, StreamFunctionProperties streamFunctionProperties) { 47 | this.kafkaStreamsFunctionProcessor = kafkaStreamsFunctionProcessor; 48 | this.resolvableTypeMap = resolvableTypeMap; 49 | this.kafkaStreamsBindableProxyFactories = kafkaStreamsBindableProxyFactories; 50 | this.methods = methods; 51 | this.streamFunctionProperties = streamFunctionProperties; 52 | } 53 | 54 | @PostConstruct 55 | void invoke() { 56 | final String definition = streamFunctionProperties.getDefinition(); 57 | final String[] functionUnits = StringUtils.hasText(definition) ? definition.split(";") : new String[]{}; 58 | 59 | if (functionUnits.length == 0) { 60 | resolvableTypeMap.forEach((key, value) -> { 61 | Optional proxyFactory = 62 | Arrays.stream(kafkaStreamsBindableProxyFactories).filter(p -> p.getFunctionName().equals(key)).findFirst(); 63 | this.kafkaStreamsFunctionProcessor.setupFunctionInvokerForKafkaStreams(value, key, proxyFactory.get(), methods.get(key), null); 64 | }); 65 | } 66 | 67 | for (String functionUnit : functionUnits) { 68 | if (functionUnit.contains("|")) { 69 | final String[] composedFunctions = functionUnit.split("\\|"); 70 | String[] derivedNameFromComposed = new String[]{""}; 71 | for (String split : composedFunctions) { 72 | derivedNameFromComposed[0] = derivedNameFromComposed[0].concat(split); 73 | } 74 | Optional proxyFactory = 75 | Arrays.stream(kafkaStreamsBindableProxyFactories).filter(p -> p.getFunctionName().equals(derivedNameFromComposed[0])).findFirst(); 76 | proxyFactory.ifPresent(kafkaStreamsBindableProxyFactory -> 77 | this.kafkaStreamsFunctionProcessor.setupFunctionInvokerForKafkaStreams(resolvableTypeMap.get(composedFunctions[0]), 78 | derivedNameFromComposed[0], kafkaStreamsBindableProxyFactory, methods.get(derivedNameFromComposed[0]), resolvableTypeMap.get(composedFunctions[composedFunctions.length - 1]), composedFunctions)); 79 | } 80 | else { 81 | Optional proxyFactory = 82 | Arrays.stream(kafkaStreamsBindableProxyFactories).filter(p -> p.getFunctionName().equals(functionUnit)).findFirst(); 83 | proxyFactory.ifPresent(kafkaStreamsBindableProxyFactory -> 84 | this.kafkaStreamsFunctionProcessor.setupFunctionInvokerForKafkaStreams(resolvableTypeMap.get(functionUnit), functionUnit, 85 | kafkaStreamsBindableProxyFactory, methods.get(functionUnit), null)); 86 | } 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsBindingProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams.properties; 18 | 19 | import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider; 20 | 21 | /** 22 | * Extended binding properties holder that delegates to Kafka Streams producer and 23 | * consumer properties. 24 | * 25 | * @author Marius Bogoevici 26 | */ 27 | public class KafkaStreamsBindingProperties implements BinderSpecificPropertiesProvider { 28 | 29 | private KafkaStreamsConsumerProperties consumer = new KafkaStreamsConsumerProperties(); 30 | 31 | private KafkaStreamsProducerProperties producer = new KafkaStreamsProducerProperties(); 32 | 33 | public KafkaStreamsConsumerProperties getConsumer() { 34 | return this.consumer; 35 | } 36 | 37 | public void setConsumer(KafkaStreamsConsumerProperties consumer) { 38 | this.consumer = consumer; 39 | } 40 | 41 | public KafkaStreamsProducerProperties getProducer() { 42 | return this.producer; 43 | } 44 | 45 | public void setProducer(KafkaStreamsProducerProperties producer) { 46 | this.producer = producer; 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsConsumerProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams.properties; 18 | 19 | import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; 20 | import org.springframework.cloud.stream.binder.kafka.streams.DeserializationExceptionHandler; 21 | 22 | /** 23 | * Extended properties for Kafka Streams consumer. 24 | * 25 | * @author Marius Bogoevici 26 | * @author Soby Chacko 27 | */ 28 | public class KafkaStreamsConsumerProperties extends KafkaConsumerProperties { 29 | 30 | private String applicationId; 31 | 32 | /** 33 | * Key serde specified per binding. 34 | */ 35 | private String keySerde; 36 | 37 | /** 38 | * Value serde specified per binding. 39 | */ 40 | private String valueSerde; 41 | 42 | /** 43 | * Materialized as a KeyValueStore. 44 | */ 45 | private String materializedAs; 46 | 47 | /** 48 | * Per input binding deserialization handler. 49 | */ 50 | private DeserializationExceptionHandler deserializationExceptionHandler; 51 | 52 | /** 53 | * {@link org.apache.kafka.streams.processor.TimestampExtractor} bean name to use for this consumer. 54 | */ 55 | private String timestampExtractorBeanName; 56 | 57 | /** 58 | * Comma separated list of supported event types for this binding. 59 | */ 60 | private String eventTypes; 61 | 62 | /** 63 | * Record level header key for event type. 64 | * If the default value is overridden, then that is expected on each record header if eventType based 65 | * routing is enabled on this binding (by setting eventTypes). 66 | */ 67 | private String eventTypeHeaderKey = "event_type"; 68 | 69 | /** 70 | * Custom name for the source component from which the processor is consuming from. 71 | */ 72 | private String consumedAs; 73 | 74 | public String getApplicationId() { 75 | return this.applicationId; 76 | } 77 | 78 | public void setApplicationId(String applicationId) { 79 | this.applicationId = applicationId; 80 | } 81 | 82 | public String getKeySerde() { 83 | return this.keySerde; 84 | } 85 | 86 | public void setKeySerde(String keySerde) { 87 | this.keySerde = keySerde; 88 | } 89 | 90 | public String getValueSerde() { 91 | return this.valueSerde; 92 | } 93 | 94 | public void setValueSerde(String valueSerde) { 95 | this.valueSerde = valueSerde; 96 | } 97 | 98 | public String getMaterializedAs() { 99 | return this.materializedAs; 100 | } 101 | 102 | public void setMaterializedAs(String materializedAs) { 103 | this.materializedAs = materializedAs; 104 | } 105 | 106 | public String getTimestampExtractorBeanName() { 107 | return timestampExtractorBeanName; 108 | } 109 | 110 | public void setTimestampExtractorBeanName(String timestampExtractorBeanName) { 111 | this.timestampExtractorBeanName = timestampExtractorBeanName; 112 | } 113 | 114 | public DeserializationExceptionHandler getDeserializationExceptionHandler() { 115 | return deserializationExceptionHandler; 116 | } 117 | 118 | public void setDeserializationExceptionHandler(DeserializationExceptionHandler deserializationExceptionHandler) { 119 | this.deserializationExceptionHandler = deserializationExceptionHandler; 120 | } 121 | 122 | public String getEventTypes() { 123 | return eventTypes; 124 | } 125 | 126 | public void setEventTypes(String eventTypes) { 127 | this.eventTypes = eventTypes; 128 | } 129 | 130 | public String getEventTypeHeaderKey() { 131 | return this.eventTypeHeaderKey; 132 | } 133 | 134 | public void setEventTypeHeaderKey(String eventTypeHeaderKey) { 135 | this.eventTypeHeaderKey = eventTypeHeaderKey; 136 | } 137 | 138 | public String getConsumedAs() { 139 | return consumedAs; 140 | } 141 | 142 | public void setConsumedAs(String consumedAs) { 143 | this.consumedAs = consumedAs; 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsExtendedBindingProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017-2018 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams.properties; 18 | 19 | import java.util.Map; 20 | 21 | import org.springframework.boot.context.properties.ConfigurationProperties; 22 | import org.springframework.cloud.stream.binder.AbstractExtendedBindingProperties; 23 | import org.springframework.cloud.stream.binder.BinderSpecificPropertiesProvider; 24 | 25 | /** 26 | * Kafka streams specific extended binding properties class that extends from 27 | * {@link AbstractExtendedBindingProperties}. 28 | * 29 | * @author Marius Bogoevici 30 | * @author Oleg Zhurakousky 31 | */ 32 | @ConfigurationProperties("spring.cloud.stream.kafka.streams") 33 | public class KafkaStreamsExtendedBindingProperties 34 | // @checkstyle:off 35 | extends 36 | AbstractExtendedBindingProperties { 37 | 38 | // @checkstyle:on 39 | private static final String DEFAULTS_PREFIX = "spring.cloud.stream.kafka.streams.default"; 40 | 41 | @Override 42 | public String getDefaultsPrefix() { 43 | return DEFAULTS_PREFIX; 44 | } 45 | 46 | @Override 47 | public Map getBindings() { 48 | return this.doGetBindings(); 49 | } 50 | 51 | @Override 52 | public Class getExtendedPropertiesEntryClass() { 53 | return KafkaStreamsBindingProperties.class; 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/java/org/springframework/cloud/stream/binder/kafka/streams/properties/KafkaStreamsProducerProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017-2018 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams.properties; 18 | 19 | import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; 20 | 21 | /** 22 | * Extended properties for Kafka Streams producer. 23 | * 24 | * @author Marius Bogoevici 25 | * @author Soby Chacko 26 | */ 27 | public class KafkaStreamsProducerProperties extends KafkaProducerProperties { 28 | 29 | /** 30 | * Key serde specified per binding. 31 | */ 32 | private String keySerde; 33 | 34 | /** 35 | * Value serde specified per binding. 36 | */ 37 | private String valueSerde; 38 | 39 | /** 40 | * {@link org.apache.kafka.streams.processor.StreamPartitioner} to be used on Kafka Streams producer. 41 | */ 42 | private String streamPartitionerBeanName; 43 | 44 | /** 45 | * Custom name for the sink component to which the processor is producing to. 46 | */ 47 | private String producedAs; 48 | 49 | public String getKeySerde() { 50 | return this.keySerde; 51 | } 52 | 53 | public void setKeySerde(String keySerde) { 54 | this.keySerde = keySerde; 55 | } 56 | 57 | public String getValueSerde() { 58 | return this.valueSerde; 59 | } 60 | 61 | public void setValueSerde(String valueSerde) { 62 | this.valueSerde = valueSerde; 63 | } 64 | 65 | public String getStreamPartitionerBeanName() { 66 | return this.streamPartitionerBeanName; 67 | } 68 | 69 | public void setStreamPartitionerBeanName(String streamPartitionerBeanName) { 70 | this.streamPartitionerBeanName = streamPartitionerBeanName; 71 | } 72 | 73 | public String getProducedAs() { 74 | return producedAs; 75 | } 76 | 77 | public void setProducedAs(String producedAs) { 78 | this.producedAs = producedAs; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/resources/META-INF/spring.binders: -------------------------------------------------------------------------------- 1 | kstream:\ 2 | org.springframework.cloud.stream.binder.kafka.streams.KStreamBinderConfiguration 3 | ktable:\ 4 | org.springframework.cloud.stream.binder.kafka.streams.KTableBinderConfiguration 5 | globalktable:\ 6 | org.springframework.cloud.stream.binder.kafka.streams.GlobalKTableBinderConfiguration 7 | 8 | 9 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/main/resources/META-INF/spring.factories: -------------------------------------------------------------------------------- 1 | org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ 2 | org.springframework.cloud.stream.binder.kafka.streams.ExtendedBindingHandlerMappingsProviderAutoConfiguration,\ 3 | org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsBinderSupportAutoConfiguration,\ 4 | org.springframework.cloud.stream.binder.kafka.streams.function.KafkaStreamsFunctionAutoConfiguration,\ 5 | org.springframework.cloud.stream.binder.kafka.streams.endpoint.KafkaStreamsTopologyEndpointAutoConfiguration 6 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/ExtendedBindingHandlerMappingsProviderAutoConfigurationTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams; 18 | 19 | import org.junit.jupiter.api.Test; 20 | 21 | import org.springframework.boot.autoconfigure.EnableAutoConfiguration; 22 | import org.springframework.boot.test.context.runner.ApplicationContextRunner; 23 | import org.springframework.cloud.stream.binder.kafka.streams.properties.KafkaStreamsExtendedBindingProperties; 24 | 25 | import static org.assertj.core.api.Assertions.assertThat; 26 | 27 | /** 28 | * Tests for {@link ExtendedBindingHandlerMappingsProviderAutoConfiguration}. 29 | */ 30 | class ExtendedBindingHandlerMappingsProviderAutoConfigurationTests { 31 | 32 | private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() 33 | .withUserConfiguration(KafkaStreamsTestApp.class) 34 | .withPropertyValues( 35 | "spring.cloud.stream.kafka.streams.default.consumer.application-id: testApp123", 36 | "spring.cloud.stream.kafka.streams.default.consumer.consumed-as: default-consumer", 37 | "spring.cloud.stream.kafka.streams.default.consumer.materialized-as: default-materializer", 38 | "spring.cloud.stream.kafka.streams.default.producer.produced-as: default-producer", 39 | "spring.cloud.stream.kafka.streams.default.producer.key-serde: default-foo"); 40 | 41 | @Test 42 | void defaultsUsedWhenNoCustomBindingProperties() { 43 | this.contextRunner.run((context) -> { 44 | assertThat(context) 45 | .hasNotFailed() 46 | .hasSingleBean(KafkaStreamsExtendedBindingProperties.class); 47 | KafkaStreamsExtendedBindingProperties extendedBindingProperties = context.getBean(KafkaStreamsExtendedBindingProperties.class); 48 | assertThat(extendedBindingProperties.getExtendedConsumerProperties("process-in-0")) 49 | .hasFieldOrPropertyWithValue("applicationId", "testApp123") 50 | .hasFieldOrPropertyWithValue("consumedAs", "default-consumer") 51 | .hasFieldOrPropertyWithValue("materializedAs", "default-materializer"); 52 | assertThat(extendedBindingProperties.getExtendedProducerProperties("process-out-0")) 53 | .hasFieldOrPropertyWithValue("producedAs", "default-producer") 54 | .hasFieldOrPropertyWithValue("keySerde", "default-foo"); 55 | }); 56 | } 57 | 58 | @Test 59 | void defaultsRespectedWhenCustomBindingProperties() { 60 | this.contextRunner 61 | .withPropertyValues( 62 | "spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.consumed-as: custom-consumer", 63 | "spring.cloud.stream.kafka.streams.bindings.process-out-0.producer.produced-as: custom-producer") 64 | .run((context) -> { 65 | assertThat(context) 66 | .hasNotFailed() 67 | .hasSingleBean(KafkaStreamsExtendedBindingProperties.class); 68 | KafkaStreamsExtendedBindingProperties extendedBindingProperties = context.getBean(KafkaStreamsExtendedBindingProperties.class); 69 | assertThat(extendedBindingProperties.getExtendedConsumerProperties("process-in-0")) 70 | .hasFieldOrPropertyWithValue("applicationId", "testApp123") 71 | .hasFieldOrPropertyWithValue("consumedAs", "custom-consumer") 72 | .hasFieldOrPropertyWithValue("materializedAs", "default-materializer"); 73 | assertThat(extendedBindingProperties.getExtendedProducerProperties("process-out-0")) 74 | .hasFieldOrPropertyWithValue("producedAs", "custom-producer") 75 | .hasFieldOrPropertyWithValue("keySerde", "default-foo"); 76 | }); 77 | } 78 | 79 | @EnableAutoConfiguration 80 | static class KafkaStreamsTestApp { 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/bootstrap/KafkaStreamsBinderJaasInitTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021-2021 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams.bootstrap; 18 | 19 | import java.util.function.Consumer; 20 | 21 | import javax.security.auth.login.AppConfigurationEntry; 22 | 23 | import org.apache.kafka.common.security.JaasUtils; 24 | import org.apache.kafka.streams.kstream.KStream; 25 | import org.junit.AfterClass; 26 | import org.junit.BeforeClass; 27 | import org.junit.ClassRule; 28 | import org.junit.Test; 29 | 30 | import org.springframework.boot.WebApplicationType; 31 | import org.springframework.boot.autoconfigure.SpringBootApplication; 32 | import org.springframework.boot.builder.SpringApplicationBuilder; 33 | import org.springframework.context.ConfigurableApplicationContext; 34 | import org.springframework.context.annotation.Bean; 35 | import org.springframework.kafka.test.rule.EmbeddedKafkaRule; 36 | 37 | import static org.assertj.core.api.Assertions.assertThat; 38 | 39 | public class KafkaStreamsBinderJaasInitTests { 40 | 41 | @ClassRule 42 | public static EmbeddedKafkaRule embeddedKafka = new EmbeddedKafkaRule(1, true, 10); 43 | 44 | private static String JAVA_LOGIN_CONFIG_PARAM_VALUE; 45 | 46 | @BeforeClass 47 | public static void beforeAll() { 48 | JAVA_LOGIN_CONFIG_PARAM_VALUE = System.getProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM); 49 | System.clearProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM); 50 | } 51 | 52 | @AfterClass 53 | public static void afterAll() { 54 | if (JAVA_LOGIN_CONFIG_PARAM_VALUE != null) { 55 | System.setProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM, JAVA_LOGIN_CONFIG_PARAM_VALUE); 56 | } 57 | } 58 | 59 | @Test 60 | public void testKafkaStreamsBinderJaasInitialization() { 61 | ConfigurableApplicationContext applicationContext = new SpringApplicationBuilder( 62 | KafkaStreamsBinderJaasInitTestsApplication.class).web(WebApplicationType.NONE).run( 63 | "--spring.cloud.function.definition=foo", 64 | "--spring.cloud.stream.kafka.streams.bindings.foo-in-0.consumer.application-id" 65 | + "=testKafkaStreamsBinderJaasInitialization-jaas-id", 66 | "--spring.cloud.stream.kafka.streams.binder.jaas.loginModule=org.apache.kafka.common.security.plain.PlainLoginModule", 67 | "--spring.cloud.stream.kafka.streams.binder.jaas.options.username=foo", 68 | "--spring.cloud.stream.kafka.streams.binder.jaas.options.password=bar", 69 | "--spring.cloud.stream.kafka.streams.binder.brokers=" 70 | + embeddedKafka.getEmbeddedKafka().getBrokersAsString()); 71 | javax.security.auth.login.Configuration configuration = javax.security.auth.login.Configuration 72 | .getConfiguration(); 73 | final AppConfigurationEntry[] kafkaConfiguration = configuration 74 | .getAppConfigurationEntry("KafkaClient"); 75 | assertThat(kafkaConfiguration).hasSize(1); 76 | assertThat(kafkaConfiguration[0].getOptions().get("username")).isEqualTo("foo"); 77 | assertThat(kafkaConfiguration[0].getOptions().get("password")).isEqualTo("bar"); 78 | assertThat(kafkaConfiguration[0].getControlFlag()) 79 | .isEqualTo(AppConfigurationEntry.LoginModuleControlFlag.REQUIRED); 80 | applicationContext.close(); 81 | } 82 | 83 | @SpringBootApplication 84 | static class KafkaStreamsBinderJaasInitTestsApplication { 85 | 86 | @Bean 87 | public Consumer> foo() { 88 | return s -> { 89 | // No-op consumer 90 | }; 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/integration/KafkaStreamsBinderDestinationIsPatternTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams.integration; 18 | 19 | import java.util.Map; 20 | import java.util.function.Function; 21 | 22 | import org.apache.kafka.clients.consumer.ConsumerConfig; 23 | import org.apache.kafka.streams.kstream.KStream; 24 | import org.junit.AfterClass; 25 | import org.junit.BeforeClass; 26 | import org.junit.ClassRule; 27 | import org.junit.Test; 28 | 29 | import org.springframework.boot.SpringApplication; 30 | import org.springframework.boot.WebApplicationType; 31 | import org.springframework.boot.autoconfigure.EnableAutoConfiguration; 32 | import org.springframework.context.ConfigurableApplicationContext; 33 | import org.springframework.context.annotation.Bean; 34 | import org.springframework.kafka.core.DefaultKafkaConsumerFactory; 35 | import org.springframework.kafka.core.DefaultKafkaProducerFactory; 36 | import org.springframework.kafka.core.KafkaTemplate; 37 | import org.springframework.kafka.test.EmbeddedKafkaBroker; 38 | import org.springframework.kafka.test.rule.EmbeddedKafkaRule; 39 | import org.springframework.kafka.test.utils.KafkaTestUtils; 40 | 41 | import static org.assertj.core.api.Assertions.assertThat; 42 | 43 | /** 44 | * @author Michael Stoettinger 45 | */ 46 | public class KafkaStreamsBinderDestinationIsPatternTests { 47 | 48 | @ClassRule 49 | public static EmbeddedKafkaRule embeddedKafkaRule = new EmbeddedKafkaRule(1, true, 50 | "in.1", "in.2", "out"); 51 | 52 | private static EmbeddedKafkaBroker embeddedKafka = embeddedKafkaRule 53 | .getEmbeddedKafka(); 54 | 55 | private static org.apache.kafka.clients.consumer.Consumer consumer; 56 | 57 | @BeforeClass 58 | public static void setUp() { 59 | Map consumerProps = KafkaTestUtils.consumerProps("group", "true", 60 | embeddedKafka); 61 | consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); 62 | DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory<>(consumerProps); 63 | consumer = cf.createConsumer(); 64 | embeddedKafka.consumeFromEmbeddedTopics(consumer, "out"); 65 | } 66 | 67 | @AfterClass 68 | public static void tearDown() { 69 | consumer.close(); 70 | } 71 | 72 | @Test 73 | public void test() { 74 | SpringApplication app = new SpringApplication(ConsumingApplication.class); 75 | app.setWebApplicationType(WebApplicationType.NONE); 76 | ConfigurableApplicationContext context = app.run("--server.port=0", 77 | "--spring.cloud.stream.bindings.process-out-0.destination=out", 78 | "--spring.cloud.stream.bindings.process-in-0.destination=in.*", 79 | "--spring.cloud.stream.kafka.streams.bindings.process-in-0.consumer.destinationIsPattern=true", 80 | "--spring.cloud.stream.kafka.streams.binder.brokers=" 81 | + embeddedKafka.getBrokersAsString()); 82 | try { 83 | Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); 84 | DefaultKafkaProducerFactory producerFactory = new DefaultKafkaProducerFactory<>( 85 | senderProps); 86 | KafkaTemplate template = new KafkaTemplate<>(producerFactory, true); 87 | 88 | // send message to both topics that fit the pattern 89 | template.send("in.1", "foo1"); 90 | assertThat(KafkaTestUtils.getSingleRecord(consumer, "out").value()) 91 | .isEqualTo("foo1"); 92 | template.send("in.2", "foo2"); 93 | assertThat(KafkaTestUtils.getSingleRecord(consumer, "out").value()) 94 | .isEqualTo("foo2"); 95 | } 96 | finally { 97 | context.close(); 98 | } 99 | } 100 | 101 | @EnableAutoConfiguration 102 | public static class ConsumingApplication { 103 | 104 | @Bean 105 | public Function, KStream> process() { 106 | return input -> input; 107 | } 108 | 109 | } 110 | 111 | } 112 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/test/java/org/springframework/cloud/stream/binder/kafka/streams/serde/CollectionSerdeTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.streams.serde; 18 | 19 | import java.util.ArrayList; 20 | import java.util.Collection; 21 | import java.util.Iterator; 22 | import java.util.List; 23 | 24 | import org.junit.Test; 25 | 26 | import static org.assertj.core.api.Assertions.assertThat; 27 | 28 | /** 29 | * 30 | * @author Soby Chacko 31 | */ 32 | public class CollectionSerdeTest { 33 | 34 | @Test 35 | public void testCollectionsSerde() { 36 | 37 | Foo foo1 = new Foo(); 38 | foo1.setData("data-1"); 39 | foo1.setNum(1); 40 | 41 | Foo foo2 = new Foo(); 42 | foo2.setData("data-2"); 43 | foo2.setNum(2); 44 | 45 | List foos = new ArrayList<>(); 46 | foos.add(foo1); 47 | foos.add(foo2); 48 | 49 | CollectionSerde collectionSerde = new CollectionSerde<>(Foo.class, ArrayList.class); 50 | byte[] serialized = collectionSerde.serializer().serialize("", foos); 51 | 52 | Collection deserialized = collectionSerde.deserializer().deserialize("", serialized); 53 | 54 | Iterator iterator = deserialized.iterator(); 55 | Foo foo1Retrieved = iterator.next(); 56 | assertThat(foo1Retrieved.getData()).isEqualTo("data-1"); 57 | assertThat(foo1Retrieved.getNum()).isEqualTo(1); 58 | 59 | Foo foo2Retrieved = iterator.next(); 60 | assertThat(foo2Retrieved.getData()).isEqualTo("data-2"); 61 | assertThat(foo2Retrieved.getNum()).isEqualTo(2); 62 | 63 | } 64 | 65 | static class Foo { 66 | 67 | private int num; 68 | private String data; 69 | 70 | Foo() { 71 | } 72 | 73 | public int getNum() { 74 | return num; 75 | } 76 | 77 | public void setNum(int num) { 78 | this.num = num; 79 | } 80 | 81 | public String getData() { 82 | return data; 83 | } 84 | 85 | public void setData(String data) { 86 | this.data = data; 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka-streams/src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{ISO8601} %5p %t %c{2}:%L - %m%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/.jdk8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spring-attic/spring-cloud-stream-binder-kafka/31ea10683422d38d8185c6b236b8954200562b77/spring-cloud-stream-binder-kafka/.jdk8 -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/.settings/org.eclipse.jdt.ui.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.ui.ignorelowercasenames=true 3 | org.eclipse.jdt.ui.importorder=java;javax;com;io.micrometer;org;org.springframework;ch.qos;\#; 4 | org.eclipse.jdt.ui.ondemandthreshold=99 5 | org.eclipse.jdt.ui.staticondemandthreshold=99 6 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | spring-cloud-stream-binder-kafka 6 | jar 7 | spring-cloud-stream-binder-kafka 8 | Kafka binder implementation 9 | 10 | 11 | org.springframework.cloud 12 | spring-cloud-stream-binder-kafka-parent 13 | 4.0.0-SNAPSHOT 14 | 15 | 16 | 17 | 18 | org.springframework.cloud 19 | spring-cloud-stream-binder-kafka-core 20 | 21 | 22 | org.springframework.boot 23 | spring-boot-starter-actuator 24 | true 25 | 26 | 27 | org.springframework.boot 28 | spring-boot-configuration-processor 29 | true 30 | 31 | 32 | org.springframework.cloud 33 | spring-cloud-stream 34 | 35 | 36 | org.springframework.boot 37 | spring-boot-autoconfigure 38 | true 39 | 40 | 41 | org.apache.kafka 42 | kafka-clients 43 | 44 | 45 | org.springframework.kafka 46 | spring-kafka 47 | 48 | 49 | org.springframework.boot 50 | spring-boot-test 51 | test 52 | 53 | 54 | org.springframework.kafka 55 | spring-kafka-test 56 | test 57 | 58 | 59 | org.springframework.cloud 60 | spring-cloud-stream-binder-test 61 | test 62 | 63 | 64 | 65 | org.apache.kafka 66 | kafka-clients 67 | test 68 | 69 | 70 | org.apache.kafka 71 | kafka_2.13 72 | 73 | 74 | org.apache.kafka 75 | kafka_2.13 76 | test 77 | 78 | 79 | org.awaitility 80 | awaitility 81 | test 82 | 83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderEnvironmentPostProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | import org.apache.kafka.common.serialization.ByteArrayDeserializer; 23 | import org.apache.kafka.common.serialization.ByteArraySerializer; 24 | 25 | import org.springframework.boot.SpringApplication; 26 | import org.springframework.boot.env.EnvironmentPostProcessor; 27 | import org.springframework.core.env.ConfigurableEnvironment; 28 | import org.springframework.core.env.MapPropertySource; 29 | 30 | /** 31 | * An {@link EnvironmentPostProcessor} that sets some common configuration properties (log 32 | * config etc.,) for Kafka binder. 33 | * 34 | * @author Ilayaperumal Gopinathan 35 | */ 36 | public class KafkaBinderEnvironmentPostProcessor implements EnvironmentPostProcessor { 37 | 38 | private static final String SPRING_KAFKA = "spring.kafka"; 39 | 40 | private static final String SPRING_KAFKA_PRODUCER = SPRING_KAFKA + ".producer"; 41 | 42 | private static final String SPRING_KAFKA_CONSUMER = SPRING_KAFKA + ".consumer"; 43 | 44 | private static final String SPRING_KAFKA_PRODUCER_KEY_SERIALIZER = SPRING_KAFKA_PRODUCER 45 | + "." + "keySerializer"; 46 | 47 | private static final String SPRING_KAFKA_PRODUCER_VALUE_SERIALIZER = SPRING_KAFKA_PRODUCER 48 | + "." + "valueSerializer"; 49 | 50 | private static final String SPRING_KAFKA_CONSUMER_KEY_DESERIALIZER = SPRING_KAFKA_CONSUMER 51 | + "." + "keyDeserializer"; 52 | 53 | private static final String SPRING_KAFKA_CONSUMER_VALUE_DESERIALIZER = SPRING_KAFKA_CONSUMER 54 | + "." + "valueDeserializer"; 55 | 56 | private static final String KAFKA_BINDER_DEFAULT_PROPERTIES = "kafkaBinderDefaultProperties"; 57 | 58 | @Override 59 | public void postProcessEnvironment(ConfigurableEnvironment environment, 60 | SpringApplication application) { 61 | if (!environment.getPropertySources().contains(KAFKA_BINDER_DEFAULT_PROPERTIES)) { 62 | Map kafkaBinderDefaultProperties = new HashMap<>(); 63 | kafkaBinderDefaultProperties.put("logging.level.org.I0Itec.zkclient", 64 | "ERROR"); 65 | kafkaBinderDefaultProperties.put("logging.level.kafka.server.KafkaConfig", 66 | "ERROR"); 67 | kafkaBinderDefaultProperties 68 | .put("logging.level.kafka.admin.AdminClient.AdminConfig", "ERROR"); 69 | kafkaBinderDefaultProperties.put(SPRING_KAFKA_PRODUCER_KEY_SERIALIZER, 70 | ByteArraySerializer.class.getName()); 71 | kafkaBinderDefaultProperties.put(SPRING_KAFKA_PRODUCER_VALUE_SERIALIZER, 72 | ByteArraySerializer.class.getName()); 73 | kafkaBinderDefaultProperties.put(SPRING_KAFKA_CONSUMER_KEY_DESERIALIZER, 74 | ByteArrayDeserializer.class.getName()); 75 | kafkaBinderDefaultProperties.put(SPRING_KAFKA_CONSUMER_VALUE_DESERIALIZER, 76 | ByteArrayDeserializer.class.getName()); 77 | environment.getPropertySources().addLast(new MapPropertySource( 78 | KAFKA_BINDER_DEFAULT_PROPERTIES, kafkaBinderDefaultProperties)); 79 | } 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderHealth.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2022 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka; 18 | 19 | import org.springframework.boot.actuate.health.HealthIndicator; 20 | 21 | /** 22 | * Marker interface used for custom KafkaBinderHealth indicator implementations. 23 | * 24 | * @author Soby Chacko 25 | * @since 3.2.2 26 | */ 27 | public interface KafkaBinderHealth extends HealthIndicator { 28 | 29 | } 30 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaBindingRebalanceListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka; 18 | 19 | import java.util.Collection; 20 | 21 | import org.apache.kafka.clients.consumer.Consumer; 22 | import org.apache.kafka.common.TopicPartition; 23 | 24 | /** 25 | * A rebalance listener that provides access to the binding name consumer object. It can 26 | * be used to perform seek operations on the consumer after a rebalance. 27 | * 28 | * @author Gary Russell 29 | * @since 2.1 30 | * 31 | */ 32 | public interface KafkaBindingRebalanceListener { 33 | 34 | /** 35 | * Invoked by the container before any pending offsets are committed. 36 | * @param bindingName the name of the binding. 37 | * @param consumer the consumer. 38 | * @param partitions the partitions. 39 | */ 40 | default void onPartitionsRevokedBeforeCommit(String bindingName, 41 | Consumer consumer, Collection partitions) { 42 | // do nothing 43 | } 44 | 45 | /** 46 | * Invoked by the container after any pending offsets are committed. 47 | * @param bindingName the name of the binding. 48 | * @param consumer the consumer. 49 | * @param partitions the partitions. 50 | */ 51 | default void onPartitionsRevokedAfterCommit(String bindingName, 52 | Consumer consumer, Collection partitions) { 53 | // do nothing 54 | } 55 | 56 | /** 57 | * Invoked when partitions are initially assigned or after a rebalance. Applications 58 | * might only want to perform seek operations on an initial assignment. While the 59 | * 'initial' argument is true for each thread (when concurrency is greater than 1), 60 | * implementations should keep track of exactly which partitions have been sought. 61 | * There is a race in that a rebalance could occur during startup and so a topic/ 62 | * partition that has been sought on one thread may be re-assigned to another 63 | * thread and you may not wish to re-seek it at that time. 64 | * @param bindingName the name of the binding. 65 | * @param consumer the consumer. 66 | * @param partitions the partitions. 67 | * @param initial true if this is the initial assignment on the current thread. 68 | */ 69 | default void onPartitionsAssigned(String bindingName, Consumer consumer, 70 | Collection partitions, boolean initial) { 71 | // do nothing 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaExpressionEvaluatingInterceptor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka; 18 | 19 | import org.springframework.expression.EvaluationContext; 20 | import org.springframework.expression.Expression; 21 | import org.springframework.integration.support.MessageBuilder; 22 | import org.springframework.messaging.Message; 23 | import org.springframework.messaging.MessageChannel; 24 | import org.springframework.messaging.support.ChannelInterceptor; 25 | import org.springframework.util.Assert; 26 | 27 | /** 28 | * Interceptor to evaluate expressions for outbound messages before serialization. 29 | * 30 | * @author Gary Russell 31 | * @since 3.0 32 | * 33 | */ 34 | public class KafkaExpressionEvaluatingInterceptor implements ChannelInterceptor { 35 | 36 | /** 37 | * Name for the evaluated message key header. 38 | */ 39 | public static final String MESSAGE_KEY_HEADER = "scst_messageKey"; 40 | 41 | private final Expression messageKeyExpression; 42 | 43 | private final EvaluationContext evaluationContext; 44 | 45 | /** 46 | * Construct an instance with the provided expressions and evaluation context. At 47 | * least one expression muse be non-null. 48 | * @param messageKeyExpression the routing key expression. 49 | * @param evaluationContext the evaluation context. 50 | */ 51 | public KafkaExpressionEvaluatingInterceptor(Expression messageKeyExpression, EvaluationContext evaluationContext) { 52 | Assert.notNull(messageKeyExpression != null, "A message key expression is required"); 53 | Assert.notNull(evaluationContext, "the 'evaluationContext' cannot be null"); 54 | this.messageKeyExpression = messageKeyExpression; 55 | this.evaluationContext = evaluationContext; 56 | } 57 | 58 | @Override 59 | public Message preSend(Message message, MessageChannel channel) { 60 | MessageBuilder builder = MessageBuilder.fromMessage(message); 61 | if (this.messageKeyExpression != null) { 62 | builder.setHeader(MESSAGE_KEY_HEADER, 63 | this.messageKeyExpression.getValue(this.evaluationContext, message)); 64 | } 65 | return builder.build(); 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/KafkaNullConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka; 18 | 19 | import java.util.Collections; 20 | 21 | import org.springframework.kafka.support.KafkaNull; 22 | import org.springframework.messaging.Message; 23 | import org.springframework.messaging.MessageHeaders; 24 | import org.springframework.messaging.converter.AbstractMessageConverter; 25 | import org.springframework.messaging.converter.MessageConverter; 26 | import org.springframework.util.MimeTypeUtils; 27 | 28 | /** 29 | * A {@link MessageConverter} that supports {@link KafkaNull} payloads. 30 | * 31 | * @author Gary Russell 32 | * @author Aldo Sinanaj 33 | * @since 2.2 34 | */ 35 | public class KafkaNullConverter extends AbstractMessageConverter { 36 | 37 | public KafkaNullConverter() { 38 | super(Collections.singletonList(MimeTypeUtils.ALL)); 39 | } 40 | 41 | @Override 42 | protected boolean supportsMimeType(MessageHeaders headers) { 43 | return true; 44 | } 45 | 46 | @Override 47 | protected boolean supports(Class aClass) { 48 | return KafkaNull.class.equals(aClass); 49 | } 50 | 51 | @Override 52 | protected boolean canConvertFrom(Message message, Class targetClass) { 53 | return message.getPayload() instanceof KafkaNull; 54 | } 55 | 56 | @Override 57 | protected Object convertFromInternal(Message message, Class targetClass, 58 | Object conversionHint) { 59 | return message.getPayload(); 60 | } 61 | 62 | @Override 63 | protected Object convertToInternal(Object payload, MessageHeaders headers, 64 | Object conversionHint) { 65 | return payload; 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/ListenerContainerWithDlqAndRetryCustomizer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021-2021 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka; 18 | 19 | import java.util.function.BiFunction; 20 | 21 | import org.apache.kafka.clients.consumer.ConsumerRecord; 22 | import org.apache.kafka.common.TopicPartition; 23 | 24 | import org.springframework.cloud.stream.config.ListenerContainerCustomizer; 25 | import org.springframework.kafka.listener.AbstractMessageListenerContainer; 26 | import org.springframework.lang.Nullable; 27 | import org.springframework.util.backoff.BackOff; 28 | 29 | /** 30 | * An extension of {@link ListenerContainerCustomizer} that provides access to dead letter 31 | * metadata. 32 | * 33 | * @author Gary Russell 34 | * @since 3.2 35 | * 36 | */ 37 | public interface ListenerContainerWithDlqAndRetryCustomizer 38 | extends ListenerContainerCustomizer> { 39 | 40 | @Override 41 | default void configure(AbstractMessageListenerContainer container, String destinationName, String group) { 42 | } 43 | 44 | /** 45 | * Configure the container. 46 | * @param container the container. 47 | * @param destinationName the destination name. 48 | * @param group the group. 49 | * @param dlqDestinationResolver a destination resolver for the dead letter topic (if 50 | * enableDlq). 51 | * @param backOff the backOff using retry properties (if configured). 52 | * @see #retryAndDlqInBinding(String, String) 53 | */ 54 | void configure(AbstractMessageListenerContainer container, String destinationName, String group, 55 | @Nullable BiFunction, Exception, TopicPartition> dlqDestinationResolver, 56 | @Nullable BackOff backOff); 57 | 58 | /** 59 | * Return false to move retries and DLQ from the binding to a customized error handler 60 | * using the retry metadata and/or a {@code DeadLetterPublishingRecoverer} when 61 | * configured via 62 | * {@link #configure(AbstractMessageListenerContainer, String, String, BiFunction, BackOff)}. 63 | * @param destinationName the destination name. 64 | * @param group the group. 65 | * @return true to disable retrie in the binding 66 | */ 67 | default boolean retryAndDlqInBinding(String destinationName, String group) { 68 | return true; 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/ClientFactoryCustomizer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2020 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.config; 18 | 19 | import org.springframework.kafka.core.ConsumerFactory; 20 | import org.springframework.kafka.core.ProducerFactory; 21 | 22 | /** 23 | * Called by the binder to customize the factories. 24 | * 25 | * @author Gary Russell 26 | * @since 3.0.6 27 | * 28 | */ 29 | public interface ClientFactoryCustomizer { 30 | 31 | default void configure(ProducerFactory pf) { 32 | } 33 | 34 | default void configure(ConsumerFactory cf) { 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/ConsumerConfigCustomizer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2020 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.config; 18 | 19 | import java.util.Map; 20 | 21 | /** 22 | * This customizer is called by the binder to customize consumer configuration in 23 | * Kafka Consumer factory. 24 | * 25 | * @author Soby Chacko 26 | * @since 3.0.9 27 | */ 28 | @FunctionalInterface 29 | public interface ConsumerConfigCustomizer { 30 | 31 | void configure(Map consumerProperties, String bindingName, String destination); 32 | } 33 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/ExtendedBindingHandlerMappingsProviderConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.config; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | import org.springframework.boot.context.properties.source.ConfigurationPropertyName; 23 | import org.springframework.cloud.stream.config.BindingHandlerAdvise.MappingsProvider; 24 | import org.springframework.context.annotation.Bean; 25 | import org.springframework.context.annotation.Configuration; 26 | 27 | /** 28 | * Configuration for extended binding metadata. 29 | * 30 | * @author Oleg Zhurakousky 31 | */ 32 | 33 | @Configuration 34 | public class ExtendedBindingHandlerMappingsProviderConfiguration { 35 | 36 | @Bean 37 | public MappingsProvider kafkaExtendedPropertiesDefaultMappingsProvider() { 38 | return () -> { 39 | Map mappings = new HashMap<>(); 40 | mappings.put( 41 | ConfigurationPropertyName.of("spring.cloud.stream.kafka.bindings"), 42 | ConfigurationPropertyName.of("spring.cloud.stream.kafka.default")); 43 | return mappings; 44 | }; 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/KafkaBinderHealthIndicatorConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2022 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.config; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | import org.apache.kafka.clients.consumer.ConsumerConfig; 23 | import org.apache.kafka.common.serialization.ByteArrayDeserializer; 24 | 25 | import org.springframework.boot.actuate.autoconfigure.health.ConditionalOnEnabledHealthIndicator; 26 | import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; 27 | import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; 28 | import org.springframework.cloud.stream.binder.kafka.KafkaBinderHealth; 29 | import org.springframework.cloud.stream.binder.kafka.KafkaBinderHealthIndicator; 30 | import org.springframework.cloud.stream.binder.kafka.KafkaMessageChannelBinder; 31 | import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; 32 | import org.springframework.context.annotation.Bean; 33 | import org.springframework.context.annotation.Configuration; 34 | import org.springframework.kafka.core.ConsumerFactory; 35 | import org.springframework.kafka.core.DefaultKafkaConsumerFactory; 36 | import org.springframework.util.ObjectUtils; 37 | 38 | /** 39 | * Configuration class for Kafka binder health indicator beans. 40 | * 41 | * @author Oleg Zhurakousky 42 | * @author Chukwubuikem Ume-Ugwa 43 | * @author Soby Chacko 44 | */ 45 | 46 | @Configuration(proxyBeanMethods = false) 47 | @ConditionalOnClass(name = "org.springframework.boot.actuate.health.HealthIndicator") 48 | @ConditionalOnEnabledHealthIndicator("binders") 49 | @ConditionalOnMissingBean(KafkaBinderHealth.class) 50 | public class KafkaBinderHealthIndicatorConfiguration { 51 | 52 | @Bean 53 | public KafkaBinderHealthIndicator kafkaBinderHealthIndicator( 54 | KafkaMessageChannelBinder kafkaMessageChannelBinder, 55 | KafkaBinderConfigurationProperties configurationProperties) { 56 | Map props = new HashMap<>(); 57 | props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, 58 | ByteArrayDeserializer.class); 59 | props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, 60 | ByteArrayDeserializer.class); 61 | Map mergedConfig = configurationProperties 62 | .mergedConsumerConfiguration(); 63 | if (!ObjectUtils.isEmpty(mergedConfig)) { 64 | props.putAll(mergedConfig); 65 | } 66 | if (!props.containsKey(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)) { 67 | props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, 68 | configurationProperties.getKafkaConnectionString()); 69 | } 70 | ConsumerFactory consumerFactory = new DefaultKafkaConsumerFactory<>(props); 71 | KafkaBinderHealthIndicator indicator = new KafkaBinderHealthIndicator( 72 | kafkaMessageChannelBinder, consumerFactory); 73 | indicator.setTimeout(configurationProperties.getHealthTimeout()); 74 | indicator.setConsiderDownWhenAnyPartitionHasNoLeader(configurationProperties.isConsiderDownWhenAnyPartitionHasNoLeader()); 75 | return indicator; 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/main/java/org/springframework/cloud/stream/binder/kafka/config/ProducerConfigCustomizer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020-2020 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.config; 18 | 19 | import java.util.Map; 20 | 21 | /** 22 | * This customizer is called by the binder to customize producer configuration in 23 | * Kafka Producer factory. 24 | * 25 | * @author Soby Chacko 26 | * @since 3.0.9 27 | */ 28 | @FunctionalInterface 29 | public interface ProducerConfigCustomizer { 30 | 31 | void configure(Map producerProperties, String bindingName, String destination); 32 | } 33 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/main/resources/META-INF/spring.binders: -------------------------------------------------------------------------------- 1 | kafka:\ 2 | org.springframework.cloud.stream.binder.kafka.config.KafkaBinderConfiguration 3 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/main/resources/META-INF/spring.factories: -------------------------------------------------------------------------------- 1 | org.springframework.boot.env.EnvironmentPostProcessor:\ 2 | org.springframework.cloud.stream.binder.kafka.KafkaBinderEnvironmentPostProcessor 3 | org.springframework.boot.autoconfigure.EnableAutoConfiguration:\ 4 | org.springframework.cloud.stream.binder.kafka.config.ExtendedBindingHandlerMappingsProviderConfiguration 5 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/AbstractKafkaTestBinder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2018 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka; 18 | 19 | import org.springframework.cloud.stream.binder.AbstractPollableConsumerTestBinder; 20 | import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; 21 | import org.springframework.cloud.stream.binder.ExtendedProducerProperties; 22 | import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; 23 | import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; 24 | import org.springframework.context.ApplicationContext; 25 | 26 | /** 27 | * @author Soby Chacko 28 | * @author Gary Russell 29 | */ 30 | public abstract class AbstractKafkaTestBinder extends 31 | // @checkstyle:off 32 | AbstractPollableConsumerTestBinder, ExtendedProducerProperties> { 33 | 34 | // @checkstyle:on 35 | private ApplicationContext applicationContext; 36 | 37 | @Override 38 | public void cleanup() { 39 | // do nothing - the rule will take care of that 40 | } 41 | 42 | protected final void setApplicationContext(ApplicationContext context) { 43 | this.applicationContext = context; 44 | } 45 | 46 | public ApplicationContext getApplicationContext() { 47 | return this.applicationContext; 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/AdminConfigTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka; 18 | 19 | import java.util.Arrays; 20 | 21 | import org.junit.Test; 22 | import org.junit.runner.RunWith; 23 | 24 | import org.springframework.beans.factory.annotation.Autowired; 25 | import org.springframework.boot.test.context.SpringBootTest; 26 | import org.springframework.cloud.stream.binder.kafka.config.KafkaBinderConfiguration; 27 | import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; 28 | import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; 29 | import org.springframework.cloud.stream.binder.kafka.properties.KafkaTopicProperties; 30 | import org.springframework.cloud.stream.config.BindingServiceConfiguration; 31 | import org.springframework.integration.config.EnableIntegration; 32 | import org.springframework.test.context.TestPropertySource; 33 | import org.springframework.test.context.junit4.SpringRunner; 34 | 35 | import static org.assertj.core.api.Assertions.assertThat; 36 | 37 | /** 38 | * @author Gary Russell 39 | * @author Aldo Sinanaj 40 | * @since 2.0 41 | * 42 | */ 43 | @RunWith(SpringRunner.class) 44 | @SpringBootTest(classes = { KafkaBinderConfiguration.class, 45 | BindingServiceConfiguration.class }) 46 | @TestPropertySource(properties = { 47 | "spring.cloud.stream.kafka.bindings.input.consumer.topic.replication-factor=2", 48 | "spring.cloud.stream.kafka.bindings.input.consumer.topic.replicas-assignments.0=0,1", 49 | "spring.cloud.stream.kafka.bindings.input.consumer.topic.properties.message.format.version=0.9.0.0", 50 | "spring.cloud.stream.kafka.bindings.secondInput.consumer.topic.replication-factor=3", 51 | "spring.cloud.stream.kafka.bindings.secondInput.consumer.topic.replicas-assignments.0=0,1", 52 | "spring.cloud.stream.kafka.bindings.secondInput.consumer.topic.properties.message.format.version=0.9.1.0", 53 | "spring.cloud.stream.kafka.bindings.output.producer.topic.replication-factor=2", 54 | "spring.cloud.stream.kafka.bindings.output.producer.topic.replicas-assignments.0=0,1", 55 | "spring.cloud.stream.kafka.bindings.output.producer.topic.properties.message.format.version=0.9.0.0", 56 | "spring.main.allow-bean-definition-overriding=true" }) 57 | @EnableIntegration 58 | public class AdminConfigTests { 59 | 60 | @Autowired 61 | private KafkaMessageChannelBinder binder; 62 | 63 | @Test 64 | public void testConsumerTopicProperties() { 65 | final KafkaConsumerProperties consumerProperties = this.binder 66 | .getExtendedConsumerProperties("secondInput"); 67 | final KafkaTopicProperties kafkaTopicProperties = consumerProperties.getTopic(); 68 | 69 | assertThat(kafkaTopicProperties.getReplicationFactor()).isEqualTo((short) 3); 70 | assertThat(kafkaTopicProperties.getReplicasAssignments().get(0)) 71 | .isEqualTo(Arrays.asList(0, 1)); 72 | assertThat(kafkaTopicProperties.getProperties().get("message.format.version")) 73 | .isEqualTo("0.9.1.0"); 74 | } 75 | 76 | @Test 77 | public void testProducerTopicProperties() { 78 | final KafkaProducerProperties producerProperties = this.binder 79 | .getExtendedProducerProperties("output"); 80 | final KafkaTopicProperties kafkaTopicProperties = producerProperties.getTopic(); 81 | 82 | assertThat(kafkaTopicProperties.getReplicationFactor()).isEqualTo((short) 2); 83 | assertThat(kafkaTopicProperties.getReplicasAssignments().get(0)) 84 | .isEqualTo(Arrays.asList(0, 1)); 85 | assertThat(kafkaTopicProperties.getProperties().get("message.format.version")) 86 | .isEqualTo("0.9.0.0"); 87 | } 88 | 89 | } 90 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/AutoCreateTopicDisabledTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka; 18 | 19 | import java.util.Collections; 20 | 21 | import kafka.server.KafkaConfig; 22 | import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; 23 | import org.junit.ClassRule; 24 | import org.junit.Rule; 25 | import org.junit.Test; 26 | import org.junit.rules.ExpectedException; 27 | 28 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 29 | import org.springframework.cloud.stream.binder.BinderException; 30 | import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; 31 | import org.springframework.cloud.stream.binder.ExtendedProducerProperties; 32 | import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; 33 | import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; 34 | import org.springframework.cloud.stream.binder.kafka.properties.KafkaProducerProperties; 35 | import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; 36 | import org.springframework.integration.channel.DirectChannel; 37 | import org.springframework.kafka.test.rule.EmbeddedKafkaRule; 38 | import org.springframework.retry.policy.SimpleRetryPolicy; 39 | import org.springframework.retry.support.RetryTemplate; 40 | 41 | import static org.hamcrest.CoreMatchers.isA; 42 | 43 | /** 44 | * @author Soby Chacko 45 | */ 46 | public class AutoCreateTopicDisabledTests { 47 | 48 | @Rule 49 | public ExpectedException expectedException = ExpectedException.none(); 50 | 51 | @ClassRule 52 | public static EmbeddedKafkaRule embeddedKafka = new EmbeddedKafkaRule(1, true, 1) 53 | .brokerProperty(KafkaConfig.AutoCreateTopicsEnableProp(), "false"); 54 | 55 | @Test 56 | public void testAutoCreateTopicDisabledFailsOnConsumerIfTopicNonExistentOnBroker() 57 | throws Throwable { 58 | 59 | KafkaProperties kafkaProperties = new TestKafkaProperties(); 60 | kafkaProperties.setBootstrapServers(Collections 61 | .singletonList(embeddedKafka.getEmbeddedKafka().getBrokersAsString())); 62 | KafkaBinderConfigurationProperties configurationProperties = new KafkaBinderConfigurationProperties( 63 | kafkaProperties); 64 | // disable auto create topic on the binder. 65 | configurationProperties.setAutoCreateTopics(false); 66 | 67 | KafkaTopicProvisioner provisioningProvider = new KafkaTopicProvisioner( 68 | configurationProperties, kafkaProperties, null); 69 | provisioningProvider.setMetadataRetryOperations(new RetryTemplate()); 70 | 71 | KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder( 72 | configurationProperties, provisioningProvider); 73 | 74 | final String testTopicName = "nonExistent" + System.currentTimeMillis(); 75 | 76 | ExtendedConsumerProperties properties = new ExtendedConsumerProperties<>( 77 | new KafkaConsumerProperties()); 78 | 79 | expectedException.expect(BinderException.class); 80 | expectedException.expectCause(isA(UnknownTopicOrPartitionException.class)); 81 | binder.createConsumerEndpoint(() -> testTopicName, "group", properties); 82 | } 83 | 84 | @Test 85 | public void testAutoCreateTopicDisabledFailsOnProducerIfTopicNonExistentOnBroker() 86 | throws Throwable { 87 | 88 | KafkaProperties kafkaProperties = new TestKafkaProperties(); 89 | kafkaProperties.setBootstrapServers(Collections 90 | .singletonList(embeddedKafka.getEmbeddedKafka().getBrokersAsString())); 91 | 92 | KafkaBinderConfigurationProperties configurationProperties = new KafkaBinderConfigurationProperties( 93 | kafkaProperties); 94 | // disable auto create topic on the binder. 95 | configurationProperties.setAutoCreateTopics(false); 96 | // reduce the wait time on the producer blocking operations. 97 | configurationProperties.getConfiguration().put("max.block.ms", "3000"); 98 | 99 | KafkaTopicProvisioner provisioningProvider = new KafkaTopicProvisioner( 100 | configurationProperties, kafkaProperties, null); 101 | SimpleRetryPolicy simpleRetryPolicy = new SimpleRetryPolicy(1); 102 | final RetryTemplate metadataRetryOperations = new RetryTemplate(); 103 | metadataRetryOperations.setRetryPolicy(simpleRetryPolicy); 104 | provisioningProvider.setMetadataRetryOperations(metadataRetryOperations); 105 | 106 | KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder( 107 | configurationProperties, provisioningProvider); 108 | 109 | final String testTopicName = "nonExistent" + System.currentTimeMillis(); 110 | 111 | ExtendedProducerProperties properties = new ExtendedProducerProperties<>( 112 | new KafkaProducerProperties()); 113 | 114 | expectedException.expect(BinderException.class); 115 | expectedException.expectCause(isA(UnknownTopicOrPartitionException.class)); 116 | 117 | binder.bindProducer(testTopicName, new DirectChannel(), properties); 118 | 119 | } 120 | 121 | } 122 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaBinderConfigurationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016-2017 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka; 18 | 19 | import java.lang.reflect.Field; 20 | 21 | import org.junit.Test; 22 | import org.junit.runner.RunWith; 23 | 24 | import org.springframework.beans.factory.annotation.Autowired; 25 | import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; 26 | import org.springframework.boot.test.context.SpringBootTest; 27 | import org.springframework.cloud.stream.binder.kafka.config.KafkaBinderConfiguration; 28 | import org.springframework.kafka.support.ProducerListener; 29 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 30 | import org.springframework.util.ReflectionUtils; 31 | 32 | import static org.assertj.core.api.Assertions.assertThat; 33 | 34 | /** 35 | * @author Ilayaperumal Gopinathan 36 | */ 37 | @RunWith(SpringJUnit4ClassRunner.class) 38 | @SpringBootTest(classes = { KafkaBinderConfiguration.class, KafkaAutoConfiguration.class, 39 | KafkaBinderConfigurationTest.class }) 40 | public class KafkaBinderConfigurationTest { 41 | 42 | @Autowired 43 | private KafkaMessageChannelBinder kafkaMessageChannelBinder; 44 | 45 | @Test 46 | public void testKafkaBinderProducerListener() { 47 | assertThat(this.kafkaMessageChannelBinder).isNotNull(); 48 | Field producerListenerField = ReflectionUtils.findField( 49 | KafkaMessageChannelBinder.class, "producerListener", 50 | ProducerListener.class); 51 | ReflectionUtils.makeAccessible(producerListenerField); 52 | ProducerListener producerListener = (ProducerListener) ReflectionUtils 53 | .getField(producerListenerField, this.kafkaMessageChannelBinder); 54 | assertThat(producerListener).isNotNull(); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/KafkaTestBinder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015-2018 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka; 18 | 19 | import org.springframework.cloud.stream.binder.ExtendedConsumerProperties; 20 | import org.springframework.cloud.stream.binder.kafka.properties.KafkaBinderConfigurationProperties; 21 | import org.springframework.cloud.stream.binder.kafka.properties.KafkaConsumerProperties; 22 | import org.springframework.cloud.stream.binder.kafka.provisioning.KafkaTopicProvisioner; 23 | import org.springframework.cloud.stream.binder.kafka.utils.DlqDestinationResolver; 24 | import org.springframework.cloud.stream.binder.kafka.utils.DlqPartitionFunction; 25 | import org.springframework.cloud.stream.provisioning.ConsumerDestination; 26 | import org.springframework.context.annotation.AnnotationConfigApplicationContext; 27 | import org.springframework.context.annotation.Configuration; 28 | import org.springframework.integration.config.EnableIntegration; 29 | import org.springframework.kafka.support.LoggingProducerListener; 30 | import org.springframework.kafka.support.ProducerListener; 31 | 32 | /** 33 | * Test support class for {@link KafkaMessageChannelBinder}. 34 | * 35 | * @author Eric Bottard 36 | * @author Marius Bogoevici 37 | * @author David Turanski 38 | * @author Gary Russell 39 | * @author Soby Chacko 40 | */ 41 | public class KafkaTestBinder extends AbstractKafkaTestBinder { 42 | 43 | KafkaTestBinder(KafkaBinderConfigurationProperties binderConfiguration, 44 | KafkaTopicProvisioner kafkaTopicProvisioner) { 45 | 46 | this(binderConfiguration, kafkaTopicProvisioner, null, null); 47 | } 48 | 49 | @SuppressWarnings({ "rawtypes", "unchecked" }) 50 | KafkaTestBinder(KafkaBinderConfigurationProperties binderConfiguration, 51 | KafkaTopicProvisioner kafkaTopicProvisioner, DlqPartitionFunction dlqPartitionFunction, 52 | DlqDestinationResolver dlqDestinationResolver) { 53 | 54 | try { 55 | KafkaMessageChannelBinder binder = new KafkaMessageChannelBinder( 56 | binderConfiguration, kafkaTopicProvisioner, null, null, null, dlqPartitionFunction, dlqDestinationResolver) { 57 | 58 | /* 59 | * Some tests use multiple instance indexes for the same topic; we need to 60 | * make the error infrastructure beans unique. 61 | */ 62 | @Override 63 | protected String errorsBaseName(ConsumerDestination destination, 64 | String group, 65 | ExtendedConsumerProperties consumerProperties) { 66 | return super.errorsBaseName(destination, group, consumerProperties) 67 | + "-" + consumerProperties.getInstanceIndex(); 68 | } 69 | 70 | }; 71 | 72 | ProducerListener producerListener = new LoggingProducerListener(); 73 | binder.setProducerListener(producerListener); 74 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext( 75 | Config.class); 76 | setApplicationContext(context); 77 | binder.setApplicationContext(context); 78 | binder.afterPropertiesSet(); 79 | this.setPollableConsumerBinder(binder); 80 | } 81 | catch (Exception e) { 82 | throw new RuntimeException(e); 83 | } 84 | } 85 | 86 | @Configuration 87 | @EnableIntegration 88 | static class Config { 89 | 90 | } 91 | 92 | } 93 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/RawKafkaPartitionTestSupport.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka; 18 | 19 | import org.springframework.cloud.stream.binder.PartitionKeyExtractorStrategy; 20 | import org.springframework.cloud.stream.binder.PartitionSelectorStrategy; 21 | import org.springframework.messaging.Message; 22 | 23 | /** 24 | * @author Marius Bogoevici 25 | */ 26 | public class RawKafkaPartitionTestSupport 27 | implements PartitionKeyExtractorStrategy, PartitionSelectorStrategy { 28 | 29 | @Override 30 | public int selectPartition(Object key, int divisor) { 31 | return ((byte[]) key)[0] % divisor; 32 | } 33 | 34 | @Override 35 | public Object extractKey(Message message) { 36 | return message.getPayload(); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/TestKafkaProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka; 18 | 19 | import org.apache.kafka.common.serialization.ByteArrayDeserializer; 20 | import org.apache.kafka.common.serialization.ByteArraySerializer; 21 | 22 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties; 23 | 24 | /** 25 | * Test {@link KafkaProperties} initialized in the same way as the 26 | * {@code KafkaBinderEnvironmentPostProcessor} initializes the properties. 27 | * 28 | * @author Gary Russell 29 | * @since 2.1 30 | * 31 | */ 32 | public class TestKafkaProperties extends KafkaProperties { 33 | 34 | public TestKafkaProperties() { 35 | getConsumer().setKeyDeserializer(ByteArrayDeserializer.class); 36 | getConsumer().setValueDeserializer(ByteArrayDeserializer.class); 37 | getProducer().setKeySerializer(ByteArraySerializer.class); 38 | getProducer().setValueSerializer(ByteArraySerializer.class); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/bootstrap/KafkaBinderBootstrapTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.bootstrap; 18 | 19 | import org.junit.ClassRule; 20 | import org.junit.Test; 21 | 22 | import org.springframework.boot.WebApplicationType; 23 | import org.springframework.boot.autoconfigure.SpringBootApplication; 24 | import org.springframework.boot.builder.SpringApplicationBuilder; 25 | import org.springframework.context.ConfigurableApplicationContext; 26 | import org.springframework.kafka.test.rule.EmbeddedKafkaRule; 27 | 28 | /** 29 | * @author Marius Bogoevici 30 | */ 31 | public class KafkaBinderBootstrapTest { 32 | 33 | @ClassRule 34 | public static EmbeddedKafkaRule embeddedKafka = new EmbeddedKafkaRule(1, true, 10); 35 | 36 | @Test 37 | public void testKafkaBinderConfiguration() throws Exception { 38 | ConfigurableApplicationContext applicationContext = new SpringApplicationBuilder( 39 | SimpleApplication.class).web(WebApplicationType.NONE).run( 40 | "--spring.cloud.stream.kafka.binder.brokers=" 41 | + embeddedKafka.getEmbeddedKafka().getBrokersAsString(), 42 | "--spring.cloud.stream.kafka.binder.zkNodes=" + embeddedKafka 43 | .getEmbeddedKafka().getZookeeperConnectionString()); 44 | applicationContext.close(); 45 | } 46 | 47 | @SpringBootApplication 48 | static class SimpleApplication { 49 | 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/bootstrap/KafkaBinderCustomHealthCheckTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2022 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.bootstrap; 18 | 19 | import org.junit.ClassRule; 20 | import org.junit.Test; 21 | 22 | import org.springframework.beans.factory.NoSuchBeanDefinitionException; 23 | import org.springframework.boot.WebApplicationType; 24 | import org.springframework.boot.actuate.health.Health; 25 | import org.springframework.boot.autoconfigure.SpringBootApplication; 26 | import org.springframework.boot.builder.SpringApplicationBuilder; 27 | import org.springframework.cloud.stream.binder.kafka.KafkaBinderHealth; 28 | import org.springframework.cloud.stream.binder.kafka.KafkaBinderHealthIndicator; 29 | import org.springframework.context.ConfigurableApplicationContext; 30 | import org.springframework.context.annotation.Bean; 31 | import org.springframework.kafka.test.rule.EmbeddedKafkaRule; 32 | 33 | import static org.assertj.core.api.AssertionsForClassTypes.assertThat; 34 | import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; 35 | 36 | /** 37 | * @author Soby Chacko 38 | */ 39 | public class KafkaBinderCustomHealthCheckTests { 40 | 41 | @ClassRule 42 | public static EmbeddedKafkaRule embeddedKafka = new EmbeddedKafkaRule(1, true, 10); 43 | 44 | @Test 45 | public void testCustomHealthIndicatorIsActivated() { 46 | ConfigurableApplicationContext applicationContext = new SpringApplicationBuilder( 47 | CustomHealthCheckApplication.class).web(WebApplicationType.NONE).run( 48 | "--spring.cloud.stream.kafka.binder.brokers=" 49 | + embeddedKafka.getEmbeddedKafka().getBrokersAsString()); 50 | final KafkaBinderHealth kafkaBinderHealth = applicationContext.getBean(KafkaBinderHealth.class); 51 | assertThat(kafkaBinderHealth).isInstanceOf(CustomHealthIndicator.class); 52 | assertThatThrownBy(() -> applicationContext.getBean(KafkaBinderHealthIndicator.class)).isInstanceOf(NoSuchBeanDefinitionException.class); 53 | applicationContext.close(); 54 | } 55 | 56 | @SpringBootApplication 57 | static class CustomHealthCheckApplication { 58 | 59 | @Bean 60 | public CustomHealthIndicator kafkaBinderHealthIndicator() { 61 | return new CustomHealthIndicator(); 62 | } 63 | } 64 | 65 | static class CustomHealthIndicator implements KafkaBinderHealth { 66 | 67 | @Override 68 | public Health health() { 69 | return null; 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaNullConverterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016-2021 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.integration; 18 | 19 | import java.util.concurrent.CountDownLatch; 20 | import java.util.concurrent.TimeUnit; 21 | import java.util.function.Consumer; 22 | 23 | import org.junit.AfterClass; 24 | import org.junit.BeforeClass; 25 | import org.junit.ClassRule; 26 | import org.junit.Ignore; 27 | import org.junit.Test; 28 | import org.junit.runner.RunWith; 29 | 30 | import org.springframework.beans.factory.annotation.Autowired; 31 | import org.springframework.boot.autoconfigure.EnableAutoConfiguration; 32 | import org.springframework.boot.test.context.SpringBootTest; 33 | import org.springframework.cloud.stream.function.StreamBridge; 34 | import org.springframework.context.ApplicationContext; 35 | import org.springframework.context.annotation.Bean; 36 | import org.springframework.context.annotation.Configuration; 37 | import org.springframework.kafka.annotation.KafkaListener; 38 | import org.springframework.kafka.support.KafkaNull; 39 | import org.springframework.kafka.test.rule.EmbeddedKafkaRule; 40 | import org.springframework.messaging.MessageChannel; 41 | import org.springframework.messaging.handler.annotation.Payload; 42 | import org.springframework.messaging.support.GenericMessage; 43 | import org.springframework.test.annotation.DirtiesContext; 44 | import org.springframework.test.context.junit4.SpringRunner; 45 | 46 | import static org.assertj.core.api.Assertions.assertThat; 47 | 48 | /** 49 | * @author Aldo Sinanaj 50 | * @author Gary Russell 51 | * @author Soby Chacko 52 | */ 53 | @RunWith(SpringRunner.class) 54 | @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { 55 | "spring.kafka.consumer.auto-offset-reset=earliest", 56 | "spring.cloud.stream.function.bindings.inputListen-in-0=kafkaNullInput"}) 57 | @DirtiesContext 58 | public class KafkaNullConverterTest { 59 | 60 | private static final String KAFKA_BROKERS_PROPERTY = "spring.kafka.bootstrap-servers"; 61 | 62 | @Autowired 63 | private ApplicationContext context; 64 | 65 | @Autowired 66 | private KafkaNullConverterTestConfig config; 67 | 68 | @ClassRule 69 | public static EmbeddedKafkaRule kafkaEmbedded = new EmbeddedKafkaRule(1, true); 70 | 71 | @BeforeClass 72 | public static void setup() { 73 | System.setProperty(KAFKA_BROKERS_PROPERTY, 74 | kafkaEmbedded.getEmbeddedKafka().getBrokersAsString()); 75 | } 76 | 77 | @AfterClass 78 | public static void clean() { 79 | System.clearProperty(KAFKA_BROKERS_PROPERTY); 80 | } 81 | 82 | @Test 83 | @Ignore 84 | public void testKafkaNullConverterOutput() throws InterruptedException { 85 | final StreamBridge streamBridge = context.getBean(StreamBridge.class); 86 | 87 | streamBridge.send("kafkaNullOutput", new GenericMessage<>(KafkaNull.INSTANCE)); 88 | 89 | assertThat(this.config.countDownLatchOutput.await(10, TimeUnit.SECONDS)).isTrue(); 90 | assertThat(this.config.outputPayload).isNull(); 91 | } 92 | 93 | @Test 94 | public void testKafkaNullConverterInput() throws InterruptedException { 95 | 96 | final MessageChannel kafkaNullInput = context.getBean("kafkaNullInput", MessageChannel.class); 97 | 98 | kafkaNullInput.send(new GenericMessage<>(KafkaNull.INSTANCE)); 99 | 100 | assertThat(this.config.countDownLatchInput.await(10, TimeUnit.SECONDS)).isTrue(); 101 | assertThat(this.config.inputPayload).isNull(); 102 | } 103 | 104 | @EnableAutoConfiguration 105 | @Configuration 106 | public static class KafkaNullConverterTestConfig { 107 | 108 | final CountDownLatch countDownLatchOutput = new CountDownLatch(1); 109 | 110 | final CountDownLatch countDownLatchInput = new CountDownLatch(1); 111 | 112 | volatile byte[] outputPayload = new byte[0]; 113 | 114 | volatile byte[] inputPayload = new byte[0]; 115 | 116 | @KafkaListener(id = "foo", topics = "kafkaNullOutput") 117 | public void listen(@Payload(required = false) byte[] in) { 118 | this.outputPayload = in; 119 | countDownLatchOutput.countDown(); 120 | } 121 | 122 | @Bean 123 | public Consumer inputListen() { 124 | return in -> { 125 | this.inputPayload = in; 126 | countDownLatchInput.countDown(); 127 | }; 128 | } 129 | 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/KafkaRetryDlqBinderOrContainerTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021-2021 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.integration; 18 | 19 | import java.util.function.BiFunction; 20 | import java.util.function.Consumer; 21 | 22 | import org.apache.kafka.clients.consumer.ConsumerRecord; 23 | import org.apache.kafka.common.TopicPartition; 24 | import org.junit.jupiter.api.Test; 25 | 26 | import org.springframework.beans.factory.annotation.Autowired; 27 | import org.springframework.boot.autoconfigure.SpringBootApplication; 28 | import org.springframework.boot.test.context.SpringBootTest; 29 | import org.springframework.cloud.stream.binder.Binding; 30 | import org.springframework.cloud.stream.binder.kafka.ListenerContainerWithDlqAndRetryCustomizer; 31 | import org.springframework.cloud.stream.binding.BindingsLifecycleController; 32 | import org.springframework.context.annotation.Bean; 33 | import org.springframework.kafka.core.KafkaOperations; 34 | import org.springframework.kafka.listener.AbstractMessageListenerContainer; 35 | import org.springframework.kafka.listener.CommonErrorHandler; 36 | import org.springframework.kafka.listener.ConsumerRecordRecoverer; 37 | import org.springframework.kafka.listener.DeadLetterPublishingRecoverer; 38 | import org.springframework.kafka.listener.DefaultErrorHandler; 39 | import org.springframework.kafka.support.ExponentialBackOffWithMaxRetries; 40 | import org.springframework.kafka.test.context.EmbeddedKafka; 41 | import org.springframework.kafka.test.utils.KafkaTestUtils; 42 | import org.springframework.lang.Nullable; 43 | import org.springframework.test.annotation.DirtiesContext; 44 | import org.springframework.util.backoff.BackOff; 45 | 46 | import static org.assertj.core.api.Assertions.assertThat; 47 | import static org.mockito.Mockito.mock; 48 | 49 | /** 50 | * @author Gary Russell 51 | */ 52 | @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { 53 | "spring.cloud.function.definition=retryInBinder;retryInContainer", 54 | "spring.cloud.stream.bindings.retryInBinder-in-0.group=foo", 55 | "spring.cloud.stream.bindings.retryInContainer-in-0.group=bar", 56 | "spring.cloud.stream.kafka.bindings.retryInBinder-in-0.consumer.enable-dlq=true", 57 | "spring.cloud.stream.kafka.bindings.retryInContainer-in-0.consumer.enable-dlq=true"}) 58 | @EmbeddedKafka(bootstrapServersProperty = "spring.kafka.bootstrap-servers") 59 | @DirtiesContext 60 | public class KafkaRetryDlqBinderOrContainerTests { 61 | 62 | @Test 63 | public void retryAndDlqInRightPlace(@Autowired BindingsLifecycleController controller) { 64 | Binding retryInBinder = controller.queryState("retryInBinder-in-0"); 65 | assertThat(KafkaTestUtils.getPropertyValue(retryInBinder, "lifecycle.retryTemplate")).isNotNull(); 66 | assertThat(KafkaTestUtils.getPropertyValue(retryInBinder, 67 | "lifecycle.messageListenerContainer.commonErrorHandler")).isNull(); 68 | Binding retryInContainer = controller.queryState("retryInContainer-in-0"); 69 | assertThat(KafkaTestUtils.getPropertyValue(retryInContainer, "lifecycle.retryTemplate")).isNull(); 70 | assertThat(KafkaTestUtils.getPropertyValue(retryInContainer, 71 | "lifecycle.messageListenerContainer.commonErrorHandler")).isInstanceOf(CommonErrorHandler.class); 72 | assertThat(KafkaTestUtils.getPropertyValue(retryInContainer, 73 | "lifecycle.messageListenerContainer.commonErrorHandler.failureTracker.backOff")) 74 | .isInstanceOf(ExponentialBackOffWithMaxRetries.class); 75 | } 76 | 77 | @SpringBootApplication 78 | public static class ConfigCustomizerTestConfig { 79 | 80 | @Bean 81 | public Consumer retryInBinder() { 82 | return str -> { }; 83 | } 84 | 85 | @Bean 86 | public Consumer retryInContainer() { 87 | return str -> { }; 88 | } 89 | 90 | @Bean 91 | ListenerContainerWithDlqAndRetryCustomizer cust() { 92 | return new ListenerContainerWithDlqAndRetryCustomizer() { 93 | 94 | @Override 95 | public void configure(AbstractMessageListenerContainer container, String destinationName, 96 | String group, 97 | BiFunction, Exception, TopicPartition> dlqDestinationResolver, 98 | @Nullable BackOff backOff) { 99 | 100 | if (destinationName.contains("Container")) { 101 | ConsumerRecordRecoverer dlpr = new DeadLetterPublishingRecoverer(mock(KafkaOperations.class), 102 | dlqDestinationResolver); 103 | container.setCommonErrorHandler(new DefaultErrorHandler(dlpr, backOff)); 104 | } 105 | } 106 | 107 | @Override 108 | public boolean retryAndDlqInBinding(String destinationName, String group) { 109 | return !destinationName.contains("Container"); 110 | } 111 | 112 | }; 113 | } 114 | 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/BaseKafkaBinderTopicPropertiesUpdateTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2021 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.integration.topic.configs; 18 | 19 | import java.util.function.Function; 20 | 21 | import org.junit.AfterClass; 22 | import org.junit.BeforeClass; 23 | import org.junit.ClassRule; 24 | import org.junit.runner.RunWith; 25 | 26 | import org.springframework.boot.autoconfigure.EnableAutoConfiguration; 27 | import org.springframework.boot.test.context.SpringBootTest; 28 | import org.springframework.context.annotation.Bean; 29 | import org.springframework.kafka.test.rule.EmbeddedKafkaRule; 30 | import org.springframework.test.annotation.DirtiesContext; 31 | import org.springframework.test.context.junit4.SpringRunner; 32 | 33 | /** 34 | * @author Heiko Does 35 | * @author Soby Chacko 36 | */ 37 | @RunWith(SpringRunner.class) 38 | @SpringBootTest( 39 | classes = BaseKafkaBinderTopicPropertiesUpdateTest.TopicAutoConfigsTestConfig.class, 40 | webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { 41 | "spring.cloud.stream.function.bindings.process-in-0=standard-in", 42 | "spring.cloud.stream.function.bindings.process-out-0=standard-out", 43 | "spring.cloud.stream.kafka.bindings.standard-out.producer.topic.properties.retention.ms=9001", 44 | "spring.cloud.stream.kafka.default.producer.topic.properties.retention.ms=-1", 45 | "spring.cloud.stream.kafka.bindings.standard-in.consumer.topic.properties.retention.ms=9001", 46 | "spring.cloud.stream.kafka.default.consumer.topic.properties.retention.ms=-1" 47 | }) 48 | @DirtiesContext 49 | public abstract class BaseKafkaBinderTopicPropertiesUpdateTest { 50 | 51 | private static final String KAFKA_BROKERS_PROPERTY = "spring.cloud.stream.kafka.binder.brokers"; 52 | 53 | @ClassRule 54 | public static EmbeddedKafkaRule kafkaEmbedded = new EmbeddedKafkaRule(1, true, "standard-in", "standard-out"); 55 | 56 | @BeforeClass 57 | public static void setup() { 58 | System.setProperty(KAFKA_BROKERS_PROPERTY, 59 | kafkaEmbedded.getEmbeddedKafka().getBrokersAsString()); 60 | } 61 | 62 | @AfterClass 63 | public static void clean() { 64 | System.clearProperty(KAFKA_BROKERS_PROPERTY); 65 | } 66 | 67 | @EnableAutoConfiguration 68 | public static class TopicAutoConfigsTestConfig { 69 | 70 | @Bean 71 | public Function process() { 72 | return payload -> payload; 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/DisabledKafkaBinderTopicPropertiesUpdateTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.integration.topic.configs; 18 | 19 | import java.util.Arrays; 20 | import java.util.HashMap; 21 | import java.util.Map; 22 | import java.util.concurrent.TimeUnit; 23 | 24 | import org.apache.kafka.clients.admin.AdminClient; 25 | import org.apache.kafka.clients.admin.AdminClientConfig; 26 | import org.apache.kafka.clients.admin.Config; 27 | import org.apache.kafka.clients.admin.DescribeConfigsResult; 28 | import org.apache.kafka.common.KafkaFuture; 29 | import org.apache.kafka.common.config.ConfigResource; 30 | import org.junit.Test; 31 | 32 | import static org.assertj.core.api.Assertions.assertThat; 33 | 34 | /** 35 | * @author Taras Danylchuk 36 | */ 37 | public class DisabledKafkaBinderTopicPropertiesUpdateTest extends BaseKafkaBinderTopicPropertiesUpdateTest { 38 | 39 | @Test 40 | public void testKafkaBinderShouldNotUpdateTopicConfigurationOnDisabledFeature() throws Exception { 41 | Map adminClientConfig = new HashMap<>(); 42 | adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaEmbedded.getEmbeddedKafka().getBrokersAsString()); 43 | AdminClient adminClient = AdminClient.create(adminClientConfig); 44 | ConfigResource standardInConfigResource = new ConfigResource(ConfigResource.Type.TOPIC, "standard-in"); 45 | ConfigResource standardOutConfigResource = new ConfigResource(ConfigResource.Type.TOPIC, "standard-out"); 46 | DescribeConfigsResult describeConfigsResult = adminClient.describeConfigs(Arrays 47 | .asList(standardInConfigResource, standardOutConfigResource)); 48 | KafkaFuture> kafkaFuture = describeConfigsResult.all(); 49 | Map configResourceConfigMap = kafkaFuture.get(3, TimeUnit.SECONDS); 50 | Config standardInTopicConfig = configResourceConfigMap.get(standardInConfigResource); 51 | assertThat(standardInTopicConfig.get("retention.ms").value()).isEqualTo("604800000"); 52 | 53 | Config standardOutTopicConfig = configResourceConfigMap.get(standardOutConfigResource); 54 | assertThat(standardOutTopicConfig.get("retention.ms").value()).isEqualTo("604800000"); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/java/org/springframework/cloud/stream/binder/kafka/integration/topic/configs/KafkaBinderTopicPropertiesUpdateTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.springframework.cloud.stream.binder.kafka.integration.topic.configs; 18 | 19 | import java.util.Arrays; 20 | import java.util.HashMap; 21 | import java.util.Map; 22 | import java.util.concurrent.TimeUnit; 23 | 24 | import org.apache.kafka.clients.admin.AdminClient; 25 | import org.apache.kafka.clients.admin.AdminClientConfig; 26 | import org.apache.kafka.clients.admin.Config; 27 | import org.apache.kafka.clients.admin.DescribeConfigsResult; 28 | import org.apache.kafka.common.KafkaFuture; 29 | import org.apache.kafka.common.config.ConfigResource; 30 | import org.junit.Test; 31 | 32 | import org.springframework.test.context.TestPropertySource; 33 | 34 | import static org.assertj.core.api.Assertions.assertThat; 35 | 36 | /** 37 | * @author Heiko Does 38 | */ 39 | @TestPropertySource(properties = "spring.cloud.stream.kafka.binder.autoAlterTopics=true") 40 | public class KafkaBinderTopicPropertiesUpdateTest extends BaseKafkaBinderTopicPropertiesUpdateTest { 41 | 42 | @Test 43 | public void testKafkaBinderUpdateTopicConfiguration() throws Exception { 44 | Map adminClientConfig = new HashMap<>(); 45 | adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaEmbedded.getEmbeddedKafka().getBrokersAsString()); 46 | AdminClient adminClient = AdminClient.create(adminClientConfig); 47 | ConfigResource standardInConfigResource = new ConfigResource(ConfigResource.Type.TOPIC, "standard-in"); 48 | ConfigResource standardOutConfigResource = new ConfigResource(ConfigResource.Type.TOPIC, "standard-out"); 49 | DescribeConfigsResult describeConfigsResult = adminClient.describeConfigs(Arrays 50 | .asList(standardInConfigResource, standardOutConfigResource)); 51 | KafkaFuture> kafkaFuture = describeConfigsResult.all(); 52 | Map configResourceConfigMap = kafkaFuture.get(3, TimeUnit.SECONDS); 53 | Config standardInTopicConfig = configResourceConfigMap.get(standardInConfigResource); 54 | assertThat(standardInTopicConfig.get("retention.ms").value()).isEqualTo("9001"); 55 | 56 | Config standardOutTopicConfig = configResourceConfigMap.get(standardOutConfigResource); 57 | assertThat(standardOutTopicConfig.get("retention.ms").value()).isEqualTo("9001"); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/resources/binder-config-autoconfig.properties: -------------------------------------------------------------------------------- 1 | spring.kafka.producer.keySerializer=org.apache.kafka.common.serialization.LongSerializer 2 | spring.kafka.producer.valueSerializer=org.apache.kafka.common.serialization.LongSerializer 3 | spring.kafka.consumer.keyDeserializer=org.apache.kafka.common.serialization.LongDeserializer 4 | spring.kafka.consumer.valueDeserializer=org.apache.kafka.common.serialization.LongDeserializer 5 | spring.kafka.producer.batchSize=10 6 | spring.kafka.bootstrapServers=10.98.09.199:9092,10.98.09.196:9092 7 | spring.kafka.producer.compressionType=snappy 8 | # Test consumer properties 9 | spring.kafka.consumer.auto-offset-reset=earliest 10 | spring.kafka.consumer.group-id=groupIdFromBootConfig 11 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/resources/binder-config.properties: -------------------------------------------------------------------------------- 1 | spring.cloud.stream.kafka.binder.brokers=10.98.09.199:9082 2 | -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/resources/jaas-sample-kafka-only.conf: -------------------------------------------------------------------------------- 1 | KafkaClient { 2 | com.sun.security.auth.module.Krb5LoginModule required 3 | useKeyTab=true 4 | storeKey=true 5 | keyTab="/etc/security/keytabs/kafka_client.keytab" 6 | principal="kafka-client-1@EXAMPLE.COM"; 7 | }; -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/resources/jaas-sample-with-zk.conf: -------------------------------------------------------------------------------- 1 | KafkaClient { 2 | com.sun.security.auth.module.Krb5LoginModule required 3 | useKeyTab=true 4 | storeKey=true 5 | keyTab="/etc/security/keytabs/kafka_client.keytab" 6 | principal="kafka-client-1@EXAMPLE.COM"; 7 | }; 8 | Client { 9 | com.sun.security.auth.module.Krb5LoginModule required 10 | useKeyTab=true 11 | storeKey=true 12 | keyTab="/etc/security/keytabs/zk_client.keytab" 13 | principal="zk-client-1@EXAMPLE.COM"; 14 | }; -------------------------------------------------------------------------------- /spring-cloud-stream-binder-kafka/src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{ISO8601} %5p %t %c{2}:%L - %m%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /update-version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #Execute this script from local checkout of spring cloud stream 4 | 5 | ./mvnw versions:update-parent -DparentVersion=[0.0.1,$2] -Pspring -DgenerateBackupPoms=false -DallowSnapshots=true 6 | ./mvnw versions:set -DnewVersion=$1 -DgenerateBackupPoms=false 7 | 8 | 9 | 10 | lines=$(find . -name 'pom.xml' | xargs egrep "SNAPSHOT|M[0-9]|RC[0-9]" | grep -v regex | wc -l) 11 | if [ $lines -eq 0 ]; then 12 | echo "No snapshots found" 13 | else 14 | echo "Snapshots found." 15 | fi 16 | 17 | lines=$(find . -name 'pom.xml' | xargs egrep "M[0-9]" | grep -v regex | wc -l) 18 | if [ $lines -eq 0 ]; then 19 | echo "No milestones found" 20 | else 21 | echo "Milestones found." 22 | fi 23 | 24 | lines=$(find . -name 'pom.xml' | xargs egrep "RC[0-9]" | grep -v regex | wc -l) 25 | if [ $lines -eq 0 ]; then 26 | echo "No release candidates found" 27 | else 28 | echo "Release candidates found." 29 | fi 30 | --------------------------------------------------------------------------------