├── example ├── ssl │ ├── password │ └── server.p12 └── docker-compose.yml ├── itests ├── src │ ├── main │ │ └── clients │ │ │ ├── dotnet │ │ │ ├── .dockerignore │ │ │ ├── Poll.cs │ │ │ ├── Sink.cs │ │ │ ├── Dockerfile │ │ │ ├── testrunner.csproj │ │ │ ├── TimeUtils.cs │ │ │ ├── Startup.cs │ │ │ ├── ConsumerPoll.cs │ │ │ ├── ProducerSink.cs │ │ │ ├── PollMaster.cs │ │ │ ├── SinkMaster.cs │ │ │ ├── PollThread.cs │ │ │ └── TestRunner.cs │ │ │ └── java │ │ │ ├── .dockerignore │ │ │ ├── settings.gradle │ │ │ ├── gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ │ ├── Dockerfile │ │ │ ├── build.gradle │ │ │ ├── src │ │ │ └── main │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── dajudge │ │ │ │ │ └── kafkaproxy │ │ │ │ │ └── tests │ │ │ │ │ ├── Startup.java │ │ │ │ │ └── ManyConnectsTest.java │ │ │ │ └── resources │ │ │ │ └── logback.xml │ │ │ ├── gradlew.bat │ │ │ └── gradlew │ └── test │ │ ├── java │ │ └── com │ │ │ └── dajudge │ │ │ └── kafkaproxy │ │ │ └── itest │ │ │ ├── util │ │ │ ├── NullFileSystem.java │ │ │ ├── KafkaProxyContainer.java │ │ │ ├── ToConsoleConsumer.java │ │ │ └── ITest.java │ │ │ ├── DotNetTest.java │ │ │ ├── JavaTest.java │ │ │ └── BaseIntegrationTest.java │ │ └── resources │ │ └── logback-test.xml └── build.gradle ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── CHANGELOG ├── .gitignore ├── core ├── src │ ├── main │ │ ├── resources │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ ├── com.dajudge.kafkaproxy.ca.CertificateAuthorityFactory │ │ │ │ └── com.dajudge.kafkaproxy.config.ConfigSource │ │ └── java │ │ │ └── com │ │ │ └── dajudge │ │ │ └── kafkaproxy │ │ │ ├── config │ │ │ ├── ConfigSource.java │ │ │ ├── Environment.java │ │ │ ├── ApplicationConfig.java │ │ │ ├── RealEnvironment.java │ │ │ ├── UpstreamSslConfigSource.java │ │ │ ├── DownstreamSslConfigSource.java │ │ │ ├── BrokerConfigSource.java │ │ │ └── KeyStoreConfigHelper.java │ │ │ ├── protocol │ │ │ ├── rewrite │ │ │ │ ├── ResponseRewriter.java │ │ │ │ ├── CompositeRewriter.java │ │ │ │ ├── MetadataRewriter.java │ │ │ │ ├── BaseReflectingRewriter.java │ │ │ │ └── FindCoordinatorRewriter.java │ │ │ ├── DecodingKafkaMessageInboundHandler.java │ │ │ ├── EncodingKafkaMessageOutboundHandler.java │ │ │ ├── RewritingKafkaMessageDuplexHandler.java │ │ │ ├── KafkaRequestStore.java │ │ │ └── KafkaMessage.java │ │ │ ├── BrokerMapping.java │ │ │ ├── BrokerMapper.java │ │ │ └── KafkaProxyApplication.java │ └── test │ │ ├── resources │ │ ├── configs │ │ │ ├── config2.yml │ │ │ └── config1.yml │ │ ├── customEntrypoint.sh │ │ └── logback-test.xml │ │ └── java │ │ └── com │ │ └── dajudge │ │ └── kafkaproxy │ │ ├── roundtrip │ │ ├── cluster │ │ │ ├── container │ │ │ │ ├── Version.java │ │ │ │ ├── ZookeeperContainer.java │ │ │ │ └── KafkaContainer.java │ │ │ ├── StarterThread.java │ │ │ ├── KafkaCluster.java │ │ │ ├── CommunicationSetupBuilder.java │ │ │ ├── TestSetup.java │ │ │ ├── KafkaWaitStrategy.java │ │ │ └── KafkaClusterBuilder.java │ │ ├── comm │ │ │ ├── CommunicationSetup.java │ │ │ ├── ClientSslConfig.java │ │ │ ├── ClientSecurity.java │ │ │ ├── PlaintextCommunicationSetup.java │ │ │ ├── ServerSecurity.java │ │ │ ├── PlaintextClientSecurity.java │ │ │ ├── PlaintextServerSecurity.java │ │ │ ├── SslCommunicationSetup.java │ │ │ ├── SslServerSecurity.java │ │ │ └── SslClientSecurity.java │ │ ├── util │ │ │ └── Util.java │ │ ├── ssl │ │ │ ├── KeyStoreData.java │ │ │ └── CertAuthority.java │ │ ├── client │ │ │ └── ClientFactory.java │ │ └── RoundtripTest.java │ │ └── config │ │ ├── BaseOptionalConfigTest.java │ │ ├── BaseConfigTest.java │ │ └── DownstreamSslConfigTest.java └── build.gradle ├── testutil ├── build.gradle └── src │ └── main │ └── java │ └── com │ └── dajudge │ └── kafkaproxy │ └── roundtrip │ └── util │ ├── TestFilesystem.java │ ├── PortFinder.java │ └── TestEnvironment.java ├── CONTRIBUTORS ├── settings.gradle ├── Dockerfile ├── .github ├── dependabot.yaml └── workflows │ ├── publish.yaml │ └── build.yaml ├── publish.sh ├── .workbench └── Dockerfile ├── HEADER ├── app ├── src │ └── main │ │ ├── resources │ │ └── application.properties │ │ └── java │ │ └── com │ │ └── dajudge │ │ └── kafkaproxy │ │ └── Startup.java └── build.gradle ├── workbench ├── pmd.xml ├── gradlew.bat └── gradlew /example/ssl/password: -------------------------------------------------------------------------------- 1 | test -------------------------------------------------------------------------------- /itests/src/main/clients/dotnet/.dockerignore: -------------------------------------------------------------------------------- 1 | .idea/ -------------------------------------------------------------------------------- /itests/src/main/clients/java/.dockerignore: -------------------------------------------------------------------------------- 1 | .gradle/ 2 | build/ 3 | out/ 4 | -------------------------------------------------------------------------------- /itests/src/main/clients/java/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = "test-java" 2 | -------------------------------------------------------------------------------- /example/ssl/server.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dajudge/kafkaproxy/HEAD/example/ssl/server.p12 -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dajudge/kafkaproxy/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | next release 2 | - Use Kafka client v2.5.0 3 | - Allow customization of bind address 4 | - Allow customization of HTTP port 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.log 2 | build/ 3 | out/ 4 | .idea/ 5 | .gradle/ 6 | 7 | .settings/ 8 | .classpath 9 | .project 10 | bin/ 11 | obj/ 12 | *.user -------------------------------------------------------------------------------- /core/src/main/resources/META-INF/services/com.dajudge.kafkaproxy.ca.CertificateAuthorityFactory: -------------------------------------------------------------------------------- 1 | com.dajudge.kafkaproxy.ca.NullCertificateAuthorityFactory 2 | 3 | -------------------------------------------------------------------------------- /itests/src/main/clients/java/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dajudge/kafkaproxy/HEAD/itests/src/main/clients/java/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /testutil/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id "java" 3 | } 4 | 5 | dependencies { 6 | implementation project(":core") 7 | implementation libProxyBase 8 | } 9 | 10 | -------------------------------------------------------------------------------- /itests/src/main/clients/dotnet/Poll.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace testrunner 4 | { 5 | public interface Poll : IDisposable 6 | { 7 | string Poll(); 8 | } 9 | } -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | kafkaproxy has been made possible thanks to contributions by these people: 2 | 3 | Alex Stockinger (mail@alexstockinger.de) 4 | Alfred Schmid (alfred.schmid@steadforce.com) 5 | -------------------------------------------------------------------------------- /itests/src/main/clients/dotnet/Sink.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace testrunner 4 | { 5 | public interface Sink : IDisposable 6 | { 7 | public void Publish(string message); 8 | } 9 | } -------------------------------------------------------------------------------- /itests/src/main/clients/dotnet/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/dotnet/sdk:3.1-alpine3.13 2 | 3 | COPY . /app/ 4 | WORKDIR /app/ 5 | 6 | RUN dotnet restore 7 | RUN dotnet build 8 | 9 | ENTRYPOINT [""] 10 | CMD ["sleep", "600"] -------------------------------------------------------------------------------- /core/src/main/resources/META-INF/services/com.dajudge.kafkaproxy.config.ConfigSource: -------------------------------------------------------------------------------- 1 | com.dajudge.kafkaproxy.config.BrokerConfigSource 2 | com.dajudge.kafkaproxy.config.DownstreamSslConfigSource 3 | com.dajudge.kafkaproxy.config.UpstreamSslConfigSource -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = "kafkaproxy" 2 | 3 | include rootDir.listFiles().findAll { 4 | it.isDirectory() && !['.','..'].contains(it) && file("$it.absolutePath/build.gradle").exists() 5 | }.collect { 6 | it.getName() 7 | }.toArray(new java.lang.String[0]) -------------------------------------------------------------------------------- /itests/src/main/clients/java/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:jammy-20221130@sha256:965fbcae990b0467ed5657caceaec165018ef44a4d2d46c7cdea80a9dff0d1ea 2 | ARG APP_DIR=app/build 3 | COPY $APP_DIR/*-runner /work/application 4 | WORKDIR /work/ 5 | EXPOSE 8080 6 | USER 1001 7 | 8 | CMD ["./application", "-Xmx64m", "-Dquarkus.http.host=0.0.0.0"] -------------------------------------------------------------------------------- /itests/src/main/clients/java/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ghcr.io/graalvm/graalvm-ce:java11-20.3 2 | 3 | COPY . /app/ 4 | WORKDIR /app/ 5 | 6 | ARG KAFKA_CLIENT_VERSION_JAVA 7 | RUN echo "Kafka client version: ${KAFKA_CLIENT_VERSION_JAVA}" 8 | RUN ./gradlew build 9 | 10 | ENTRYPOINT [""] 11 | CMD ["sleep", "600"] -------------------------------------------------------------------------------- /.github/dependabot.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gradle" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | assignees: 8 | - "dajudge" 9 | reviewers: 10 | - "dajudge" 11 | labels: 12 | - "type/dependency" 13 | open-pull-requests-limit: 10 -------------------------------------------------------------------------------- /itests/src/main/clients/dotnet/testrunner.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1 5 | testrunner 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /itests/src/main/clients/dotnet/TimeUtils.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace testrunner 4 | { 5 | public class TimeUtils 6 | { 7 | private static readonly DateTime Epoch = new DateTime 8 | (1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); 9 | 10 | public static long CurrentTimeMillis() 11 | { 12 | return (long) (DateTime.UtcNow - Epoch).TotalMilliseconds; 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /publish.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -e 4 | 5 | DIR="$(cd "$(dirname "$0")"; pwd)"; 6 | 7 | TAG=${GITHUB_REF##*/} 8 | if [ $TAG = "master" ]; then 9 | TAG="devel" 10 | fi 11 | 12 | IMAGE_NAME="dajudge/kafkaproxy:$TAG" 13 | 14 | echo "Building $IMAGE_NAME..." 15 | docker build . -t $IMAGE_NAME 16 | 17 | echo "Pushing $IMAGE_NAME to Docker Hub..." 18 | docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD 19 | docker push $IMAGE_NAME -------------------------------------------------------------------------------- /.workbench/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ghcr.io/graalvm/graalvm-ce:java11-20.3 2 | 3 | RUN yum install -y bash curl sudo wget tar 4 | 5 | RUN mkdir -p /opt && \ 6 | wget -qO- https://download.docker.com/linux/static/stable/x86_64/docker-17.12.1-ce.tgz | tar xvz -C /opt && \ 7 | ln -s /opt/docker/docker /usr/bin/docker 8 | 9 | RUN curl -L "https://github.com/docker/compose/releases/download/1.25.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/bin/docker-compose && \ 10 | chmod 755 /usr/bin/docker-compose 11 | 12 | RUN gu install native-image 13 | 14 | RUN yum install -y gcc glibc-devel zlib-devel 15 | 16 | CMD ["bash"] -------------------------------------------------------------------------------- /HEADER: -------------------------------------------------------------------------------- 1 | Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /itests/src/main/clients/dotnet/Startup.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using Confluent.Kafka; 4 | using Confluent.Kafka.Admin; 5 | 6 | namespace testrunner 7 | { 8 | public class Startup 9 | { 10 | public static void Main(string[] args) 11 | { 12 | var bootstrapServers = args[0]; 13 | var runner = new TestRunner(bootstrapServers, 10, 10); 14 | runner.CreateTopic(); 15 | runner.Produce(); 16 | runner.WaitForConsumers(); 17 | runner.Report(); 18 | runner.Dispose(); 19 | Environment.ExitCode = runner.IsSucceeded() ? 0 : 1; 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /.github/workflows/publish.yaml: -------------------------------------------------------------------------------- 1 | name: Publish release to dockerhub 2 | on: 3 | release: 4 | types: [created] 5 | jobs: 6 | publish: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | - name: Set up GraalVM 11 | uses: ayltai/setup-graalvm@v1 12 | with: 13 | java-version: 11 14 | graalvm-version: 21.2.0 15 | native-image: true 16 | - name: Build with Gradle 17 | run: ./gradlew build -x test -Dquarkus.package.type=native 18 | - name: Publish to Docker Hub 19 | run: ./publish.sh 20 | env: 21 | DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} 22 | DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} 23 | -------------------------------------------------------------------------------- /itests/src/main/clients/dotnet/ConsumerPoll.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Confluent.Kafka; 3 | 4 | namespace testrunner 5 | { 6 | public class ConsumerPoll : Poll 7 | { 8 | private readonly IConsumer _consumer; 9 | 10 | public ConsumerPoll( 11 | in IConsumer consumer 12 | ) 13 | { 14 | _consumer = consumer; 15 | } 16 | 17 | public string Poll() 18 | { 19 | return _consumer.Consume(TimeSpan.FromMilliseconds(100))?.Message?.Value; 20 | } 21 | 22 | public void Cancel() 23 | { 24 | _consumer.Close(); 25 | } 26 | 27 | public void Dispose() 28 | { 29 | _consumer.Dispose(); 30 | } 31 | } 32 | } -------------------------------------------------------------------------------- /core/src/test/resources/configs/config2.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | proxies: 16 | - proxy: 17 | hostname: kafka.example.com 18 | port: 39092 19 | broker: 20 | hostname: broker1.kafka.local 21 | port: 9092 22 | -------------------------------------------------------------------------------- /itests/src/main/clients/dotnet/ProducerSink.cs: -------------------------------------------------------------------------------- 1 | using Confluent.Kafka; 2 | 3 | namespace testrunner 4 | { 5 | public class ProducerSink : Sink 6 | { 7 | private readonly IProducer _producer; 8 | private readonly string _topic; 9 | 10 | public ProducerSink( 11 | in IProducer producer, 12 | in string topic 13 | ) 14 | { 15 | _producer = producer; 16 | _topic = topic; 17 | } 18 | 19 | public void Publish(string message) 20 | { 21 | _producer.ProduceAsync(_topic, new Message 22 | { 23 | Key = message, 24 | Value = message 25 | }).GetAwaiter().GetResult(); 26 | } 27 | 28 | public void Dispose() 29 | { 30 | _producer.Dispose(); 31 | } 32 | } 33 | } -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/cluster/container/Version.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.cluster.container; 19 | 20 | public class Version { 21 | static final String CP_VERSION = "7.0.3"; 22 | } 23 | -------------------------------------------------------------------------------- /core/src/test/resources/customEntrypoint.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | echo "Waiting for config..." 17 | while [ ! -f /tmp/config.env ]; do sleep 1; done 18 | 19 | echo "Kafka configuration: $(cat /tmp/config.env)" 20 | . /tmp/config.env 21 | 22 | KAFKA_ADVERTISED_LISTENERS=${KAFKA_ADVERTISED_LISTENERS} /etc/confluent/docker/run -------------------------------------------------------------------------------- /itests/src/main/clients/java/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: "java" 2 | 3 | repositories { 4 | mavenCentral() 5 | } 6 | 7 | def kafkaClientVersion = System.getenv("KAFKA_CLIENT_VERSION_JAVA") ?: "2.6.0" 8 | 9 | dependencies { 10 | implementation "org.apache.kafka:kafka-clients:${kafkaClientVersion}" 11 | implementation 'ch.qos.logback:logback-classic:1.2.3' 12 | 13 | // Kafka client needs jackson 14 | runtimeOnly 'com.fasterxml.jackson.core:jackson-databind:2.11.2' 15 | } 16 | 17 | task copyRuntimeLibs(type: Copy) { 18 | into new File(project.buildDir, "libs") 19 | from configurations.runtimeClasspath 20 | } 21 | 22 | jar { 23 | manifest { 24 | attributes( 25 | 'Manifest-Version': "1.0", 26 | 'Main-Class': "com.dajudge.kafkaproxy.tests.Startup", 27 | "Class-Path": configurations.runtimeClasspath.collect { it }.join(" ") 28 | ) 29 | } 30 | } 31 | 32 | build.dependsOn copyRuntimeLibs -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/comm/CommunicationSetup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.comm; 19 | 20 | public interface CommunicationSetup { 21 | ServerSecurity getServerSecurity(String dn); 22 | 23 | ClientSecurity getClientSecurity(); 24 | } 25 | -------------------------------------------------------------------------------- /app/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | quarkus.http.port=${KAFKAPROXY_HTTP_PORT:8080} 16 | quarkus.log.level=${KAFKAPROXY_LOG_LEVEL:INFO} 17 | quarkus.log.console.json=${KAFKAPROXY_ENABLE_JSON_LOGGING:false} 18 | quarkus.http.host=${KAFKAPROXY_BIND_ADDRESS:0.0.0.0} 19 | quarkus.log.console.format=%d{HH:mm:ss} %-5p [%c{2.}] (%t) [%X{CHANNEL_ID}] %s%e%n -------------------------------------------------------------------------------- /core/src/test/resources/configs/config1.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | proxies: 16 | - proxy: 17 | hostname: kafka.example.com 18 | port: 39092 19 | broker: 20 | hostname: broker1.kafka.local 21 | port: 9092 22 | - proxy: 23 | hostname: kafka.example.com 24 | port: 39093 25 | broker: 26 | hostname: broker2.kafka.local 27 | port: 9092 -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/config/ConfigSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.config; 19 | 20 | import java.util.Optional; 21 | 22 | public interface ConfigSource { 23 | String PREFIX = "KAFKAPROXY_"; 24 | 25 | Class getConfigClass(); 26 | 27 | Optional parse(final Environment environment); 28 | } 29 | -------------------------------------------------------------------------------- /core/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: "java" 2 | 3 | dependencies { 4 | implementation libProxyBase 5 | // We use the transport objects from the kafka client library 6 | implementation libKafkaClient 7 | implementation 'org.jetbrains:annotations:20.1.0' 8 | // Kafka client needs jackson 9 | runtimeOnly 'com.fasterxml.jackson.core:jackson-databind:2.11.2' 10 | // Netty is our network layer 11 | implementation 'io.netty:netty-all:4.1.45.Final' 12 | // For parsing configs 13 | implementation 'org.yaml:snakeyaml:1.25' 14 | // Used for TLS hostname verification 15 | implementation 'org.apache.httpcomponents.client5:httpclient5:5.0-beta6' 16 | // The logging API 17 | implementation 'org.slf4j:slf4j-api:1.7.29' 18 | 19 | testImplementation libTestContainers 20 | testImplementation libJunit4 21 | testImplementation libProxyBaseTestca 22 | testImplementation project(":testutil") 23 | testRuntimeOnly libLogback 24 | } 25 | 26 | test { 27 | testLogging { 28 | showStandardStreams = true 29 | } 30 | } -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/protocol/rewrite/ResponseRewriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.protocol.rewrite; 19 | 20 | import com.dajudge.kafkaproxy.protocol.KafkaMessage; 21 | import org.apache.kafka.common.requests.RequestHeader; 22 | 23 | public interface ResponseRewriter { 24 | KafkaMessage rewrite(RequestHeader requestHeader, KafkaMessage message); 25 | } 26 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/comm/ClientSslConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.comm; 19 | 20 | public interface ClientSslConfig { 21 | String getKeyStoreLocation(); 22 | 23 | char[] getKeyStorePassword(); 24 | 25 | char[] getKeyPassword(); 26 | 27 | String getKeyStoreType(); 28 | 29 | String getProxyCertStrategy(); 30 | } 31 | -------------------------------------------------------------------------------- /workbench: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | DIR="$(cd "$(dirname "$0")"; pwd)"; 4 | 5 | IMG=$(docker build -q $DIR/.workbench) 6 | 7 | DOCKER_ARGS="-it" 8 | 9 | if [ "$#" -eq 0 ]; then 10 | CMD=bash 11 | else 12 | CMD="$@" 13 | fi 14 | 15 | if [ ! -z "$CI_PROJECT_DIR" ]; then 16 | CONTAINER_OPTS="-v /root:/root -v /${CI_PROJECT_DIR}:/${CI_PROJECT_DIR}" 17 | USER=root 18 | else 19 | CONTAINER_OPTS="-v $HOME:$HOME -it" 20 | fi 21 | 22 | docker run --rm \ 23 | --net host \ 24 | -v /etc/passwd:/etc/passwd \ 25 | -v /etc/group:/etc/group \ 26 | -v /tmp:/tmp \ 27 | -e HOME:${HOME} \ 28 | -e http_proxy="$http_proxy" \ 29 | -e https_proxy="$https_proxy" \ 30 | -e no_proxy="$no_proxy" \ 31 | -e CI_COMMIT_REF_NAME="$CI_COMMIT_REF_NAME" \ 32 | -e CI_COMMIT_TAG="$CI_COMMIT_REF_NAME" \ 33 | -e DOCKERHUB_USERNAME="$DOCKERHUB_USERNAME" \ 34 | -e DOCKERHUB_PASSWORD="$DOCKERHUB_PASSWORD" \ 35 | -v "${DIR}":/project \ 36 | -v /var/run/docker.sock:/var/run/docker.sock \ 37 | $CONTAINER_OPTS $IMG \ 38 | sudo -u ${USER} -E sh -c "cd ${DIR}; ${CMD}" -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/comm/ClientSecurity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.comm; 19 | 20 | public interface ClientSecurity { 21 | String getProtocol(); 22 | 23 | String getTrustStoreLocation(); 24 | 25 | char[] getTrustStorePassword(); 26 | 27 | String getTrustStoreType(); 28 | 29 | ClientSslConfig newClient(final String dn); 30 | } 31 | -------------------------------------------------------------------------------- /itests/src/test/java/com/dajudge/kafkaproxy/itest/util/NullFileSystem.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package com.dajudge.kafkaproxy.itest.util; 18 | 19 | import com.dajudge.proxybase.certs.Filesystem; 20 | 21 | import java.io.IOException; 22 | 23 | public class NullFileSystem implements Filesystem { 24 | @Override 25 | public byte[] readFile(final String path) throws IOException { 26 | return new byte[0]; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /itests/src/test/java/com/dajudge/kafkaproxy/itest/util/KafkaProxyContainer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package com.dajudge.kafkaproxy.itest.util; 18 | 19 | import org.testcontainers.containers.GenericContainer; 20 | 21 | public class KafkaProxyContainer extends GenericContainer { 22 | public KafkaProxyContainer() { 23 | super("localhost/kafkaproxy/kafkaproxy:latest"); 24 | this.withNetworkMode("host"); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /itests/src/main/clients/java/src/main/java/com/dajudge/kafkaproxy/tests/Startup.java: -------------------------------------------------------------------------------- 1 | package com.dajudge.kafkaproxy.tests; 2 | 3 | import java.util.stream.Stream; 4 | 5 | public class Startup { 6 | public interface Test { 7 | boolean run() throws Exception; 8 | } 9 | 10 | public static void main(final String[] args) { 11 | final boolean succeeded = Stream.of( 12 | new ManyConnectsTest(args[0])::run, 13 | new MultiClientTest(args[0])::run 14 | ).map(test -> { 15 | try { 16 | final boolean result = test.run(); 17 | if (result) { 18 | System.out.println("===> Test run succeeded"); 19 | } else { 20 | System.out.println("===> Test run failed"); 21 | } 22 | return result; 23 | } catch (final Exception e) { 24 | new RuntimeException("===> Test exception", e).printStackTrace(); 25 | return false; 26 | } 27 | }).reduce(true, Boolean::logicalAnd); 28 | System.exit(succeeded ? 0 : 1); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /app/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | id 'io.quarkus' version "1.13.7.Final" 4 | } 5 | 6 | quarkusBuild { 7 | nativeArgs { 8 | autoServiceLoaderRegistration = "true" 9 | enableAllSecurityServices = "true" 10 | } 11 | } 12 | 13 | dependencies { 14 | implementation project(":core") 15 | implementation libProxyBase 16 | 17 | implementation enforcedPlatform("io.quarkus:quarkus-universe-bom:1.13.7.Final") { 18 | exclude group: 'org.apache.kafka', module: 'kafka-clients' 19 | } 20 | implementation 'io.quarkus:quarkus-resteasy' 21 | implementation 'io.quarkus:quarkus-jackson' 22 | implementation 'io.quarkus:quarkus-kafka-client' 23 | implementation 'io.quarkus:quarkus-jsonp' 24 | implementation 'io.quarkus:quarkus-logging-json' 25 | implementation 'io.quarkus:quarkus-smallrye-metrics' 26 | // Used for TLS hostname verification 27 | implementation 'org.apache.httpcomponents.client5:httpclient5:5.0-beta6' 28 | // Used for certificate signing 29 | implementation 'org.bouncycastle:bcpkix-jdk15on:1.64' 30 | 31 | // Force override to latest kafka client 32 | implementation libKafkaClient 33 | } 34 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/config/Environment.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.config; 19 | 20 | import java.util.Optional; 21 | 22 | public interface Environment { 23 | String requiredString(final String variable, final String defaultValue); 24 | 25 | String requiredString(String variable); 26 | 27 | Optional optionalString(final String variable); 28 | 29 | Optional optionalInt(final String variable); 30 | 31 | boolean requiredBoolean(String variable, boolean defaultValue); 32 | 33 | int requiredInt(String proxy_base_port); 34 | } 35 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/comm/PlaintextCommunicationSetup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.comm; 19 | 20 | public class PlaintextCommunicationSetup implements CommunicationSetup { 21 | @Override 22 | public ServerSecurity getServerSecurity(final String dn) { 23 | return new PlaintextServerSecurity(); 24 | } 25 | 26 | @Override 27 | public ClientSecurity getClientSecurity() { 28 | return new PlaintextClientSecurity(); 29 | } 30 | 31 | @Override 32 | public String toString() { 33 | return "PlaintextCommunicationSetup{}"; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /itests/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | 20 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} [%X{CHANNEL_TYPE}/%X{CHANNEL_ID}] - %msg%n 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/protocol/DecodingKafkaMessageInboundHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.protocol; 19 | 20 | import com.dajudge.proxybase.AbstractChunkedMessageStreamInboundHandler; 21 | import io.netty.channel.ChannelHandlerContext; 22 | 23 | public class DecodingKafkaMessageInboundHandler extends AbstractChunkedMessageStreamInboundHandler { 24 | @Override 25 | protected void onMessageComplete(final ChannelHandlerContext ctx, final KafkaMessage msg) { 26 | ctx.fireChannelRead(msg); 27 | } 28 | 29 | @Override 30 | protected KafkaMessage createNewMessage() { 31 | return new KafkaMessage(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/config/BaseOptionalConfigTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.config; 19 | 20 | import com.dajudge.kafkaproxy.roundtrip.util.TestEnvironment; 21 | import org.junit.Test; 22 | 23 | abstract class BaseOptionalConfigTest extends BaseConfigTest { 24 | @Test 25 | public void returns_disabled() { 26 | assertDisabled(disable(new TestEnvironment())); 27 | } 28 | 29 | @Test 30 | public void default_is_disabled() { 31 | assertDisabled(new TestEnvironment()); 32 | } 33 | 34 | abstract TestEnvironment disable(TestEnvironment testEnvironment); 35 | 36 | abstract void assertDisabled(Environment env); 37 | } 38 | -------------------------------------------------------------------------------- /itests/src/test/java/com/dajudge/kafkaproxy/itest/DotNetTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package com.dajudge.kafkaproxy.itest; 18 | 19 | import com.dajudge.kafkaproxy.itest.util.ITest; 20 | import org.junit.ClassRule; 21 | import org.junit.Test; 22 | 23 | import static org.junit.Assert.assertEquals; 24 | 25 | public class DotNetTest extends BaseIntegrationTest { 26 | @ClassRule 27 | public static final ITest CONTAINER = new ITest("localhost/kafkaproxy/itest-dotnet:latest"); 28 | 29 | @Test 30 | public void run() { 31 | final int exitCode = withKafkaProxy(proxyEndpoint -> CONTAINER.exec("dotnet", "run", proxyEndpoint)); 32 | assertEquals("Test run failed", 0, exitCode); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/config/BaseConfigTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.config; 19 | 20 | import com.dajudge.kafkaproxy.roundtrip.util.TestEnvironment; 21 | 22 | import java.util.function.Function; 23 | 24 | import static org.junit.Assert.assertNull; 25 | 26 | abstract class BaseConfigTest { 27 | abstract TestEnvironment fullEnvironment(); 28 | 29 | void assertAllowsUnset(final String envVar, final Function configProperty) { 30 | final Environment env = fullEnvironment() 31 | .withEnv(envVar, null); 32 | assertNull(configProperty.apply(parse(env))); 33 | } 34 | 35 | abstract C parse(Environment e); 36 | } 37 | -------------------------------------------------------------------------------- /testutil/src/main/java/com/dajudge/kafkaproxy/roundtrip/util/TestFilesystem.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.util; 19 | 20 | import com.dajudge.proxybase.certs.Filesystem; 21 | 22 | import java.util.HashMap; 23 | import java.util.Map; 24 | 25 | public class TestFilesystem implements Filesystem { 26 | private Map files = new HashMap<>(); 27 | 28 | public TestFilesystem withFile(final String path, final byte[] data) { 29 | if (path != null) { 30 | files.put(path, data); 31 | } 32 | return this; 33 | } 34 | 35 | @Override 36 | public byte[] readFile(final String path) { 37 | return files.get(path); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | name: CI Build 2 | on: 3 | pull_request: 4 | branches: [ master ] 5 | push: 6 | branches: [ master ] 7 | jobs: 8 | 9 | itest: 10 | runs-on: ubuntu-latest 11 | strategy: 12 | matrix: 13 | java-client-version: [ "2.6.1", "2.8.0", "3.0.0", "3.1.0"] 14 | cp-platform-version: [ 15 | "6.0.3", # Kafka 2.6.3 16 | "6.2.0", # Kafka 2.8.0 17 | "7.0.3", # Kafka 3.0.3 18 | "7.1.1", # Kafka 3.1.1 19 | ] 20 | steps: 21 | - uses: actions/checkout@v2 22 | - name: Set up GraalVM 23 | uses: ayltai/setup-graalvm@v1 24 | with: 25 | java-version: 11 26 | graalvm-version: 21.2.0 27 | native-image: true 28 | - name: Build with Gradle 29 | run: CONFLUENT_PLATFORM_VERSION="${{ matrix.cp-platform-version }}" KAFKA_CLIENT_VERSION_JAVA="${{ matrix.java-client-version }}" ./gradlew clean :itest:test -Dquarkus.package.type=native --stacktrace 30 | 31 | build: 32 | runs-on: ubuntu-latest 33 | steps: 34 | - uses: actions/checkout@v2 35 | - name: Set up GraalVM 36 | uses: ayltai/setup-graalvm@v1 37 | with: 38 | java-version: 11 39 | graalvm-version: 21.2.0 40 | native-image: true 41 | - name: Build with Gradle 42 | run: ./gradlew clean build -x :itest:test -Dquarkus.package.type=native --stacktrace 43 | -------------------------------------------------------------------------------- /pmd.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | 8 | 9 | kafkaproxy PMD ruleset 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/config/DownstreamSslConfigTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.config; 19 | 20 | import com.dajudge.kafkaproxy.roundtrip.util.TestEnvironment; 21 | import com.dajudge.proxybase.config.DownstreamSslConfig; 22 | import org.junit.Test; 23 | 24 | import static org.junit.Assert.assertFalse; 25 | 26 | public class DownstreamSslConfigTest { 27 | @Test 28 | public void downstream_truststore_is_optional() { 29 | final Environment env = new TestEnvironment() 30 | .withEnv("KAFKAPROXY_KAFKA_SSL_ENABLED", "true"); 31 | final DownstreamSslConfig config = new ApplicationConfig(env).require(DownstreamSslConfig.class); 32 | assertFalse(config.getTrustStore().isPresent()); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/protocol/EncodingKafkaMessageOutboundHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.protocol; 19 | 20 | import io.netty.channel.ChannelHandlerContext; 21 | import io.netty.channel.ChannelOutboundHandlerAdapter; 22 | import io.netty.channel.ChannelPromise; 23 | 24 | public class EncodingKafkaMessageOutboundHandler extends ChannelOutboundHandlerAdapter { 25 | @Override 26 | public void write( 27 | final ChannelHandlerContext ctx, 28 | final Object msg, 29 | final ChannelPromise promise 30 | ) throws Exception { 31 | if (msg instanceof KafkaMessage) { 32 | ctx.write(((KafkaMessage) msg).all(), promise); 33 | } else { 34 | super.write(ctx, msg, promise); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/BrokerMapping.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy; 19 | 20 | import com.dajudge.proxybase.config.Endpoint; 21 | 22 | public class BrokerMapping { 23 | private final Endpoint broker; 24 | private final Endpoint proxy; 25 | 26 | public BrokerMapping(final Endpoint broker, final Endpoint proxy) { 27 | this.broker = broker; 28 | this.proxy = proxy; 29 | } 30 | 31 | public Endpoint getBroker() { 32 | return broker; 33 | } 34 | 35 | public Endpoint getProxy() { 36 | return proxy; 37 | } 38 | 39 | @Override 40 | public String toString() { 41 | return "BrokerMapping{" + 42 | "broker=" + broker + 43 | ", proxy=" + proxy + 44 | '}'; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/comm/ServerSecurity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.comm; 19 | 20 | import org.testcontainers.images.builder.Transferable; 21 | 22 | import java.util.function.BiConsumer; 23 | 24 | public interface ServerSecurity { 25 | String getClientProtocol(); 26 | 27 | String getTrustStoreLocation(); 28 | 29 | char[] getTrustStorePassword(); 30 | 31 | String getTrustStoreType(); 32 | 33 | String getKeyStoreLocation(); 34 | 35 | char[] getKeyStorePassword(); 36 | 37 | String getKeyStoreType(); 38 | 39 | char[] getKeyPassword(); 40 | 41 | String getClientAuth(); 42 | 43 | byte[] getKeyStore(); 44 | 45 | byte[] getTrustStore(); 46 | 47 | void uploadKeyStores(BiConsumer uploader); 48 | } 49 | -------------------------------------------------------------------------------- /itests/src/test/java/com/dajudge/kafkaproxy/itest/JavaTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package com.dajudge.kafkaproxy.itest; 18 | 19 | import com.dajudge.kafkaproxy.itest.util.ITest; 20 | import org.junit.ClassRule; 21 | import org.junit.Test; 22 | 23 | import static org.junit.Assert.assertEquals; 24 | 25 | public class JavaTest extends BaseIntegrationTest { 26 | @ClassRule 27 | public static final ITest CONTAINER = new ITest("localhost/kafkaproxy/itest-java:latest") 28 | .withEnv("CONNECTION_ATTEMPTS", "5000"); 29 | 30 | @Test 31 | public void run() { 32 | final int exitCode = withKafkaProxy(proxyEndpoint -> 33 | CONTAINER.exec("/usr/bin/java", "-jar", "/app/build/libs/test-java.jar", proxyEndpoint) 34 | ); 35 | assertEquals("Test run failed", 0, exitCode); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /itests/src/main/clients/dotnet/PollMaster.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace testrunner 5 | { 6 | public delegate Poll PollFactory(); 7 | 8 | public class PollMaster : IDisposable 9 | { 10 | private readonly IList pollThreads = new List(); 11 | private long _lastMessageTimestamp; 12 | 13 | public PollMaster( 14 | in ISet messages, 15 | in int pollThreadCount, 16 | in PollFactory pollFactory 17 | ) 18 | { 19 | for (var i = 0; i < pollThreadCount; i++) 20 | { 21 | pollThreads.Add(new PollThread( 22 | pollFactory(), 23 | messages, 24 | UpdateLastMessageTimestamp 25 | )); 26 | } 27 | 28 | foreach (var pollThread in pollThreads) 29 | { 30 | pollThread.Start(); 31 | } 32 | } 33 | 34 | private void UpdateLastMessageTimestamp() 35 | { 36 | _lastMessageTimestamp = TimeUtils.CurrentTimeMillis(); 37 | } 38 | 39 | public long GetLastMessageTimestamp() 40 | { 41 | return _lastMessageTimestamp; 42 | } 43 | 44 | public void Dispose() 45 | { 46 | foreach (var pollThread in pollThreads) 47 | { 48 | pollThread.Dispose(); 49 | } 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /app/src/main/java/com/dajudge/kafkaproxy/Startup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy; 19 | 20 | 21 | import com.dajudge.kafkaproxy.config.RealEnvironment; 22 | import io.quarkus.runtime.ShutdownEvent; 23 | import io.quarkus.runtime.StartupEvent; 24 | 25 | import javax.enterprise.context.ApplicationScoped; 26 | import javax.enterprise.event.Observes; 27 | 28 | import static com.dajudge.kafkaproxy.KafkaProxyApplication.create; 29 | import static com.dajudge.proxybase.certs.Filesystem.DEFAULT_FILESYSTEM; 30 | 31 | @ApplicationScoped 32 | public class Startup { 33 | private KafkaProxyApplication application; 34 | 35 | void onStart(@Observes StartupEvent ev) { 36 | application = create(new RealEnvironment(), System::currentTimeMillis, DEFAULT_FILESYSTEM); 37 | } 38 | 39 | void onStop(@Observes ShutdownEvent ev) { 40 | application.close(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/util/Util.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.util; 19 | 20 | import org.jetbrains.annotations.NotNull; 21 | 22 | import java.util.UUID; 23 | 24 | public final class Util { 25 | private Util() { 26 | } 27 | 28 | @NotNull 29 | public static String indent(final int spaces, final String string) { 30 | return indent(new String(new char[spaces]).replace("\0", " "), string); 31 | } 32 | 33 | @NotNull 34 | public static String indent(final String prefix, final String string) { 35 | return string.replaceAll("(?m)^", prefix); 36 | } 37 | 38 | @NotNull 39 | public static String randomIdentifier() { 40 | return "i" + UUID.randomUUID().toString().replace("-", ""); 41 | } 42 | 43 | public static String safeToString(final char[] c) { 44 | return c == null ? null : new String(c); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /itests/src/main/clients/java/src/main/java/com/dajudge/kafkaproxy/tests/ManyConnectsTest.java: -------------------------------------------------------------------------------- 1 | package com.dajudge.kafkaproxy.tests; 2 | 3 | import org.apache.kafka.clients.admin.AdminClient; 4 | import org.apache.kafka.clients.admin.AdminClientConfig; 5 | 6 | import java.util.Properties; 7 | import java.util.concurrent.ExecutionException; 8 | 9 | import static java.lang.Integer.parseInt; 10 | 11 | public class ManyConnectsTest { 12 | 13 | private String bootstrapServers; 14 | 15 | public ManyConnectsTest(final String bootstrapServers) { 16 | this.bootstrapServers = bootstrapServers; 17 | } 18 | 19 | public boolean run() throws ExecutionException, InterruptedException { 20 | final int connectionAttempts = getConnectionAttempts(); 21 | System.out.println("Connection attempts to run: " + connectionAttempts); 22 | for (int i = 0; i < connectionAttempts; i++) { 23 | if ((i % 100) == 0) { 24 | System.out.println("Connection attempts #" + (i + 1)); 25 | } 26 | final Properties props = new Properties(); 27 | props.setProperty(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); 28 | try (final AdminClient client = AdminClient.create(props)) { 29 | client.listTopics().names().get(); 30 | } 31 | } 32 | return true; 33 | } 34 | 35 | private int getConnectionAttempts() { 36 | try { 37 | return parseInt(System.getenv("CONNECTION_ATTEMPTS")); 38 | } catch (final Exception e) { 39 | return 100000; 40 | } 41 | } 42 | 43 | 44 | } 45 | -------------------------------------------------------------------------------- /itests/src/test/java/com/dajudge/kafkaproxy/itest/util/ToConsoleConsumer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package com.dajudge.kafkaproxy.itest.util; 18 | 19 | import org.testcontainers.containers.output.OutputFrame; 20 | 21 | import java.io.IOException; 22 | import java.io.PrintStream; 23 | import java.util.function.Consumer; 24 | 25 | public class ToConsoleConsumer implements Consumer { 26 | private final PrintStream stream; 27 | 28 | public ToConsoleConsumer(final PrintStream stream) { 29 | this.stream = stream; 30 | } 31 | 32 | @Override 33 | public void accept(final OutputFrame outputFrame) { 34 | try { 35 | 36 | final byte[] bytes = outputFrame.getBytes(); 37 | if (bytes != null) { 38 | stream.write(bytes); 39 | stream.flush(); 40 | } 41 | } catch (final IOException e) { 42 | throw new IllegalStateException(e); 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /core/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | 20 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} [%X{CHANNEL_TYPE}/%X{CHANNEL_ID}] - %msg%n 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /itests/src/main/clients/java/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | 20 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} [%X{CHANNEL_TYPE}/%X{CHANNEL_ID}] - %msg%n 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/protocol/rewrite/CompositeRewriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.protocol.rewrite; 19 | 20 | import com.dajudge.kafkaproxy.protocol.KafkaMessage; 21 | import org.apache.kafka.common.requests.RequestHeader; 22 | 23 | import java.util.List; 24 | 25 | import static java.util.Collections.unmodifiableList; 26 | 27 | public class CompositeRewriter implements ResponseRewriter { 28 | private final List rewriters; 29 | 30 | public CompositeRewriter(final List rewriters) { 31 | this.rewriters = unmodifiableList(rewriters); 32 | } 33 | 34 | @Override 35 | public KafkaMessage rewrite( 36 | final RequestHeader requestHeader, 37 | KafkaMessage currentMessage 38 | ) { 39 | for (final ResponseRewriter rewriter : rewriters) { 40 | currentMessage = rewriter.rewrite(requestHeader, currentMessage); 41 | } 42 | return currentMessage; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/protocol/RewritingKafkaMessageDuplexHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.protocol; 19 | 20 | import io.netty.channel.ChannelDuplexHandler; 21 | import io.netty.channel.ChannelHandlerContext; 22 | import io.netty.channel.ChannelPromise; 23 | 24 | public class RewritingKafkaMessageDuplexHandler extends ChannelDuplexHandler { 25 | private final KafkaRequestStore kafkaRequestStore; 26 | 27 | public RewritingKafkaMessageDuplexHandler(final KafkaRequestStore kafkaRequestStore) { 28 | this.kafkaRequestStore = kafkaRequestStore; 29 | } 30 | 31 | @Override 32 | public void write(final ChannelHandlerContext ctx, final Object msg, final ChannelPromise promise) { 33 | kafkaRequestStore.add((KafkaMessage) msg); 34 | ctx.write(msg, promise); 35 | } 36 | 37 | @Override 38 | public void channelRead(final ChannelHandlerContext ctx, final Object msg) { 39 | ctx.fireChannelRead(kafkaRequestStore.process((KafkaMessage) msg)); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /itests/src/main/clients/dotnet/SinkMaster.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace testrunner 5 | { 6 | public delegate Sink SinkFactory(); 7 | 8 | internal class SinkWrapper : Sink 9 | { 10 | private readonly Sink _wrapped; 11 | private readonly ISet _messages; 12 | 13 | public SinkWrapper(Sink wrapped, ISet messages) 14 | { 15 | _wrapped = wrapped; 16 | _messages = messages; 17 | } 18 | 19 | public void Dispose() 20 | { 21 | _wrapped.Dispose(); 22 | } 23 | 24 | public void Publish(string message) 25 | { 26 | lock (_messages) 27 | { 28 | _messages.Add(message); 29 | } 30 | 31 | _wrapped.Publish(message); 32 | } 33 | } 34 | 35 | public class SinkMaster : IDisposable 36 | { 37 | private readonly IList _sinks = new List(); 38 | private readonly Random _rand = new Random(); 39 | 40 | public SinkMaster( 41 | in ISet messages, 42 | in int sinkCount, 43 | in SinkFactory sinkFactory 44 | ) 45 | { 46 | for (var i = 0; i < sinkCount; i++) 47 | { 48 | _sinks.Add(new SinkWrapper(sinkFactory(), messages)); 49 | } 50 | } 51 | 52 | public Sink RandomSink() 53 | { 54 | return _sinks[_rand.Next(_sinks.Count)]; 55 | } 56 | 57 | public void Dispose() 58 | { 59 | foreach (var sink in _sinks) 60 | { 61 | sink.Dispose(); 62 | } 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /itests/build.gradle: -------------------------------------------------------------------------------- 1 | import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage 2 | 3 | plugins { 4 | id "java" 5 | id 'com.bmuschko.docker-remote-api' version '6.7.0' 6 | } 7 | 8 | dependencies { 9 | testImplementation libJunit4 10 | testImplementation libTestContainers 11 | testImplementation project(":core") 12 | testImplementation project(":testutil") 13 | testImplementation libProxyBaseTestca 14 | testRuntimeOnly libLogback 15 | } 16 | 17 | task composeKafkaProxyDockerContext(type: Copy) { 18 | from project(":app").tasks.findByName("quarkusBuild").outputs.files 19 | from rootProject.file("Dockerfile") 20 | into file("build/kafkaproxy") 21 | } 22 | 23 | task buildKafkaProxy(type: DockerBuildImage) { 24 | buildArgs = [ 25 | "APP_DIR": "." 26 | ] 27 | dependsOn composeKafkaProxyDockerContext 28 | inputDir = file("build/kafkaproxy") 29 | images.add("localhost/kafkaproxy/kafkaproxy:latest") 30 | } 31 | test.dependsOn buildKafkaProxy 32 | 33 | def clients = new File(project.projectDir, "src/main/clients").listFiles().collect { it.name } 34 | clients.forEach { client -> 35 | def buildTask = task "buildClient-${client}"(type: DockerBuildImage, group: "itest-clients") { 36 | inputDir = file("src/main/clients/${client}") 37 | images.add("localhost/kafkaproxy/itest-${client}:latest") 38 | buildArgs = [ 39 | KAFKA_CLIENT_VERSION_JAVA: System.getenv("KAFKA_CLIENT_VERSION_JAVA") ?: "2.6.0", 40 | ] 41 | } 42 | test.dependsOn buildTask 43 | } 44 | 45 | 46 | test { 47 | environment "CONFLUENT_PLATFORM_VERSION", System.getenv("CONFLUENT_PLATFORM_VERSION") ?: "5.5.1" 48 | testLogging { 49 | showStandardStreams = true 50 | } 51 | } -------------------------------------------------------------------------------- /example/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | zookeeper: 4 | image: "confluentinc/cp-zookeeper:7.0.3" 5 | restart: always 6 | ports: 7 | - "2181:2181" 8 | environment: 9 | ZOOKEEPER_CLIENT_PORT: 2181 10 | 11 | kafka1: 12 | image: "confluentinc/cp-kafka:7.0.3" 13 | depends_on: 14 | - zookeeper 15 | restart: always 16 | volumes: 17 | - ./ssl:/etc/kafka/secrets 18 | environment: 19 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:9092,SSL://kafka1:9093 20 | KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,SSL://0.0.0.0:9093 21 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 22 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 23 | KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT 24 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 25 | KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 1 26 | KAFKA_BROKER_ID: 1 27 | KAFKA_SSL_KEYSTORE_FILENAME: server.p12 28 | KAFKA_SSL_KEYSTORE_PASSWORD: test 29 | KAFKA_SSL_KEY_CREDENTIALS: password 30 | KAFKA_SSL_KEYSTORE_CREDENTIALS: password 31 | 32 | kafkaproxy: 33 | image: "dajudge/kafkaproxy:0.0.18" 34 | ports: 35 | - 4000:4000 36 | depends_on: 37 | - kafka1 38 | restart: always 39 | volumes: 40 | - ./ssl:/ssl 41 | environment: 42 | KAFKAPROXY_HOSTNAME: localhost 43 | KAFKAPROXY_BASE_PORT: 4000 44 | KAFKAPROXY_BOOTSTRAP_SERVERS: kafka1:9093 45 | KAFKAPROXY_LOG_LEVEL: DEBUG 46 | KAFKAPROXY_KAFKA_SSL_ENABLED: "true" 47 | KAFKAPROXY_KAFKA_SSL_TRUSTSTORE_LOCATION: /ssl/server.p12 48 | KAFKAPROXY_KAFKA_SSL_TRUSTSTORE_TYPE: pkcs12 49 | KAFKAPROXY_KAFKA_SSL_TRUSTSTORE_PASSWORD_LOCATION: /ssl/password 50 | KAFKAPROXY_KAFKA_SSL_VERIFY_HOSTNAME: "false" 51 | -------------------------------------------------------------------------------- /testutil/src/main/java/com/dajudge/kafkaproxy/roundtrip/util/PortFinder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.util; 19 | 20 | import java.io.IOException; 21 | import java.net.ServerSocket; 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | 25 | public class PortFinder implements AutoCloseable { 26 | private final List sockets = new ArrayList<>(); 27 | 28 | @SuppressWarnings("PMD.CloseResource") 29 | public int nextPort() { 30 | try { 31 | final ServerSocket s = new ServerSocket(0); 32 | sockets.add(s); 33 | return s.getLocalPort(); 34 | } catch (final IOException e) { 35 | throw new RuntimeException("Failed to open server socket", e); 36 | } 37 | } 38 | 39 | @Override 40 | public void close() { 41 | sockets.forEach(socket -> { 42 | try { 43 | socket.close(); 44 | } catch (final IOException e) { 45 | throw new RuntimeException("Failed to close server socket", e); 46 | } 47 | }); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/cluster/StarterThread.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.cluster; 19 | 20 | import org.testcontainers.containers.GenericContainer; 21 | 22 | final class StarterThread> extends Thread { 23 | private final T container; 24 | private RuntimeException error; 25 | 26 | StarterThread(final T container) { 27 | this.container = container; 28 | } 29 | 30 | @Override 31 | public void run() { 32 | try { 33 | container.start(); 34 | } catch (final RuntimeException e) { 35 | error = e; 36 | } 37 | } 38 | 39 | void waitForStartup() { 40 | try { 41 | join(300000); 42 | if (isAlive()) { 43 | throw new RuntimeException("Starter thread did not complete in time"); 44 | } 45 | } catch (final InterruptedException e) { 46 | throw new RuntimeException("Interrupted", e); 47 | } 48 | } 49 | 50 | T getContainer() { 51 | if (error != null) { 52 | throw error; 53 | } 54 | return container; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/ssl/KeyStoreData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.ssl; 19 | 20 | public class KeyStoreData { 21 | private final byte[] keyStore; 22 | private final char[] keyStorePassword; 23 | private final char[] keyPassword; 24 | private final String type; 25 | 26 | public KeyStoreData( 27 | final byte[] keyStore, 28 | final char[] keyStorePassword, 29 | final char[] keyPassword, 30 | final String type 31 | ) { 32 | this.keyStore = keyStore.clone(); 33 | this.keyStorePassword = keyStorePassword == null ? null : keyStorePassword.clone(); 34 | this.keyPassword = keyPassword == null ? null : keyPassword.clone(); 35 | this.type = type; 36 | } 37 | 38 | public byte[] getBytes() { 39 | return keyStore.clone(); 40 | } 41 | 42 | public String getType() { 43 | return type; 44 | } 45 | 46 | public char[] getKeyStorePassword() { 47 | return keyStorePassword == null ? null : keyStorePassword.clone(); 48 | } 49 | 50 | public char[] getKeyPassword() { 51 | return keyPassword == null ? null : keyPassword.clone(); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /itests/src/main/clients/dotnet/PollThread.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Threading; 4 | 5 | namespace testrunner 6 | { 7 | public delegate void OnMessageCallback(); 8 | 9 | public class PollThread : IDisposable 10 | { 11 | private readonly Poll _poll; 12 | private readonly ISet _messages; 13 | private readonly OnMessageCallback _onMessageCallback; 14 | private readonly Thread _thread; 15 | private bool _exit; 16 | 17 | public PollThread( 18 | in Poll poll, 19 | in ISet messages, 20 | in OnMessageCallback onMessageCallback 21 | ) 22 | { 23 | _poll = poll; 24 | _messages = messages; 25 | _onMessageCallback = onMessageCallback; 26 | _thread = new Thread(Run); 27 | } 28 | 29 | public void Start() 30 | { 31 | _thread.Start(); 32 | } 33 | 34 | private void Run(object p) 35 | { 36 | try 37 | { 38 | while (!_exit) 39 | { 40 | try 41 | { 42 | var result = _poll.Poll(); 43 | if (result == null) continue; 44 | lock (_messages) 45 | { 46 | _messages.Remove(result); 47 | } 48 | 49 | _onMessageCallback(); 50 | } 51 | catch (Exception e) 52 | { 53 | Console.Error.WriteLine(e); 54 | } 55 | } 56 | } 57 | finally 58 | { 59 | _poll.Dispose(); 60 | } 61 | } 62 | 63 | public void Dispose() 64 | { 65 | _exit = true; 66 | } 67 | } 68 | } -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/cluster/KafkaCluster.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.cluster; 19 | 20 | import org.testcontainers.containers.GenericContainer; 21 | import org.testcontainers.lifecycle.Startable; 22 | 23 | import java.util.ArrayList; 24 | import java.util.HashSet; 25 | import java.util.List; 26 | import java.util.Set; 27 | 28 | import static java.util.Arrays.asList; 29 | import static java.util.Collections.reverse; 30 | 31 | public class KafkaCluster implements AutoCloseable { 32 | private final List> containers; 33 | private final String bootstrapServers; 34 | 35 | KafkaCluster( 36 | final List> containers, 37 | final String bootstrapServers 38 | ) { 39 | this.containers = new ArrayList<>(containers); 40 | reverse(this.containers); 41 | this.bootstrapServers = bootstrapServers; 42 | } 43 | 44 | @Override 45 | public void close() { 46 | containers.forEach(Startable::close); 47 | } 48 | 49 | public String getBootstrapServers() { 50 | return bootstrapServers; 51 | } 52 | 53 | public Set getBootstrapServerList() { 54 | return new HashSet<>(asList(bootstrapServers.split(","))); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/BrokerMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy; 19 | 20 | import com.dajudge.kafkaproxy.config.BrokerConfigSource; 21 | import com.dajudge.proxybase.config.Endpoint; 22 | 23 | import java.util.HashMap; 24 | import java.util.List; 25 | import java.util.Map; 26 | 27 | public class BrokerMapper { 28 | private final Map allMappings = new HashMap<>(); 29 | private final String proxyHostname; 30 | private final List bootstrapBrokers; 31 | private int nextBrokerPort; 32 | 33 | public BrokerMapper(final BrokerConfigSource.BrokerConfig brokerConfig) { 34 | nextBrokerPort = brokerConfig.getProxyBasePort(); 35 | proxyHostname = brokerConfig.getProxyHostname(); 36 | bootstrapBrokers = brokerConfig.getBootstrapBrokers(); 37 | } 38 | 39 | public synchronized BrokerMapping getBrokerMapping(final Endpoint brokerEndpoint) { 40 | return allMappings.computeIfAbsent(keyOf(brokerEndpoint), key -> new BrokerMapping( 41 | brokerEndpoint, 42 | new Endpoint(proxyHostname, nextBrokerPort++) 43 | )); 44 | } 45 | 46 | public List getBootstrapBrokers() { 47 | return bootstrapBrokers; 48 | } 49 | 50 | private String keyOf(final Endpoint host) { 51 | return host.getHost() + host.getPort(); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/config/ApplicationConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.config; 19 | 20 | import java.util.HashMap; 21 | import java.util.Map; 22 | import java.util.Optional; 23 | 24 | import static java.util.Collections.unmodifiableMap; 25 | import static java.util.ServiceLoader.load; 26 | import static java.util.stream.StreamSupport.stream; 27 | 28 | public class ApplicationConfig { 29 | private final Environment environment; 30 | private final Map, ConfigSource> sources = unmodifiableMap(new HashMap, ConfigSource>() {{ 31 | stream(load(ConfigSource.class).spliterator(), false) 32 | .forEach(source -> put(source.getConfigClass(), source)); 33 | }}); 34 | 35 | public ApplicationConfig(final Environment environment) { 36 | this.environment = environment; 37 | } 38 | 39 | @SuppressWarnings("unchecked") 40 | public Optional optional(final Class configClass) { 41 | return (Optional) sources.get(configClass).parse(environment); 42 | } 43 | 44 | @SuppressWarnings("unchecked") 45 | public T require(final Class configClass) { 46 | return ((Optional) sources.get(configClass).parse(environment)) 47 | .orElseThrow(() -> new IllegalArgumentException( 48 | "Could not find config of type " + configClass.getName() 49 | )); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/comm/PlaintextClientSecurity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.comm; 19 | 20 | public class PlaintextClientSecurity implements ClientSecurity { 21 | @Override 22 | public String getProtocol() { 23 | return "PLAINTEXT"; 24 | } 25 | 26 | @Override 27 | public String getTrustStoreLocation() { 28 | return null; 29 | } 30 | 31 | @Override 32 | public char[] getTrustStorePassword() { 33 | return null; 34 | } 35 | 36 | @Override 37 | public String getTrustStoreType() { 38 | return null; 39 | } 40 | 41 | @Override 42 | public ClientSslConfig newClient(final String dn) { 43 | return new ClientSslConfig() { 44 | @Override 45 | public String getKeyStoreLocation() { 46 | return null; 47 | } 48 | 49 | @Override 50 | public char[] getKeyStorePassword() { 51 | return null; 52 | } 53 | 54 | @Override 55 | public char[] getKeyPassword() { 56 | return null; 57 | } 58 | 59 | @Override 60 | public String getKeyStoreType() { 61 | return null; 62 | } 63 | 64 | @Override 65 | public String getProxyCertStrategy() { 66 | return "NONE"; 67 | } 68 | }; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/cluster/CommunicationSetupBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.cluster; 19 | 20 | import com.dajudge.kafkaproxy.roundtrip.comm.CommunicationSetup; 21 | import com.dajudge.kafkaproxy.roundtrip.comm.PlaintextCommunicationSetup; 22 | import com.dajudge.kafkaproxy.roundtrip.comm.SslCommunicationSetup; 23 | import org.jetbrains.annotations.NotNull; 24 | 25 | public class CommunicationSetupBuilder { 26 | private CommunicationSetup comm; 27 | 28 | public CommunicationSetupBuilder() { 29 | } 30 | 31 | public CommunicationSetupBuilder withPlaintext() { 32 | comm = new PlaintextCommunicationSetup(); 33 | return this; 34 | } 35 | 36 | public CommunicationSetupBuilder withSsl(final String keyStoreType) { 37 | return withSsl(false, keyStoreType); 38 | } 39 | 40 | public CommunicationSetupBuilder withMutualTls(final String keyStoreType) { 41 | return withSsl(true, keyStoreType); 42 | } 43 | 44 | @NotNull 45 | private CommunicationSetupBuilder withSsl(final boolean requireClientAuth, final String keyStoreType) { 46 | comm = new SslCommunicationSetup( 47 | "CN=clientCA", 48 | "CN=brokerCA", 49 | requireClientAuth, 50 | keyStoreType 51 | ); 52 | return this; 53 | } 54 | 55 | public CommunicationSetup build() { 56 | return comm; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/config/RealEnvironment.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.config; 19 | 20 | import java.util.Optional; 21 | 22 | import static java.lang.System.getenv; 23 | 24 | public class RealEnvironment implements Environment { 25 | @Override 26 | public String requiredString(final String variable, final String defaultValue) { 27 | final String value = getenv(variable); 28 | return value == null ? defaultValue : value; 29 | } 30 | 31 | @Override 32 | public String requiredString(final String variable) { 33 | return Optional.ofNullable(requiredString(variable, null)) 34 | .orElseThrow(() -> new IllegalArgumentException("No such environment variable: " + variable)); 35 | } 36 | 37 | @Override 38 | public Optional optionalString(final String variable) { 39 | return Optional.ofNullable(requiredString(variable, null)); 40 | } 41 | 42 | @Override 43 | public Optional optionalInt(final String variable) { 44 | return optionalString(variable).map(Integer::parseInt); 45 | } 46 | 47 | @Override 48 | public boolean requiredBoolean(final String variable, final boolean defaultValue) { 49 | return optionalString(variable).map(Boolean::parseBoolean).orElse(defaultValue); 50 | } 51 | 52 | @Override 53 | public int requiredInt(final String variable) { 54 | return Integer.parseInt(requiredString(variable)); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/cluster/container/ZookeeperContainer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.cluster.container; 19 | 20 | import org.testcontainers.containers.GenericContainer; 21 | import org.testcontainers.containers.Network; 22 | import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; 23 | 24 | import static com.dajudge.kafkaproxy.roundtrip.cluster.container.Version.CP_VERSION; 25 | import static java.lang.String.valueOf; 26 | import static org.testcontainers.images.PullPolicy.alwaysPull; 27 | 28 | public class ZookeeperContainer extends GenericContainer { 29 | private static final String NETWORK_ALIAS = "zookeeper"; 30 | private static final int ZOOKEEPER_PORT = 2181; 31 | 32 | public ZookeeperContainer(final Network network) { 33 | super("confluentinc/cp-zookeeper:" + CP_VERSION); 34 | this.withNetworkAliases(NETWORK_ALIAS) 35 | .withImagePullPolicy(alwaysPull()) 36 | .withNetwork(network) 37 | .withEnv("ZOOKEEPER_CLIENT_PORT", valueOf(ZOOKEEPER_PORT)) 38 | .waitingFor(new LogMessageWaitStrategy().withRegEx(".*binding to port.*")); 39 | } 40 | 41 | public String getEndpoint() { 42 | return NETWORK_ALIAS + ":" + ZOOKEEPER_PORT; 43 | } 44 | 45 | @Override 46 | public boolean equals(final Object o) { 47 | return super.equals(o); 48 | } 49 | 50 | @Override 51 | public int hashCode() { 52 | return super.hashCode(); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/comm/PlaintextServerSecurity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.comm; 19 | 20 | import org.testcontainers.images.builder.Transferable; 21 | 22 | import java.util.function.BiConsumer; 23 | 24 | public class PlaintextServerSecurity implements ServerSecurity { 25 | @Override 26 | public String getClientProtocol() { 27 | return "PLAINTEXT"; 28 | } 29 | 30 | @Override 31 | public String getTrustStoreLocation() { 32 | return null; 33 | } 34 | 35 | @Override 36 | public char[] getTrustStorePassword() { 37 | return null; 38 | } 39 | 40 | @Override 41 | public String getTrustStoreType() { 42 | return null; 43 | } 44 | 45 | @Override 46 | public String getKeyStoreLocation() { 47 | return null; 48 | } 49 | 50 | @Override 51 | public char[] getKeyStorePassword() { 52 | return null; 53 | } 54 | 55 | @Override 56 | public String getKeyStoreType() { 57 | return null; 58 | } 59 | 60 | @Override 61 | public char[] getKeyPassword() { 62 | return null; 63 | } 64 | 65 | @Override 66 | public String getClientAuth() { 67 | return "none"; 68 | } 69 | 70 | @Override 71 | public byte[] getKeyStore() { 72 | return null; 73 | } 74 | 75 | @Override 76 | public byte[] getTrustStore() { 77 | return null; 78 | } 79 | 80 | @Override 81 | public void uploadKeyStores(final BiConsumer uploader) { 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/config/UpstreamSslConfigSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.config; 19 | 20 | import com.dajudge.proxybase.config.UpstreamSslConfig; 21 | 22 | import java.util.Optional; 23 | 24 | import static com.dajudge.kafkaproxy.config.KeyStoreConfigHelper.requiredKeyStoreConfig; 25 | 26 | 27 | public class UpstreamSslConfigSource implements ConfigSource { 28 | private static final String PREFIX_CLIENT_SSL = PREFIX + "CLIENT_SSL_"; 29 | private static final String PROP_CLIENT_SSL_ENABLED = PREFIX_CLIENT_SSL + "ENABLED"; 30 | private static final String PROP_CLIENT_SSL_AUTH_REQUIRED = PREFIX_CLIENT_SSL + "AUTH_REQUIRED"; 31 | 32 | private static final boolean DEFAULT_CLIENT_SSL_ENABLED = false; 33 | private static final boolean DEFAULT_CLIENT_AUTH_REQUIRED = false; 34 | 35 | @Override 36 | public Class getConfigClass() { 37 | return UpstreamSslConfig.class; 38 | } 39 | 40 | @Override 41 | public Optional parse(final Environment environment) { 42 | if (!environment.requiredBoolean(PROP_CLIENT_SSL_ENABLED, DEFAULT_CLIENT_SSL_ENABLED)) { 43 | return Optional.empty(); 44 | } 45 | return Optional.of(new UpstreamSslConfig( 46 | KeyStoreConfigHelper.loadTrustStoreConfig(environment, PREFIX_CLIENT_SSL), 47 | requiredKeyStoreConfig(environment, PREFIX_CLIENT_SSL), 48 | environment.requiredBoolean(PROP_CLIENT_SSL_AUTH_REQUIRED, DEFAULT_CLIENT_AUTH_REQUIRED) 49 | )); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/config/DownstreamSslConfigSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.config; 19 | 20 | 21 | import com.dajudge.proxybase.config.DownstreamSslConfig; 22 | 23 | import java.util.Optional; 24 | 25 | import static com.dajudge.kafkaproxy.config.KeyStoreConfigHelper.*; 26 | 27 | public class DownstreamSslConfigSource implements ConfigSource { 28 | private static final String KAFKA_SSL_PREFIX = PREFIX + "KAFKA_SSL_"; 29 | private static final String ENV_KAFKA_SSL_ENABLED = KAFKA_SSL_PREFIX + "ENABLED"; 30 | private static final String ENV_KAFKA_SSL_VERIFY_HOSTNAME = KAFKA_SSL_PREFIX + "VERIFY_HOSTNAME"; 31 | private static final boolean DEFAULT_KAFKA_SSL_ENABLED = false; 32 | private static final boolean DEFAULT_KAFKA_SSL_VERIFY_HOSTNAME = true; 33 | 34 | @Override 35 | public Class getConfigClass() { 36 | return DownstreamSslConfig.class; 37 | } 38 | 39 | @Override 40 | public Optional parse(final Environment environment) { 41 | final boolean enabled = environment.requiredBoolean(ENV_KAFKA_SSL_ENABLED, DEFAULT_KAFKA_SSL_ENABLED); 42 | if (!enabled) { 43 | return Optional.empty(); 44 | } 45 | final DownstreamSslConfig downstreamConfig = new DownstreamSslConfig( 46 | loadTrustStoreConfig(environment, KAFKA_SSL_PREFIX), 47 | optionalKeyStoreConfig(environment, KAFKA_SSL_PREFIX), 48 | environment.requiredBoolean(ENV_KAFKA_SSL_VERIFY_HOSTNAME, DEFAULT_KAFKA_SSL_VERIFY_HOSTNAME) 49 | ); 50 | return Optional.of(downstreamConfig); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/cluster/TestSetup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.cluster; 19 | 20 | import com.dajudge.kafkaproxy.KafkaProxyApplication; 21 | import com.dajudge.kafkaproxy.roundtrip.client.ClientFactory; 22 | 23 | public class TestSetup implements AutoCloseable { 24 | private final KafkaCluster kafka; 25 | private final KafkaProxyApplication proxy; 26 | private final ClientFactory proxiedClientFactory; 27 | private final ClientFactory directClientFactory; 28 | private final int proxyBootstrapPort; 29 | 30 | TestSetup( 31 | final KafkaCluster kafka, 32 | final KafkaProxyApplication proxy, 33 | final ClientFactory proxiedClientFactory, 34 | final ClientFactory directClientFactory, 35 | final int proxyBootstrapPort 36 | ) { 37 | this.kafka = kafka; 38 | this.proxy = proxy; 39 | this.proxiedClientFactory = proxiedClientFactory; 40 | this.directClientFactory = directClientFactory; 41 | this.proxyBootstrapPort = proxyBootstrapPort; 42 | } 43 | 44 | public ClientFactory getProxiedClientFactory() { 45 | return proxiedClientFactory; 46 | } 47 | 48 | public ClientFactory getDirectClientFactory() { 49 | return directClientFactory; 50 | } 51 | 52 | public KafkaCluster getKafka() { 53 | return kafka; 54 | } 55 | 56 | public KafkaProxyApplication getProxy() { 57 | return proxy; 58 | } 59 | 60 | @Override 61 | public void close() { 62 | kafka.close(); 63 | proxy.close(); 64 | } 65 | 66 | public int getProxyBootstrapPort() { 67 | return proxyBootstrapPort; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/cluster/KafkaWaitStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.cluster; 19 | 20 | import com.dajudge.kafkaproxy.roundtrip.client.ClientFactory; 21 | import com.dajudge.kafkaproxy.roundtrip.comm.ClientSecurity; 22 | import org.apache.kafka.clients.admin.AdminClient; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | import org.testcontainers.containers.wait.strategy.AbstractWaitStrategy; 26 | 27 | import java.util.concurrent.ExecutionException; 28 | import java.util.concurrent.TimeoutException; 29 | 30 | import static java.util.concurrent.TimeUnit.SECONDS; 31 | 32 | public class KafkaWaitStrategy extends AbstractWaitStrategy { 33 | private static final Logger LOG = LoggerFactory.getLogger(KafkaWaitStrategy.class); 34 | private final int internalClientPort; 35 | private final ClientSecurity clientSecurity; 36 | 37 | public KafkaWaitStrategy( 38 | final int internalClientPort, 39 | final ClientSecurity clientSecurity 40 | ) { 41 | this.internalClientPort = internalClientPort; 42 | this.clientSecurity = clientSecurity; 43 | } 44 | 45 | @Override 46 | protected void waitUntilReady() { 47 | final String bootstrapServers = "localhost:" + waitStrategyTarget.getMappedPort(internalClientPort); 48 | try (final AdminClient admin = new ClientFactory(bootstrapServers, clientSecurity).admin("CN=admin")) { 49 | admin.describeCluster().nodes().get(startupTimeout.getSeconds(), SECONDS).size(); 50 | } catch (final InterruptedException | ExecutionException | TimeoutException e) { 51 | throw new RuntimeException("Failed to wait for kafka broker", e); 52 | } 53 | LOG.info("{} is available.", waitStrategyTarget.getContainerInfo().getName()); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/comm/SslCommunicationSetup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.comm; 19 | 20 | import com.dajudge.kafkaproxy.roundtrip.ssl.CertAuthority; 21 | 22 | import java.util.Optional; 23 | 24 | import static java.util.Optional.empty; 25 | 26 | public class SslCommunicationSetup implements CommunicationSetup { 27 | private final CertAuthority brokerAuthority; 28 | private final CertAuthority clientAuthority; 29 | private final boolean requireClientAuth; 30 | private final String keyStoreType; 31 | 32 | public SslCommunicationSetup( 33 | final String clientCaDn, 34 | final String brokerCaDn, 35 | final boolean requireClientAuth, 36 | final String keyStoreType 37 | ) { 38 | this.brokerAuthority = new CertAuthority(brokerCaDn); 39 | this.clientAuthority = new CertAuthority(clientCaDn); 40 | this.requireClientAuth = requireClientAuth; 41 | this.keyStoreType = keyStoreType; 42 | } 43 | 44 | @Override 45 | public ServerSecurity getServerSecurity(final String dn) { 46 | return new SslServerSecurity( 47 | brokerAuthority.createSignedKeyPair(dn, keyStoreType), 48 | clientAuthority.getTrustStore(keyStoreType), 49 | requireClientAuth 50 | ); 51 | } 52 | 53 | 54 | @Override 55 | public ClientSecurity getClientSecurity() { 56 | return new SslClientSecurity( 57 | brokerAuthority.getTrustStore(keyStoreType), 58 | requireClientAuth ? Optional.of(dn -> clientAuthority.createSignedKeyPair(dn, keyStoreType)) : empty(), 59 | requireClientAuth ? "KEYSTORE" : "NONE" 60 | ); 61 | } 62 | 63 | @Override 64 | public String toString() { 65 | return "SslCommunicationSetup{" + 66 | "requireClientAuth=" + requireClientAuth + 67 | '}'; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /testutil/src/main/java/com/dajudge/kafkaproxy/roundtrip/util/TestEnvironment.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.util; 19 | 20 | import com.dajudge.kafkaproxy.config.Environment; 21 | 22 | import java.util.HashMap; 23 | import java.util.Map; 24 | import java.util.Optional; 25 | import java.util.function.Consumer; 26 | 27 | import static java.lang.Integer.parseInt; 28 | 29 | public class TestEnvironment implements Environment { 30 | private final Map env = new HashMap<>(); 31 | 32 | public TestEnvironment withEnv(final String var, final String value) { 33 | env.put(var, value); 34 | return this; 35 | } 36 | 37 | @Override 38 | public String requiredString(final String variable, final String defaultValue) { 39 | return optionalString(variable).orElse(defaultValue); 40 | } 41 | 42 | @Override 43 | public String requiredString(final String variable) { 44 | final String value = requiredString(variable, null); 45 | return Optional.ofNullable(value) 46 | .orElseThrow(() -> new IllegalArgumentException("Environment variable not set: " + variable)); 47 | } 48 | 49 | @Override 50 | public Optional optionalString(final String variable) { 51 | return Optional.ofNullable(env.get(variable)); 52 | } 53 | 54 | @Override 55 | public Optional optionalInt(final String variable) { 56 | return optionalString(variable).map(Integer::parseInt); 57 | } 58 | 59 | @Override 60 | public boolean requiredBoolean(final String variable, final boolean defaultValue) { 61 | return optionalString(variable).map(Boolean::parseBoolean).orElse(defaultValue); 62 | } 63 | 64 | @Override 65 | public int requiredInt(final String variable) { 66 | return parseInt(requiredString(variable)); 67 | } 68 | 69 | public void dump(final Consumer dumper) { 70 | env.forEach((k, v) -> dumper.accept(k + "=" + v)); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/protocol/KafkaRequestStore.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.protocol; 19 | 20 | import com.dajudge.kafkaproxy.protocol.rewrite.ResponseRewriter; 21 | import org.apache.kafka.common.requests.RequestHeader; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import java.util.HashMap; 26 | import java.util.Map; 27 | 28 | import static java.lang.String.format; 29 | import static java.util.Collections.synchronizedMap; 30 | 31 | public class KafkaRequestStore { 32 | private static final Logger LOG = LoggerFactory.getLogger(KafkaRequestStore.class); 33 | private final Map requests = synchronizedMap(new HashMap<>()); 34 | private final ResponseRewriter rewriter; 35 | 36 | public KafkaRequestStore(final ResponseRewriter rewriter) { 37 | this.rewriter = rewriter; 38 | } 39 | 40 | public void add(final KafkaMessage request) { 41 | final RequestHeader requestHeader = request.requestHeader(); 42 | if (LOG.isDebugEnabled()) { 43 | LOG.trace("Adding client request: {} (inflight {}) ", requestHeader, requests.keySet()); 44 | } 45 | requests.put(requestHeader.correlationId(), requestHeader); 46 | } 47 | 48 | public KafkaMessage process(final KafkaMessage response) { 49 | // Peek at the correlation ID 50 | final int correlationId = response.correlationId(); 51 | if (LOG.isDebugEnabled()) { 52 | LOG.trace( 53 | "Processing response with correlation ID {} (inflight: {})", 54 | correlationId, 55 | requests.keySet() 56 | ); 57 | } 58 | final RequestHeader requestHeader = requests.remove(correlationId); 59 | if (requestHeader == null) { 60 | throw new RuntimeException(format( 61 | "Failed to correlate response correlation ID %d", 62 | correlationId 63 | )); 64 | } 65 | 66 | return rewriter.rewrite(requestHeader, response); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/protocol/rewrite/MetadataRewriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.protocol.rewrite; 19 | 20 | import com.dajudge.kafkaproxy.BrokerMapping; 21 | import com.dajudge.proxybase.config.Endpoint; 22 | import org.apache.kafka.common.message.MetadataResponseData; 23 | import org.apache.kafka.common.protocol.ApiKeys; 24 | import org.apache.kafka.common.requests.MetadataResponse; 25 | import org.apache.kafka.common.requests.RequestHeader; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | import java.lang.reflect.Field; 30 | import java.util.function.Function; 31 | 32 | public class MetadataRewriter extends BaseReflectingRewriter { 33 | private static final Logger LOG = LoggerFactory.getLogger(MetadataRewriter.class); 34 | private final Function brokerResolver; 35 | 36 | public MetadataRewriter(final Function brokerResolver) { 37 | this.brokerResolver = brokerResolver; 38 | } 39 | 40 | @Override 41 | @SuppressWarnings("PMD.AvoidAccessibilityAlteration") 42 | protected void rewrite(final MetadataResponse response) throws NoSuchFieldException, IllegalAccessException { 43 | final Field field = MetadataResponse.class.getDeclaredField("data"); 44 | field.setAccessible(true); 45 | final MetadataResponseData data = (MetadataResponseData) field.get(response); 46 | data.brokers().forEach(b -> { 47 | final BrokerMapping mapping = brokerResolver.apply(new Endpoint(b.host(), b.port())); 48 | if (mapping == null) { 49 | LOG.error("Unknown broker node seen in {}: {}:{}", ApiKeys.METADATA, b.host(), b.port()); 50 | } else { 51 | LOG.debug( 52 | "Rewriting {}: {}:{} -> {}:{}", 53 | ApiKeys.METADATA, 54 | b.host(), 55 | b.port(), 56 | mapping.getProxy().getHost(), 57 | mapping.getProxy().getPort() 58 | ); 59 | b.setHost(mapping.getProxy().getHost()); 60 | b.setPort(mapping.getProxy().getPort()); 61 | } 62 | }); 63 | } 64 | 65 | @Override 66 | public boolean appliesTo(final RequestHeader requestHeader) { 67 | return requestHeader.apiKey() == ApiKeys.METADATA; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /itests/src/main/clients/java/gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem http://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 33 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 34 | 35 | @rem Find java.exe 36 | if defined JAVA_HOME goto findJavaFromJavaHome 37 | 38 | set JAVA_EXE=java.exe 39 | %JAVA_EXE% -version >NUL 2>&1 40 | if "%ERRORLEVEL%" == "0" goto init 41 | 42 | echo. 43 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 44 | echo. 45 | echo Please set the JAVA_HOME variable in your environment to match the 46 | echo location of your Java installation. 47 | 48 | goto fail 49 | 50 | :findJavaFromJavaHome 51 | set JAVA_HOME=%JAVA_HOME:"=% 52 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 53 | 54 | if exist "%JAVA_EXE%" goto init 55 | 56 | echo. 57 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 58 | echo. 59 | echo Please set the JAVA_HOME variable in your environment to match the 60 | echo location of your Java installation. 61 | 62 | goto fail 63 | 64 | :init 65 | @rem Get command-line arguments, handling Windows variants 66 | 67 | if not "%OS%" == "Windows_NT" goto win9xME_args 68 | 69 | :win9xME_args 70 | @rem Slurp the command line arguments. 71 | set CMD_LINE_ARGS= 72 | set _SKIP=2 73 | 74 | :win9xME_args_slurp 75 | if "x%~1" == "x" goto execute 76 | 77 | set CMD_LINE_ARGS=%* 78 | 79 | :execute 80 | @rem Setup the command line 81 | 82 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 83 | 84 | @rem Execute Gradle 85 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 86 | 87 | :end 88 | @rem End local scope for the variables with windows NT shell 89 | if "%ERRORLEVEL%"=="0" goto mainEnd 90 | 91 | :fail 92 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 93 | rem the _cmd.exe /c_ return code! 94 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 95 | exit /b 1 96 | 97 | :mainEnd 98 | if "%OS%"=="Windows_NT" endlocal 99 | 100 | :omega 101 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/comm/SslServerSecurity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.comm; 19 | 20 | import com.dajudge.kafkaproxy.roundtrip.ssl.KeyStoreData; 21 | import org.testcontainers.images.builder.Transferable; 22 | 23 | import java.util.function.BiConsumer; 24 | 25 | public class SslServerSecurity implements ServerSecurity { 26 | private static final String KEYSTORE_LOCATION = "/tmp/keystore.jks"; 27 | private static final String TRUSTSTORE_LOCATION = "/tmp/truststore.jks"; 28 | 29 | private final KeyStoreData keyStore; 30 | private final KeyStoreData trustStore; 31 | private final boolean requireClientAuth; 32 | 33 | public SslServerSecurity( 34 | final KeyStoreData keyStore, 35 | final KeyStoreData trustStore, 36 | final boolean requireClientAuth 37 | ) { 38 | this.keyStore = keyStore; 39 | this.trustStore = trustStore; 40 | this.requireClientAuth = requireClientAuth; 41 | } 42 | 43 | @Override 44 | public String getClientProtocol() { 45 | return "SSL"; 46 | } 47 | 48 | @Override 49 | public String getTrustStoreLocation() { 50 | return TRUSTSTORE_LOCATION; 51 | } 52 | 53 | @Override 54 | public char[] getTrustStorePassword() { 55 | return trustStore.getKeyStorePassword(); 56 | } 57 | 58 | @Override 59 | public String getTrustStoreType() { 60 | return trustStore.getType(); 61 | } 62 | 63 | @Override 64 | public String getKeyStoreLocation() { 65 | return KEYSTORE_LOCATION; 66 | } 67 | 68 | @Override 69 | public char[] getKeyStorePassword() { 70 | return keyStore.getKeyStorePassword(); 71 | } 72 | 73 | @Override 74 | public String getKeyStoreType() { 75 | return keyStore.getType(); 76 | } 77 | 78 | @Override 79 | public char[] getKeyPassword() { 80 | return keyStore.getKeyPassword(); 81 | } 82 | 83 | @Override 84 | public String getClientAuth() { 85 | return requireClientAuth ? "required" : "none"; 86 | } 87 | 88 | @Override 89 | public byte[] getKeyStore() { 90 | return keyStore.getBytes(); 91 | } 92 | 93 | @Override 94 | public byte[] getTrustStore() { 95 | return trustStore.getBytes(); 96 | } 97 | 98 | @Override 99 | public void uploadKeyStores(final BiConsumer uploader) { 100 | uploader.accept(Transferable.of(keyStore.getBytes()), KEYSTORE_LOCATION); 101 | uploader.accept(Transferable.of(trustStore.getBytes()), TRUSTSTORE_LOCATION); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem http://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 33 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 34 | 35 | @rem Find java.exe 36 | if defined JAVA_HOME goto findJavaFromJavaHome 37 | 38 | set JAVA_EXE=java.exe 39 | %JAVA_EXE% -version >NUL 2>&1 40 | if "%ERRORLEVEL%" == "0" goto init 41 | 42 | echo. 43 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 44 | echo. 45 | echo Please set the JAVA_HOME variable in your environment to match the 46 | echo location of your Java installation. 47 | 48 | goto fail 49 | 50 | :findJavaFromJavaHome 51 | set JAVA_HOME=%JAVA_HOME:"=% 52 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 53 | 54 | if exist "%JAVA_EXE%" goto init 55 | 56 | echo. 57 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 58 | echo. 59 | echo Please set the JAVA_HOME variable in your environment to match the 60 | echo location of your Java installation. 61 | 62 | goto fail 63 | 64 | :init 65 | @rem Get command-line arguments, handling Windows variants 66 | 67 | if not "%OS%" == "Windows_NT" goto win9xME_args 68 | 69 | :win9xME_args 70 | @rem Slurp the command line arguments. 71 | set CMD_LINE_ARGS= 72 | set _SKIP=2 73 | 74 | :win9xME_args_slurp 75 | if "x%~1" == "x" goto execute 76 | 77 | set CMD_LINE_ARGS=%* 78 | 79 | :execute 80 | @rem Setup the command line 81 | 82 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 83 | 84 | @rem Execute Gradle 85 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 86 | 87 | :end 88 | @rem End local scope for the variables with windows NT shell 89 | if "%ERRORLEVEL%"=="0" goto mainEnd 90 | 91 | :fail 92 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 93 | rem the _cmd.exe /c_ return code! 94 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 95 | exit /b 1 96 | 97 | :mainEnd 98 | if "%OS%"=="Windows_NT" endlocal 99 | 100 | :omega 101 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/config/BrokerConfigSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.config; 19 | 20 | 21 | import com.dajudge.proxybase.config.Endpoint; 22 | 23 | import java.util.List; 24 | import java.util.Optional; 25 | import java.util.stream.Stream; 26 | 27 | import static java.lang.Integer.parseUnsignedInt; 28 | import static java.util.stream.Collectors.toList; 29 | 30 | public class BrokerConfigSource implements ConfigSource { 31 | 32 | @Override 33 | public Class getConfigClass() { 34 | return BrokerConfig.class; 35 | } 36 | 37 | @Override 38 | public Optional parse(final Environment environment) { 39 | return Optional.of(new BrokerConfig( 40 | getBootstrapBrokers(environment), 41 | environment.requiredString("KAFKAPROXY_HOSTNAME"), 42 | environment.requiredInt("KAFKAPROXY_BASE_PORT"), 43 | environment.requiredString("KAFKAPROXY_BIND_ADDRESS", "0.0.0.0") 44 | )); 45 | } 46 | 47 | private List getBootstrapBrokers(final Environment environment) { 48 | final String bootstrapServers = environment.requiredString("KAFKAPROXY_BOOTSTRAP_SERVERS"); 49 | return Stream.of(bootstrapServers.split(",")) 50 | .map(bootstrapServer -> { 51 | final String[] bootstrapServerParts = bootstrapServer.split(":"); 52 | return new Endpoint(bootstrapServerParts[0], parseUnsignedInt(bootstrapServerParts[1])); 53 | }) 54 | .collect(toList()); 55 | 56 | } 57 | 58 | public static class BrokerConfig { 59 | private final List bootstrapBrokers; 60 | private final String proxyHostname; 61 | private final int proxyBasePort; 62 | private final String bindAddress; 63 | 64 | public BrokerConfig( 65 | final List bootstrapBrokers, 66 | final String proxyHostname, 67 | final int proxyBasePort, 68 | final String bindAddress 69 | ) { 70 | this.bootstrapBrokers = bootstrapBrokers; 71 | this.proxyHostname = proxyHostname; 72 | this.proxyBasePort = proxyBasePort; 73 | this.bindAddress = bindAddress; 74 | } 75 | 76 | public List getBootstrapBrokers() { 77 | return bootstrapBrokers; 78 | } 79 | 80 | public String getProxyHostname() { 81 | return proxyHostname; 82 | } 83 | 84 | public int getProxyBasePort() { 85 | return proxyBasePort; 86 | } 87 | 88 | public String getBindAddress() { 89 | return bindAddress; 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/protocol/rewrite/BaseReflectingRewriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.protocol.rewrite; 19 | 20 | import com.dajudge.kafkaproxy.protocol.KafkaMessage; 21 | import org.apache.kafka.common.requests.AbstractResponse; 22 | import org.apache.kafka.common.requests.RequestHeader; 23 | import org.apache.kafka.common.requests.ResponseHeader; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | import static io.netty.buffer.Unpooled.wrappedBuffer; 28 | import static org.apache.kafka.common.requests.RequestUtils.serialize; 29 | 30 | public abstract class BaseReflectingRewriter implements ResponseRewriter { 31 | private static final Logger LOG = LoggerFactory.getLogger(BaseReflectingRewriter.class); 32 | 33 | public final KafkaMessage rewrite( 34 | final RequestHeader requestHeader, 35 | final KafkaMessage message 36 | ) { 37 | if (!appliesTo(requestHeader)) { 38 | LOG.trace("Not rewriting {} with rewriter {}", requestHeader, getClass().getSimpleName()); 39 | return message; 40 | } 41 | LOG.trace("Rewriting {} with rewriter {}", requestHeader, getClass().getSimpleName()); 42 | try { 43 | return rewriteMessage(requestHeader, message); 44 | } finally { 45 | message.release(); 46 | } 47 | } 48 | 49 | private KafkaMessage rewriteMessage(final RequestHeader requestHeader, final KafkaMessage message) { 50 | final ResponseHeader header = message.responseHeader(requestHeader); 51 | if (LOG.isTraceEnabled()) { 52 | LOG.trace("Original message: {}", message.responseBody(requestHeader)); 53 | } 54 | final T rewrittenMessage = rewriteMessageBody(requestHeader, message); 55 | if (LOG.isTraceEnabled()) { 56 | LOG.trace("Rewritten message: {}", rewrittenMessage); 57 | } 58 | return new KafkaMessage(wrappedBuffer(serialize( 59 | header.data(), 60 | header.headerVersion(), 61 | rewrittenMessage.data(), 62 | requestHeader.apiVersion() 63 | ))); 64 | } 65 | 66 | private T rewriteMessageBody(final RequestHeader requestHeader, final KafkaMessage message) { 67 | final T response = message.responseBody(requestHeader); 68 | try { 69 | rewrite(response); 70 | } catch (final NoSuchFieldException | IllegalAccessException e) { 71 | throw new RuntimeException("Failed to rewrite metadata response", e); 72 | } 73 | return response; 74 | } 75 | 76 | protected abstract boolean appliesTo(RequestHeader requestHeader); 77 | 78 | protected abstract void rewrite(final T response) throws NoSuchFieldException, IllegalAccessException; 79 | } 80 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/ssl/CertAuthority.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.ssl; 19 | 20 | 21 | import com.dajudge.proxybase.ca.Helpers; 22 | 23 | import java.security.KeyPair; 24 | import java.security.cert.Certificate; 25 | import java.security.cert.X509Certificate; 26 | import java.time.Duration; 27 | import java.util.Date; 28 | 29 | import static com.dajudge.kafkaproxy.roundtrip.util.Util.randomIdentifier; 30 | import static com.dajudge.proxybase.ca.Helpers.createKeyStore; 31 | import static com.dajudge.proxybase.ca.Helpers.serialize; 32 | import static java.time.temporal.ChronoUnit.DAYS; 33 | 34 | public class CertAuthority { 35 | private final KeyPair keyPair; 36 | private final X509Certificate cert; 37 | 38 | public CertAuthority(final String dn) { 39 | keyPair = Helpers.keyPair(); 40 | final Date now = Helpers.now(System::currentTimeMillis); 41 | cert = Helpers.selfSignedCert( 42 | dn, 43 | keyPair, 44 | now, 45 | Helpers.plus(now, Duration.of(10, DAYS)), 46 | "SHA256withRSA", 47 | true 48 | ); 49 | } 50 | 51 | public KeyStoreData createSignedKeyPair(final String dn, final String keystoreType) { 52 | final KeyPair newKeyPair = Helpers.keyPair(); 53 | final Date now = Helpers.now(System::currentTimeMillis); 54 | final X509Certificate newCert = Helpers.sign( 55 | dn, 56 | cert.getSubjectDN().getName(), 57 | keyPair.getPrivate(), 58 | "SHA256withRSA", 59 | newKeyPair.getPublic(), 60 | now, 61 | Helpers.plus(now, Duration.of(10, DAYS)), 62 | false 63 | ); 64 | final char[] keyStorePassword = randomIdentifier().toCharArray(); 65 | final char[] keyPassword = randomIdentifier().toCharArray(); 66 | final byte[] keyStore = keyStoreOf(newKeyPair, newCert, keyStorePassword, keyPassword, keystoreType); 67 | return new KeyStoreData(keyStore, keyStorePassword, keyPassword, keystoreType); 68 | } 69 | 70 | private byte[] keyStoreOf( 71 | final KeyPair keyPair, 72 | final X509Certificate cert, 73 | final char[] keyStorePassword, 74 | final char[] keyPassword, 75 | final String type 76 | ) { 77 | return serialize(createKeyStore(keyStore -> { 78 | keyStore.setKeyEntry("key", keyPair.getPrivate(), keyPassword, new Certificate[]{cert}); 79 | }, type), keyStorePassword); 80 | } 81 | 82 | public KeyStoreData getTrustStore(final String type) { 83 | final char[] keyStorePassword = randomIdentifier().toCharArray(); 84 | final byte[] keyStore = serialize(createKeyStore(keystore -> { 85 | keystore.setCertificateEntry("ca", cert); 86 | }, type), keyStorePassword); 87 | return new KeyStoreData(keyStore, keyStorePassword, null, type); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /itests/src/test/java/com/dajudge/kafkaproxy/itest/util/ITest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package com.dajudge.kafkaproxy.itest.util; 18 | 19 | import com.github.dockerjava.api.DockerClient; 20 | import com.github.dockerjava.api.command.ExecCreateCmdResponse; 21 | import com.github.dockerjava.api.command.InspectContainerResponse; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | import org.testcontainers.DockerClientFactory; 25 | import org.testcontainers.containers.GenericContainer; 26 | import org.testcontainers.containers.output.FrameConsumerResultCallback; 27 | import org.testcontainers.containers.output.OutputFrame; 28 | import org.testcontainers.images.AbstractImagePullPolicy; 29 | import org.testcontainers.images.ImageData; 30 | import org.testcontainers.utility.DockerImageName; 31 | 32 | import java.io.IOException; 33 | 34 | public class ITest extends GenericContainer { 35 | private static final Logger LOG = LoggerFactory.getLogger(ITest.class); 36 | private static final AbstractImagePullPolicy NEVER = new AbstractImagePullPolicy() { 37 | @Override 38 | protected boolean shouldPullCached( 39 | final DockerImageName dockerImageName, 40 | final ImageData imageData 41 | ) { 42 | return false; 43 | } 44 | }; 45 | 46 | public ITest(final String image) { 47 | super(image); 48 | this.withImagePullPolicy(NEVER) 49 | .withNetworkMode("host") 50 | .withLogConsumer(outputFrame -> LOG.info(outputFrame.getUtf8String())); 51 | } 52 | 53 | public Integer exec(final String... command) throws InterruptedException, IOException { 54 | final InspectContainerResponse containerInfo = getContainerInfo(); 55 | final String containerId = containerInfo.getId(); 56 | @SuppressWarnings("PMD.CloseResource") 57 | final DockerClient dockerClient = DockerClientFactory.instance().client(); 58 | final ExecCreateCmdResponse execCreateCmdResponse = dockerClient.execCreateCmd(containerId) 59 | .withAttachStdout(true) 60 | .withAttachStderr(true) 61 | .withCmd(command) 62 | .exec(); 63 | 64 | @SuppressWarnings("PMD.CloseResource") 65 | final FrameConsumerResultCallback callback = new FrameConsumerResultCallback(); 66 | Throwable error = null; 67 | 68 | try { 69 | callback.addConsumer(OutputFrame.OutputType.STDOUT, new ToConsoleConsumer(System.out)); 70 | callback.addConsumer(OutputFrame.OutputType.STDERR, new ToConsoleConsumer(System.out)); 71 | dockerClient.execStartCmd(execCreateCmdResponse.getId()).exec(callback).awaitCompletion(); 72 | } catch (final Exception err) { 73 | error = err; 74 | throw err; 75 | } finally { 76 | if (error != null) { 77 | try { 78 | callback.close(); 79 | } catch (final Exception closeError) { 80 | error.addSuppressed(closeError); 81 | } 82 | } else { 83 | callback.close(); 84 | } 85 | } 86 | 87 | return dockerClient.inspectExecCmd(execCreateCmdResponse.getId()).exec().getExitCode(); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /itests/src/test/java/com/dajudge/kafkaproxy/itest/BaseIntegrationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package com.dajudge.kafkaproxy.itest; 18 | 19 | import com.dajudge.kafkaproxy.itest.util.KafkaProxyContainer; 20 | import com.dajudge.kafkaproxy.roundtrip.util.PortFinder; 21 | import org.junit.ClassRule; 22 | import org.testcontainers.containers.KafkaContainer; 23 | import org.testcontainers.utility.DockerImageName; 24 | 25 | import static java.lang.String.format; 26 | import static java.lang.String.valueOf; 27 | import static org.testcontainers.containers.KafkaContainer.KAFKA_PORT; 28 | 29 | public class BaseIntegrationTest { 30 | private static final String CONFLUENT_PLATFORM_VERSION = System.getenv("CONFLUENT_PLATFORM_VERSION"); 31 | private static final String KAFKA_IMAGE = "confluentinc/cp-kafka:" + CONFLUENT_PLATFORM_VERSION; 32 | @ClassRule 33 | public static final KafkaContainer KAFKA = new KafkaContainer(DockerImageName.parse(KAFKA_IMAGE)); 34 | 35 | public T withKafkaProxy(final ThrowingFunction runnable) { 36 | try (final ProxyContainer proxy = createKafkaProxy(KAFKA.getHost(), KAFKA.getMappedPort(KAFKA_PORT))) { 37 | return runnable.apply(proxy.getProxyEndpoint()); 38 | } catch (final Exception e) { 39 | throw new AssertionError(e); 40 | } 41 | } 42 | 43 | private ProxyContainer createKafkaProxy(final String kafkaHost, final int kafkaPort) { 44 | final KafkaProxyPorts ports = findFreePorts(); 45 | final KafkaProxyContainer container = new KafkaProxyContainer() 46 | .withEnv("KAFKAPROXY_BOOTSTRAP_SERVERS", format("%s:%s", kafkaHost, kafkaPort)) 47 | .withEnv("KAFKAPROXY_BASE_PORT", valueOf(ports.proxyPort)) 48 | .withEnv("KAFKAPROXY_HTTP_PORT", valueOf(ports.httpPort)) 49 | .withEnv("KAFKAPROXY_HOSTNAME", "localhost"); 50 | container.start(); 51 | return new ProxyContainer(container, format("localhost:%d", ports.proxyPort)); 52 | } 53 | 54 | private KafkaProxyPorts findFreePorts() { 55 | try (final PortFinder portFinder = new PortFinder()) { 56 | return new KafkaProxyPorts(portFinder.nextPort(), portFinder.nextPort()); 57 | } 58 | } 59 | 60 | private static class KafkaProxyPorts { 61 | private final int proxyPort; 62 | private final int httpPort; 63 | 64 | private KafkaProxyPorts(final int proxyPort, final int httpPort) { 65 | this.proxyPort = proxyPort; 66 | this.httpPort = httpPort; 67 | } 68 | } 69 | 70 | private static class ProxyContainer implements AutoCloseable { 71 | private final KafkaProxyContainer container; 72 | private final String proxyEndpoint; 73 | 74 | private ProxyContainer(final KafkaProxyContainer container, final String proxyEndpoint) { 75 | this.container = container; 76 | this.proxyEndpoint = proxyEndpoint; 77 | } 78 | 79 | @Override 80 | public void close() { 81 | container.close(); 82 | } 83 | 84 | public String getProxyEndpoint() { 85 | return proxyEndpoint; 86 | } 87 | } 88 | 89 | public interface ThrowingFunction { 90 | O apply(I t) throws Exception; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/protocol/KafkaMessage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.protocol; 19 | 20 | import com.dajudge.proxybase.AbstractChunkedMessage; 21 | import io.netty.buffer.ByteBuf; 22 | import org.apache.kafka.common.protocol.ApiKeys; 23 | import org.apache.kafka.common.requests.AbstractResponse; 24 | import org.apache.kafka.common.requests.RequestHeader; 25 | import org.apache.kafka.common.requests.ResponseHeader; 26 | 27 | import java.nio.ByteBuffer; 28 | import java.util.List; 29 | import java.util.function.Function; 30 | 31 | import static io.netty.buffer.Unpooled.buffer; 32 | import static java.util.Arrays.asList; 33 | import static org.apache.kafka.common.requests.AbstractResponse.parseResponse; 34 | 35 | public class KafkaMessage extends AbstractChunkedMessage { 36 | private static final int KAFKA_HEADER_LENGTH = 4; 37 | private static final int COMPLETE_CHUNK_COUNT = 2; 38 | 39 | public KafkaMessage() { 40 | super(KAFKA_HEADER_LENGTH); 41 | } 42 | 43 | public KafkaMessage(final ByteBuf payload) { 44 | super(asList(lengthHeader(payload.readableBytes()), payload)); 45 | } 46 | 47 | private static ByteBuf lengthHeader(final int payloadBytes) { 48 | final ByteBuf buffer = buffer(4); 49 | buffer.writeInt(payloadBytes); 50 | return buffer; 51 | } 52 | 53 | @Override 54 | protected int nextChunkSize(final List list) { 55 | if (list.size() == COMPLETE_CHUNK_COUNT) { 56 | return NO_MORE_CHUNKS; 57 | } 58 | final ByteBuf header = list.get(0); 59 | try { 60 | return header.readInt(); 61 | } finally { 62 | header.resetReaderIndex(); 63 | } 64 | } 65 | 66 | public int correlationId() { 67 | return withPayload(ByteBuf::readInt); 68 | } 69 | 70 | public RequestHeader requestHeader() { 71 | return withPayload(payload -> RequestHeader.parse(payload.nioBuffer())); 72 | } 73 | 74 | public ResponseHeader responseHeader(final RequestHeader requestHeader) { 75 | final short responseHeaderVersion = requestHeader.apiKey().responseHeaderVersion(requestHeader.apiVersion()); 76 | return withPayload(payload -> ResponseHeader.parse(payload.nioBuffer(), responseHeaderVersion)); 77 | } 78 | 79 | public T responseBody(final RequestHeader requestHeader) { 80 | final short apiVersion = requestHeader.apiVersion(); 81 | final ApiKeys apiKey = requestHeader.apiKey(); 82 | return withPayload(payload -> { 83 | final ByteBuffer nioBuffer = payload.nioBuffer(); 84 | ResponseHeader.parse(nioBuffer, apiKey.responseHeaderVersion(apiVersion)); // Skip over header 85 | // It's up to the caller to know what this message contains 86 | @SuppressWarnings("unchecked") final T response = (T) parseResponse(apiKey, nioBuffer, apiVersion); 87 | return response; 88 | }); 89 | } 90 | 91 | private T withPayload(final Function f) { 92 | final ByteBuf payload = getChunks().get(1); 93 | try { 94 | return f.apply(payload); 95 | } finally { 96 | payload.resetReaderIndex(); 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/protocol/rewrite/FindCoordinatorRewriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.protocol.rewrite; 19 | 20 | import com.dajudge.kafkaproxy.BrokerMapping; 21 | import com.dajudge.proxybase.config.Endpoint; 22 | import org.apache.kafka.common.message.FindCoordinatorResponseData; 23 | import org.apache.kafka.common.protocol.ApiKeys; 24 | import org.apache.kafka.common.requests.FindCoordinatorResponse; 25 | import org.apache.kafka.common.requests.RequestHeader; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | import java.lang.reflect.Field; 30 | import java.util.function.Function; 31 | 32 | import static io.netty.util.internal.StringUtil.isNullOrEmpty; 33 | 34 | public class FindCoordinatorRewriter extends BaseReflectingRewriter { 35 | private static final Logger LOG = LoggerFactory.getLogger(FindCoordinatorRewriter.class); 36 | private final Function brokerResolver; 37 | 38 | public FindCoordinatorRewriter(final Function brokerResolver) { 39 | this.brokerResolver = brokerResolver; 40 | } 41 | 42 | @Override 43 | public boolean appliesTo(final RequestHeader requestHeader) { 44 | return requestHeader.apiKey() == ApiKeys.FIND_COORDINATOR; 45 | } 46 | 47 | 48 | @Override 49 | @SuppressWarnings("PMD.AvoidAccessibilityAlteration") 50 | protected void rewrite(final FindCoordinatorResponse response) throws NoSuchFieldException, IllegalAccessException { 51 | final Field field = FindCoordinatorResponse.class.getDeclaredField("data"); 52 | field.setAccessible(true); 53 | final FindCoordinatorResponseData data = (FindCoordinatorResponseData) field.get(response); 54 | if (!isNullOrEmpty(data.host())) { 55 | final BrokerMapping mapping = brokerResolver.apply(new Endpoint(data.host(), data.port())); 56 | LOG.debug( 57 | "Rewriting {}: {}:{} (sole) -> {}:{}", 58 | ApiKeys.FIND_COORDINATOR, 59 | data.host(), 60 | data.port(), 61 | mapping.getProxy().getHost(), 62 | mapping.getProxy().getPort() 63 | ); 64 | data.setHost(mapping.getProxy().getHost()); 65 | data.setPort(mapping.getProxy().getPort()); 66 | } 67 | if (data.coordinators() != null) { 68 | data.coordinators() 69 | .stream() 70 | .filter(it -> !isNullOrEmpty(it.host())) 71 | .forEach(coordinator -> { 72 | final Endpoint coordinatorEndpoint = new Endpoint(coordinator.host(), coordinator.port()); 73 | final BrokerMapping mapping = brokerResolver.apply(coordinatorEndpoint); 74 | LOG.debug( 75 | "Rewriting {}: {}:{} (list) -> {}:{}", 76 | ApiKeys.FIND_COORDINATOR, 77 | coordinator.host(), 78 | coordinator.port(), 79 | mapping.getProxy().getHost(), 80 | mapping.getProxy().getPort() 81 | ); 82 | coordinator.setHost(mapping.getProxy().getHost()); 83 | coordinator.setPort(mapping.getProxy().getPort()); 84 | }); 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/comm/SslClientSecurity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.comm; 19 | 20 | import com.dajudge.kafkaproxy.roundtrip.ssl.KeyStoreData; 21 | 22 | import java.io.File; 23 | import java.io.FileOutputStream; 24 | import java.io.IOException; 25 | import java.util.Optional; 26 | import java.util.function.Function; 27 | 28 | import static java.io.File.createTempFile; 29 | 30 | public class SslClientSecurity implements ClientSecurity { 31 | private final File trustStoreFile; 32 | private final Optional> keyStoreFactory; 33 | private final KeyStoreData trustStore; 34 | private final String proxyCertStrategy; 35 | 36 | public SslClientSecurity( 37 | final KeyStoreData trustStore, 38 | final Optional> keyStoreFactory, 39 | final String proxyCertStrategy 40 | ) { 41 | this.trustStore = trustStore; 42 | trustStoreFile = writeToTemp(trustStore); 43 | this.keyStoreFactory = keyStoreFactory; 44 | this.proxyCertStrategy = proxyCertStrategy; 45 | } 46 | 47 | private static File writeToTemp(final KeyStoreData trustStore) { 48 | try { 49 | File keyStoreFile = createTempFile("kafkaproxy-test-", ".jks"); 50 | keyStoreFile.deleteOnExit(); 51 | try (final FileOutputStream fos = new FileOutputStream(keyStoreFile)) { 52 | fos.write(trustStore.getBytes()); 53 | } 54 | return keyStoreFile; 55 | } catch (final IOException e) { 56 | throw new RuntimeException("Failed to write keystore to /tmp", e); 57 | } 58 | } 59 | 60 | @Override 61 | public String getProtocol() { 62 | return "SSL"; 63 | } 64 | 65 | 66 | @Override 67 | public String getTrustStoreLocation() { 68 | return trustStoreFile.getAbsolutePath(); 69 | } 70 | 71 | @Override 72 | public char[] getTrustStorePassword() { 73 | return trustStore.getKeyStorePassword(); 74 | } 75 | 76 | @Override 77 | public String getTrustStoreType() { 78 | return trustStore.getType(); 79 | } 80 | 81 | @Override 82 | public ClientSslConfig newClient(final String dn) { 83 | final Optional keyStore = keyStoreFactory.map(f -> f.apply(dn)); 84 | final Optional keyStoreFile = keyStore.map(SslClientSecurity::writeToTemp); 85 | return new ClientSslConfig() { 86 | @Override 87 | public String getKeyStoreLocation() { 88 | return keyStoreFile.map(File::getAbsolutePath).orElse(null); 89 | } 90 | 91 | @Override 92 | public char[] getKeyStorePassword() { 93 | return keyStore.map(KeyStoreData::getKeyStorePassword).orElse(null); 94 | } 95 | 96 | @Override 97 | public char[] getKeyPassword() { 98 | return keyStore.map(KeyStoreData::getKeyPassword).orElse(null); 99 | } 100 | 101 | @Override 102 | public String getKeyStoreType() { 103 | return keyStore.map(KeyStoreData::getType).orElse(null); 104 | } 105 | 106 | @Override 107 | public String getProxyCertStrategy() { 108 | return proxyCertStrategy; 109 | } 110 | }; 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/config/KeyStoreConfigHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.config; 19 | 20 | import com.dajudge.proxybase.certs.KeyStoreConfig; 21 | 22 | import java.util.Optional; 23 | 24 | final class KeyStoreConfigHelper { 25 | private static final String SUFFIX_PASSWORD = "PASSWORD"; 26 | private static final String SUFFIX_REFRESH_SECS = "REFRESH_SECS"; 27 | private static final String SUFFIX_TYPE = "TYPE"; 28 | private static final String SUFFIX_LOCATION = "LOCATION"; 29 | private static final String SUFFIX_PASSWORD_LOCATION = SUFFIX_PASSWORD + "_" + SUFFIX_LOCATION; 30 | private static final String QUALIFIER_TRUSTSTORE = "TRUSTSTORE_"; 31 | private static final String QUALIFIER_KEYSTORE = "KEYSTORE_"; 32 | private static final String QUALIFIER_KEY = "KEY_"; 33 | private static final String DEFAULT_TYPE = "jks"; 34 | private static final int DEFAULT_REFRESH_SECS = 300; 35 | 36 | private KeyStoreConfigHelper() { 37 | } 38 | 39 | static KeyStoreConfig requiredKeyStoreConfig(final Environment environment, final String prefix) { 40 | return loadKeyStoreConfig(environment, prefix, true).orElseThrow(IllegalStateException::new); 41 | } 42 | 43 | static Optional optionalKeyStoreConfig(final Environment environment, final String prefix) { 44 | return loadKeyStoreConfig(environment, prefix, false); 45 | } 46 | 47 | static Optional loadTrustStoreConfig( 48 | final Environment environment, 49 | final String prefix 50 | ) { 51 | final String truststorePrefix = prefix + QUALIFIER_TRUSTSTORE; 52 | return environment.optionalString(truststorePrefix + SUFFIX_LOCATION).map(path -> new KeyStoreConfig( 53 | path, 54 | environment.optionalString(truststorePrefix + SUFFIX_PASSWORD).orElse("").toCharArray(), 55 | environment.optionalString(truststorePrefix + SUFFIX_PASSWORD_LOCATION).orElse(null), 56 | null, 57 | null, 58 | environment.optionalString(truststorePrefix + SUFFIX_TYPE).orElse(DEFAULT_TYPE), 59 | environment.optionalInt(truststorePrefix + SUFFIX_REFRESH_SECS).orElse(DEFAULT_REFRESH_SECS) * 1000 60 | )); 61 | } 62 | 63 | private static Optional loadKeyStoreConfig( 64 | final Environment environment, 65 | final String prefix, 66 | final boolean required 67 | ) { 68 | final String keystorePrefix = prefix + QUALIFIER_KEYSTORE; 69 | final String keyPrefix = prefix + QUALIFIER_KEY; 70 | if (required) { 71 | environment.requiredString(keystorePrefix + SUFFIX_LOCATION); 72 | } 73 | return environment.optionalString(keystorePrefix + SUFFIX_LOCATION).map(path -> new KeyStoreConfig( 74 | path, 75 | environment.optionalString(keystorePrefix + SUFFIX_PASSWORD).orElse("").toCharArray(), 76 | environment.optionalString(keystorePrefix + SUFFIX_PASSWORD_LOCATION).orElse(null), 77 | environment.optionalString(keyPrefix + SUFFIX_PASSWORD).orElse("").toCharArray(), 78 | environment.optionalString(keyPrefix + SUFFIX_PASSWORD_LOCATION).orElse(null), 79 | environment.optionalString(keystorePrefix + SUFFIX_TYPE).orElse(DEFAULT_TYPE), 80 | environment.optionalInt(keystorePrefix + SUFFIX_REFRESH_SECS).orElse(DEFAULT_REFRESH_SECS) * 1000 81 | )); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/client/ClientFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.client; 19 | 20 | import com.dajudge.kafkaproxy.roundtrip.comm.ClientSecurity; 21 | import com.dajudge.kafkaproxy.roundtrip.comm.ClientSslConfig; 22 | import org.apache.kafka.clients.CommonClientConfigs; 23 | import org.apache.kafka.clients.admin.AdminClient; 24 | import org.apache.kafka.clients.consumer.ConsumerConfig; 25 | import org.apache.kafka.clients.consumer.KafkaConsumer; 26 | import org.apache.kafka.clients.producer.KafkaProducer; 27 | import org.apache.kafka.common.config.SslConfigs; 28 | import org.apache.kafka.common.serialization.StringDeserializer; 29 | import org.apache.kafka.common.serialization.StringSerializer; 30 | import org.slf4j.Logger; 31 | import org.slf4j.LoggerFactory; 32 | 33 | import java.util.HashMap; 34 | import java.util.Map; 35 | 36 | import static com.dajudge.kafkaproxy.roundtrip.util.Util.indent; 37 | import static com.dajudge.kafkaproxy.roundtrip.util.Util.safeToString; 38 | import static java.util.stream.Collectors.joining; 39 | 40 | public class ClientFactory { 41 | private static final Logger LOG = LoggerFactory.getLogger(ClientFactory.class); 42 | private final String bootstrapServers; 43 | private final ClientSecurity clientSecurity; 44 | 45 | public ClientFactory( 46 | final String bootstrapServers, 47 | final ClientSecurity clientSecurity 48 | ) { 49 | this.bootstrapServers = bootstrapServers; 50 | this.clientSecurity = clientSecurity; 51 | } 52 | 53 | private Map connectConfig(final ClientSslConfig sslConfig) { 54 | return new HashMap() {{ 55 | put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); 56 | put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, clientSecurity.getProtocol()); 57 | put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, sslConfig.getKeyStoreLocation()); 58 | put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, safeToString(sslConfig.getKeyStorePassword())); 59 | put(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, sslConfig.getKeyStoreType()); 60 | put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, safeToString(sslConfig.getKeyPassword())); 61 | put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, clientSecurity.getTrustStoreLocation()); 62 | put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, safeToString(clientSecurity.getTrustStorePassword())); 63 | put(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG, clientSecurity.getTrustStoreType()); 64 | }}; 65 | } 66 | 67 | public AdminClient admin(final String dn) { 68 | final Map config = connectConfig(clientSecurity.newClient(dn)); 69 | dumpConfig("Admin", config); 70 | return AdminClient.create(config); 71 | } 72 | 73 | public KafkaProducer producer(final String dn) { 74 | final Map config = connectConfig(clientSecurity.newClient(dn)); 75 | dumpConfig("Producer", config); 76 | return new KafkaProducer<>(config, new StringSerializer(), new StringSerializer()); 77 | } 78 | 79 | public KafkaConsumer consumer(final String dn, final String groupId) { 80 | final Map config = new HashMap(connectConfig(clientSecurity.newClient(dn))) {{ 81 | put(ConsumerConfig.GROUP_ID_CONFIG, groupId); 82 | put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); 83 | put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); 84 | }}; 85 | dumpConfig("Consumer", config); 86 | return new KafkaConsumer<>(config, new StringDeserializer(), new StringDeserializer()); 87 | } 88 | 89 | private void dumpConfig(final String type, final Map config) { 90 | LOG.info("{}:\n{}", type, indent(4, config.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(joining("\n")))); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /itests/src/main/clients/dotnet/TestRunner.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Threading; 4 | using Confluent.Kafka; 5 | using Confluent.Kafka.Admin; 6 | 7 | namespace testrunner 8 | { 9 | public class TestRunner : IDisposable 10 | { 11 | private readonly ISet _messages = new HashSet(); 12 | private readonly PollMaster _pollMaster; 13 | private readonly SinkMaster _sinkMaster; 14 | 15 | private readonly string _bootstrapServers; 16 | private readonly string _topic; 17 | private readonly string _groupId; 18 | 19 | private int _producedMessages; 20 | 21 | private Sink CreateSink() 22 | { 23 | var producer = new ProducerBuilder(new ProducerConfig 24 | { 25 | BootstrapServers = _bootstrapServers 26 | }).Build(); 27 | return new ProducerSink(producer, _topic); 28 | } 29 | 30 | private Poll CreatePoll() 31 | { 32 | var consumer = new ConsumerBuilder(new ConsumerConfig 33 | { 34 | BootstrapServers = _bootstrapServers, 35 | GroupId = _groupId, 36 | AutoOffsetReset = AutoOffsetReset.Earliest, 37 | EnableAutoCommit = false, 38 | EnableAutoOffsetStore = false, 39 | SocketTimeoutMs = (int)TimeSpan.FromSeconds(1).TotalMilliseconds, 40 | }).Build(); 41 | consumer.Subscribe(_topic); 42 | return new ConsumerPoll(consumer); 43 | } 44 | 45 | public TestRunner( 46 | in string bootstrapServers, 47 | in int sinkCount, 48 | in int pollCount 49 | ) 50 | { 51 | _bootstrapServers = bootstrapServers; 52 | _groupId = Guid.NewGuid().ToString(); 53 | _topic = Guid.NewGuid().ToString(); 54 | _pollMaster = new PollMaster(_messages, pollCount, CreatePoll); 55 | _sinkMaster = new SinkMaster(_messages, sinkCount, CreateSink); 56 | } 57 | 58 | public void CreateTopic() 59 | { 60 | using (var admin = new AdminClientBuilder(new AdminClientConfig 61 | { 62 | BootstrapServers = _bootstrapServers 63 | }).Build()) 64 | { 65 | Console.WriteLine("Creating topic: " + _topic); 66 | var topicSpec = new TopicSpecification {NumPartitions = 10, ReplicationFactor = 1, Name = _topic}; 67 | admin.CreateTopicsAsync(new[] {topicSpec}).GetAwaiter().GetResult(); 68 | } 69 | } 70 | 71 | public void Produce() 72 | { 73 | Console.WriteLine("Producing to " + _topic); 74 | var start = TimeUtils.CurrentTimeMillis(); 75 | var lastStats = TimeUtils.CurrentTimeMillis(); 76 | while (TimeUtils.CurrentTimeMillis() - start < 10000) 77 | { 78 | if (TimeUtils.CurrentTimeMillis() - lastStats > 1000) 79 | { 80 | lastStats = TimeUtils.CurrentTimeMillis(); 81 | Console.WriteLine("sent: " + _producedMessages + " inflight: " + _messages.Count); 82 | } 83 | 84 | _sinkMaster.RandomSink().Publish("" + _producedMessages); 85 | _producedMessages++; 86 | if (_producedMessages % 100 == 0) 87 | { 88 | Thread.Yield(); 89 | } 90 | } 91 | } 92 | 93 | public void WaitForConsumers() 94 | { 95 | Console.WriteLine("Producing phase complete, waiting for consumers to finish..."); 96 | while (_messages.Count != 0 && 97 | (TimeUtils.CurrentTimeMillis() - _pollMaster.GetLastMessageTimestamp()) < 10000) 98 | { 99 | Thread.Sleep(1); 100 | } 101 | } 102 | 103 | public void Report() 104 | { 105 | Console.WriteLine("Produced messages: " + _producedMessages); 106 | if (_messages.Count > 20) 107 | { 108 | Console.WriteLine("Unseen messages: " + _messages.Count); 109 | } 110 | else 111 | { 112 | Console.WriteLine("Unseen messages: " + string.Join(",", _messages)); 113 | } 114 | } 115 | 116 | public void Dispose() 117 | { 118 | _pollMaster.Dispose(); 119 | _sinkMaster.Dispose(); 120 | } 121 | 122 | public bool IsSucceeded() 123 | { 124 | return _messages.Count == 0; 125 | } 126 | } 127 | } -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # 4 | # Copyright 2015 the original author or authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | ## 21 | ## Gradle start up script for UN*X 22 | ## 23 | ############################################################################## 24 | 25 | # Attempt to set APP_HOME 26 | # Resolve links: $0 may be a link 27 | PRG="$0" 28 | # Need this for relative symlinks. 29 | while [ -h "$PRG" ] ; do 30 | ls=`ls -ld "$PRG"` 31 | link=`expr "$ls" : '.*-> \(.*\)$'` 32 | if expr "$link" : '/.*' > /dev/null; then 33 | PRG="$link" 34 | else 35 | PRG=`dirname "$PRG"`"/$link" 36 | fi 37 | done 38 | SAVED="`pwd`" 39 | cd "`dirname \"$PRG\"`/" >/dev/null 40 | APP_HOME="`pwd -P`" 41 | cd "$SAVED" >/dev/null 42 | 43 | APP_NAME="Gradle" 44 | APP_BASE_NAME=`basename "$0"` 45 | 46 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 47 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 48 | 49 | # Use the maximum available, or set MAX_FD != -1 to use that value. 50 | MAX_FD="maximum" 51 | 52 | warn () { 53 | echo "$*" 54 | } 55 | 56 | die () { 57 | echo 58 | echo "$*" 59 | echo 60 | exit 1 61 | } 62 | 63 | # OS specific support (must be 'true' or 'false'). 64 | cygwin=false 65 | msys=false 66 | darwin=false 67 | nonstop=false 68 | case "`uname`" in 69 | CYGWIN* ) 70 | cygwin=true 71 | ;; 72 | Darwin* ) 73 | darwin=true 74 | ;; 75 | MINGW* ) 76 | msys=true 77 | ;; 78 | NONSTOP* ) 79 | nonstop=true 80 | ;; 81 | esac 82 | 83 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 84 | 85 | # Determine the Java command to use to start the JVM. 86 | if [ -n "$JAVA_HOME" ] ; then 87 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 88 | # IBM's JDK on AIX uses strange locations for the executables 89 | JAVACMD="$JAVA_HOME/jre/sh/java" 90 | else 91 | JAVACMD="$JAVA_HOME/bin/java" 92 | fi 93 | if [ ! -x "$JAVACMD" ] ; then 94 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 95 | 96 | Please set the JAVA_HOME variable in your environment to match the 97 | location of your Java installation." 98 | fi 99 | else 100 | JAVACMD="java" 101 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 102 | 103 | Please set the JAVA_HOME variable in your environment to match the 104 | location of your Java installation." 105 | fi 106 | 107 | # Increase the maximum file descriptors if we can. 108 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 109 | MAX_FD_LIMIT=`ulimit -H -n` 110 | if [ $? -eq 0 ] ; then 111 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 112 | MAX_FD="$MAX_FD_LIMIT" 113 | fi 114 | ulimit -n $MAX_FD 115 | if [ $? -ne 0 ] ; then 116 | warn "Could not set maximum file descriptor limit: $MAX_FD" 117 | fi 118 | else 119 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 120 | fi 121 | fi 122 | 123 | # For Darwin, add options to specify how the application appears in the dock 124 | if $darwin; then 125 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 126 | fi 127 | 128 | # For Cygwin, switch paths to Windows format before running java 129 | if $cygwin ; then 130 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 131 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 132 | JAVACMD=`cygpath --unix "$JAVACMD"` 133 | 134 | # We build the pattern for arguments to be converted via cygpath 135 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 136 | SEP="" 137 | for dir in $ROOTDIRSRAW ; do 138 | ROOTDIRS="$ROOTDIRS$SEP$dir" 139 | SEP="|" 140 | done 141 | OURCYGPATTERN="(^($ROOTDIRS))" 142 | # Add a user-defined pattern to the cygpath arguments 143 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 144 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 145 | fi 146 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 147 | i=0 148 | for arg in "$@" ; do 149 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 150 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 151 | 152 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 153 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 154 | else 155 | eval `echo args$i`="\"$arg\"" 156 | fi 157 | i=$((i+1)) 158 | done 159 | case $i in 160 | (0) set -- ;; 161 | (1) set -- "$args0" ;; 162 | (2) set -- "$args0" "$args1" ;; 163 | (3) set -- "$args0" "$args1" "$args2" ;; 164 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 165 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 166 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 167 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 168 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 169 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 170 | esac 171 | fi 172 | 173 | # Escape application args 174 | save () { 175 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 176 | echo " " 177 | } 178 | APP_ARGS=$(save "$@") 179 | 180 | # Collect all arguments for the java command, following the shell quoting and substitution rules 181 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 182 | 183 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 184 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 185 | cd "$(dirname "$0")" 186 | fi 187 | 188 | exec "$JAVACMD" "$@" 189 | -------------------------------------------------------------------------------- /itests/src/main/clients/java/gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # 4 | # Copyright 2015 the original author or authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | ## 21 | ## Gradle start up script for UN*X 22 | ## 23 | ############################################################################## 24 | 25 | # Attempt to set APP_HOME 26 | # Resolve links: $0 may be a link 27 | PRG="$0" 28 | # Need this for relative symlinks. 29 | while [ -h "$PRG" ] ; do 30 | ls=`ls -ld "$PRG"` 31 | link=`expr "$ls" : '.*-> \(.*\)$'` 32 | if expr "$link" : '/.*' > /dev/null; then 33 | PRG="$link" 34 | else 35 | PRG=`dirname "$PRG"`"/$link" 36 | fi 37 | done 38 | SAVED="`pwd`" 39 | cd "`dirname \"$PRG\"`/" >/dev/null 40 | APP_HOME="`pwd -P`" 41 | cd "$SAVED" >/dev/null 42 | 43 | APP_NAME="Gradle" 44 | APP_BASE_NAME=`basename "$0"` 45 | 46 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 47 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 48 | 49 | # Use the maximum available, or set MAX_FD != -1 to use that value. 50 | MAX_FD="maximum" 51 | 52 | warn () { 53 | echo "$*" 54 | } 55 | 56 | die () { 57 | echo 58 | echo "$*" 59 | echo 60 | exit 1 61 | } 62 | 63 | # OS specific support (must be 'true' or 'false'). 64 | cygwin=false 65 | msys=false 66 | darwin=false 67 | nonstop=false 68 | case "`uname`" in 69 | CYGWIN* ) 70 | cygwin=true 71 | ;; 72 | Darwin* ) 73 | darwin=true 74 | ;; 75 | MINGW* ) 76 | msys=true 77 | ;; 78 | NONSTOP* ) 79 | nonstop=true 80 | ;; 81 | esac 82 | 83 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 84 | 85 | # Determine the Java command to use to start the JVM. 86 | if [ -n "$JAVA_HOME" ] ; then 87 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 88 | # IBM's JDK on AIX uses strange locations for the executables 89 | JAVACMD="$JAVA_HOME/jre/sh/java" 90 | else 91 | JAVACMD="$JAVA_HOME/bin/java" 92 | fi 93 | if [ ! -x "$JAVACMD" ] ; then 94 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 95 | 96 | Please set the JAVA_HOME variable in your environment to match the 97 | location of your Java installation." 98 | fi 99 | else 100 | JAVACMD="java" 101 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 102 | 103 | Please set the JAVA_HOME variable in your environment to match the 104 | location of your Java installation." 105 | fi 106 | 107 | # Increase the maximum file descriptors if we can. 108 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 109 | MAX_FD_LIMIT=`ulimit -H -n` 110 | if [ $? -eq 0 ] ; then 111 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 112 | MAX_FD="$MAX_FD_LIMIT" 113 | fi 114 | ulimit -n $MAX_FD 115 | if [ $? -ne 0 ] ; then 116 | warn "Could not set maximum file descriptor limit: $MAX_FD" 117 | fi 118 | else 119 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 120 | fi 121 | fi 122 | 123 | # For Darwin, add options to specify how the application appears in the dock 124 | if $darwin; then 125 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 126 | fi 127 | 128 | # For Cygwin, switch paths to Windows format before running java 129 | if $cygwin ; then 130 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 131 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 132 | JAVACMD=`cygpath --unix "$JAVACMD"` 133 | 134 | # We build the pattern for arguments to be converted via cygpath 135 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 136 | SEP="" 137 | for dir in $ROOTDIRSRAW ; do 138 | ROOTDIRS="$ROOTDIRS$SEP$dir" 139 | SEP="|" 140 | done 141 | OURCYGPATTERN="(^($ROOTDIRS))" 142 | # Add a user-defined pattern to the cygpath arguments 143 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 144 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 145 | fi 146 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 147 | i=0 148 | for arg in "$@" ; do 149 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 150 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 151 | 152 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 153 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 154 | else 155 | eval `echo args$i`="\"$arg\"" 156 | fi 157 | i=$((i+1)) 158 | done 159 | case $i in 160 | (0) set -- ;; 161 | (1) set -- "$args0" ;; 162 | (2) set -- "$args0" "$args1" ;; 163 | (3) set -- "$args0" "$args1" "$args2" ;; 164 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 165 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 166 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 167 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 168 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 169 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 170 | esac 171 | fi 172 | 173 | # Escape application args 174 | save () { 175 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 176 | echo " " 177 | } 178 | APP_ARGS=$(save "$@") 179 | 180 | # Collect all arguments for the java command, following the shell quoting and substitution rules 181 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 182 | 183 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 184 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 185 | cd "$(dirname "$0")" 186 | fi 187 | 188 | exec "$JAVACMD" "$@" 189 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/cluster/container/KafkaContainer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.cluster.container; 19 | 20 | import com.dajudge.kafkaproxy.roundtrip.cluster.KafkaWaitStrategy; 21 | import com.dajudge.kafkaproxy.roundtrip.comm.CommunicationSetup; 22 | import com.dajudge.kafkaproxy.roundtrip.comm.ServerSecurity; 23 | import com.github.dockerjava.api.command.InspectContainerResponse; 24 | import org.jetbrains.annotations.NotNull; 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | import org.testcontainers.containers.GenericContainer; 28 | import org.testcontainers.containers.Network; 29 | import org.testcontainers.containers.output.Slf4jLogConsumer; 30 | import org.testcontainers.images.builder.Transferable; 31 | import org.testcontainers.utility.MountableFile; 32 | 33 | import java.time.Duration; 34 | import java.util.HashMap; 35 | import java.util.Map; 36 | 37 | import static com.dajudge.kafkaproxy.roundtrip.cluster.container.Version.CP_VERSION; 38 | import static com.dajudge.kafkaproxy.roundtrip.util.Util.indent; 39 | import static com.dajudge.kafkaproxy.roundtrip.util.Util.safeToString; 40 | import static java.lang.String.join; 41 | import static java.lang.String.valueOf; 42 | import static java.nio.charset.StandardCharsets.UTF_8; 43 | import static java.util.stream.Collectors.joining; 44 | import static org.testcontainers.images.PullPolicy.alwaysPull; 45 | import static org.testcontainers.utility.MountableFile.forClasspathResource; 46 | 47 | public class KafkaContainer extends GenericContainer { 48 | private static final Logger LOG = LoggerFactory.getLogger(KafkaContainer.class); 49 | private static final int KAFKA_CLIENT_PORT = 9092; 50 | private static final int KAFKA_BROKER_PORT = 9093; 51 | private static final String ENTRYPOINT_PATH = "/customEntrypoint.sh"; 52 | private final String internalHostname; 53 | private final ServerSecurity serverSecurity; 54 | 55 | public KafkaContainer( 56 | final ZookeeperContainer zookeeper, 57 | final int brokerId, 58 | final Network network, 59 | final CommunicationSetup communicationSetup 60 | ) { 61 | super("confluentinc/cp-kafka:" + CP_VERSION); 62 | this.internalHostname = "broker" + brokerId; 63 | serverSecurity = communicationSetup.getServerSecurity("CN=localhost"); 64 | this.withNetwork(network) 65 | .withImagePullPolicy(alwaysPull()) 66 | .withNetworkAliases(internalHostname) 67 | .withCreateContainerCmdModifier(mod -> { 68 | mod.withUser("root"); 69 | }) 70 | .withEnv("KAFKA_ZOOKEEPER_CONNECT", zookeeper.getEndpoint()) 71 | .withEnv("CONFLUENT_SUPPORT_METRICS_ENABLE", "0") 72 | .withEnv("KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR", "1") 73 | .withEnv("KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS", "0") 74 | .withEnv("KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS", "1") 75 | .withEnv("KAFKA_BROKER_ID", valueOf(brokerId)) 76 | .withEnv("KAFKA_NUM_PARTITIONS", "10") 77 | .withEnv("KAFKA_LISTENERS", join(",", 78 | "CLIENT://:" + KAFKA_CLIENT_PORT, 79 | "BROKER://:" + KAFKA_BROKER_PORT 80 | )) 81 | .withEnv("KAFKA_LISTENER_SECURITY_PROTOCOL_MAP", join(",", 82 | "CLIENT:" + serverSecurity.getClientProtocol(), 83 | "BROKER:PLAINTEXT" 84 | )) 85 | .withEnv("KAFKA_INTER_BROKER_LISTENER_NAME", "BROKER") 86 | .withEnv("KAFKA_SSL_TRUSTSTORE_LOCATION", serverSecurity.getTrustStoreLocation()) 87 | .withEnv("KAFKA_SSL_TRUSTSTORE_PASSWORD", safeToString(serverSecurity.getTrustStorePassword())) 88 | .withEnv("KAFKA_SSL_TRUSTSTORE_TYPE", serverSecurity.getTrustStoreType()) 89 | .withEnv("KAFKA_SSL_KEYSTORE_LOCATION", serverSecurity.getKeyStoreLocation()) 90 | .withEnv("KAFKA_SSL_KEYSTORE_PASSWORD", safeToString(serverSecurity.getKeyStorePassword())) 91 | .withEnv("KAFKA_SSL_KEYSTORE_TYPE", serverSecurity.getKeyStoreType()) 92 | .withEnv("KAFKA_SSL_KEY_PASSWORD", safeToString(serverSecurity.getKeyPassword())) 93 | .withEnv("KAFKA_SSL_CLIENT_AUTH", serverSecurity.getClientAuth()) 94 | .withExposedPorts(KAFKA_CLIENT_PORT) 95 | .withCopyFileToContainer(entrypointScript(), ENTRYPOINT_PATH) 96 | .withCommand("sh", ENTRYPOINT_PATH) 97 | .waitingFor(new KafkaWaitStrategy( 98 | KAFKA_CLIENT_PORT, 99 | communicationSetup.getClientSecurity() 100 | )) 101 | .withStartupTimeout(Duration.ofSeconds(300)) 102 | .withLogConsumer(new Slf4jLogConsumer(LOG)); 103 | } 104 | 105 | @NotNull 106 | private MountableFile entrypointScript() { 107 | return forClasspathResource("customEntrypoint.sh", 777); 108 | } 109 | 110 | @Override 111 | protected void containerIsStarting(final InspectContainerResponse containerInfo) { 112 | final String configFile = configFile(); 113 | LOG.info("Uploading keystores to container..."); 114 | serverSecurity.uploadKeyStores(this::copyFileToContainer); 115 | LOG.info("Writing config to container:\n{}", indent(4, configFile)); 116 | copyFileToContainer(Transferable.of(configFile.getBytes(UTF_8)), "/tmp/config.env"); 117 | } 118 | 119 | private String configFile() { 120 | final Map config = new HashMap() {{ 121 | put("KAFKA_ADVERTISED_LISTENERS", join(",", 122 | "CLIENT://localhost:" + getMappedPort(KAFKA_CLIENT_PORT), 123 | "BROKER://" + internalHostname + ":" + KAFKA_BROKER_PORT 124 | )); 125 | }}; 126 | return config.entrySet().stream() 127 | .map((e) -> e.getKey() + "=" + e.getValue()) 128 | .collect(joining("\n")); 129 | } 130 | 131 | public String getClientEndpoint() { 132 | return "localhost:" + getMappedPort(KAFKA_CLIENT_PORT); 133 | } 134 | 135 | @Override 136 | public boolean equals(final Object o) { 137 | return super.equals(o); 138 | } 139 | 140 | @Override 141 | public int hashCode() { 142 | return super.hashCode(); 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/RoundtripTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip; 19 | 20 | import com.dajudge.kafkaproxy.roundtrip.client.ClientFactory; 21 | import com.dajudge.kafkaproxy.roundtrip.cluster.CommunicationSetupBuilder; 22 | import com.dajudge.kafkaproxy.roundtrip.cluster.KafkaClusterBuilder; 23 | import com.dajudge.kafkaproxy.roundtrip.cluster.TestSetup; 24 | import com.dajudge.kafkaproxy.roundtrip.comm.CommunicationSetup; 25 | import org.apache.kafka.clients.admin.AdminClient; 26 | import org.apache.kafka.clients.admin.NewTopic; 27 | import org.apache.kafka.clients.consumer.ConsumerRecords; 28 | import org.apache.kafka.clients.consumer.KafkaConsumer; 29 | import org.apache.kafka.clients.producer.KafkaProducer; 30 | import org.apache.kafka.clients.producer.ProducerRecord; 31 | import org.junit.Test; 32 | import org.junit.runner.RunWith; 33 | import org.junit.runners.Parameterized; 34 | import org.slf4j.Logger; 35 | import org.slf4j.LoggerFactory; 36 | 37 | import java.util.Collection; 38 | import java.util.HashSet; 39 | import java.util.Set; 40 | import java.util.concurrent.ExecutionException; 41 | import java.util.concurrent.TimeoutException; 42 | import java.util.stream.IntStream; 43 | 44 | import static com.dajudge.kafkaproxy.roundtrip.util.Util.randomIdentifier; 45 | import static java.lang.System.currentTimeMillis; 46 | import static java.time.Duration.ofMillis; 47 | import static java.util.Arrays.asList; 48 | import static java.util.Collections.singletonList; 49 | import static java.util.concurrent.TimeUnit.SECONDS; 50 | import static java.util.stream.Collectors.toSet; 51 | import static java.util.stream.StreamSupport.stream; 52 | import static org.junit.Assert.fail; 53 | 54 | @RunWith(Parameterized.class) 55 | public class RoundtripTest { 56 | private static final Logger LOG = LoggerFactory.getLogger(RoundtripTest.class); 57 | private static final String KEYSTORE_TYPE_JKS = "jks"; 58 | private static final String KEYSTORE_TYPE_P12 = "pkcs12"; 59 | 60 | private static class TestConfiguration { 61 | private final CommunicationSetup clientComm; 62 | private final CommunicationSetup brokerComm; 63 | 64 | private TestConfiguration( 65 | final CommunicationSetup clientComm, 66 | final CommunicationSetup brokerComm 67 | ) { 68 | this.clientComm = clientComm; 69 | this.brokerComm = brokerComm; 70 | } 71 | } 72 | 73 | @Parameterized.Parameters 74 | public static Collection data() { 75 | return asList( 76 | new TestConfiguration(plaintext(), plaintext()), 77 | new TestConfiguration(plaintext(), tls(KEYSTORE_TYPE_JKS)), 78 | new TestConfiguration(plaintext(), mtls(KEYSTORE_TYPE_JKS)), 79 | new TestConfiguration(tls(KEYSTORE_TYPE_JKS), tls(KEYSTORE_TYPE_JKS)), 80 | new TestConfiguration(tls(KEYSTORE_TYPE_JKS), mtls(KEYSTORE_TYPE_JKS)), 81 | new TestConfiguration(tls(KEYSTORE_TYPE_JKS), plaintext()), 82 | new TestConfiguration(mtls(KEYSTORE_TYPE_JKS), plaintext()), 83 | new TestConfiguration(mtls(KEYSTORE_TYPE_JKS), tls(KEYSTORE_TYPE_JKS)), 84 | new TestConfiguration(mtls(KEYSTORE_TYPE_JKS), mtls(KEYSTORE_TYPE_JKS)), 85 | new TestConfiguration(mtls(KEYSTORE_TYPE_P12), mtls(KEYSTORE_TYPE_P12)) 86 | ); 87 | } 88 | 89 | @Parameterized.Parameter 90 | public TestConfiguration config; 91 | 92 | @Test 93 | public void roundtrip_test() throws Exception { 94 | try (final TestSetup testSetup = createTestSetup()) { 95 | runRoundtripTest( 96 | testSetup.getDirectClientFactory(), 97 | testSetup.getProxiedClientFactory() 98 | ); 99 | } 100 | } 101 | 102 | private TestSetup createTestSetup() { 103 | LOG.info("Broker comm: {}", config.brokerComm); 104 | LOG.info("Client comm: {}", config.clientComm); 105 | return new KafkaClusterBuilder() 106 | .withKafka(config.brokerComm) 107 | .withProxy(config.clientComm) 108 | .build(); 109 | } 110 | 111 | private void runRoundtripTest( 112 | final ClientFactory producerClientFactory, 113 | final ClientFactory consumerClientFactory 114 | ) throws InterruptedException, ExecutionException, TimeoutException { 115 | final String topic = randomIdentifier(); 116 | final String groupId = randomIdentifier(); 117 | final Set keysToSend = IntStream.range(0, 100).mapToObj(i -> "key" + i) 118 | .collect(toSet()); 119 | LOG.info("Creating topic {}...", topic); 120 | try (final AdminClient admin = producerClientFactory.admin("CN=admin")) { 121 | admin.createTopics(singletonList(new NewTopic(topic, 10, (short) 1))).all() 122 | .get(30, SECONDS); 123 | } 124 | LOG.info("Producing message to {}...", topic); 125 | try (final KafkaProducer producer = producerClientFactory.producer("CN=producer")) { 126 | keysToSend.forEach(key -> { 127 | try { 128 | producer.send(new ProducerRecord<>(topic, key, "value")).get(30, SECONDS); 129 | } catch (final InterruptedException | ExecutionException | TimeoutException e) { 130 | throw new RuntimeException(e); 131 | } 132 | }); 133 | } 134 | LOG.info("Consuming roundtrip message from {}...", topic); 135 | try (final KafkaConsumer consumer = consumerClientFactory.consumer("CN=consumer", groupId)) { 136 | consumer.subscribe(singletonList(topic)); 137 | final long start = currentTimeMillis(); 138 | final HashSet consumedKeys = new HashSet<>(); 139 | while ((currentTimeMillis() - start) < 10000) { 140 | final ConsumerRecords records = consumer.poll(ofMillis(100)); 141 | if (records.isEmpty()) { 142 | continue; 143 | } 144 | stream(records.spliterator(), false).forEach(r -> consumedKeys.add(r.key())); 145 | if (consumedKeys.equals(keysToSend)) { 146 | return; 147 | } 148 | } 149 | fail("Roundtrip did not complete in time"); 150 | } 151 | } 152 | 153 | private static CommunicationSetup tls(final String keyStoreType) { 154 | return new CommunicationSetupBuilder().withSsl(keyStoreType).build(); 155 | } 156 | 157 | private static CommunicationSetup mtls(final String keyStoreType) { 158 | return new CommunicationSetupBuilder().withMutualTls(keyStoreType).build(); 159 | } 160 | 161 | private static CommunicationSetup plaintext() { 162 | return new CommunicationSetupBuilder().withPlaintext().build(); 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /core/src/main/java/com/dajudge/kafkaproxy/KafkaProxyApplication.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy; 19 | 20 | import com.dajudge.kafkaproxy.config.ApplicationConfig; 21 | import com.dajudge.kafkaproxy.config.BrokerConfigSource.BrokerConfig; 22 | import com.dajudge.kafkaproxy.config.Environment; 23 | import com.dajudge.kafkaproxy.protocol.DecodingKafkaMessageInboundHandler; 24 | import com.dajudge.kafkaproxy.protocol.EncodingKafkaMessageOutboundHandler; 25 | import com.dajudge.kafkaproxy.protocol.KafkaRequestStore; 26 | import com.dajudge.kafkaproxy.protocol.RewritingKafkaMessageDuplexHandler; 27 | import com.dajudge.kafkaproxy.protocol.rewrite.CompositeRewriter; 28 | import com.dajudge.kafkaproxy.protocol.rewrite.FindCoordinatorRewriter; 29 | import com.dajudge.kafkaproxy.protocol.rewrite.MetadataRewriter; 30 | import com.dajudge.kafkaproxy.protocol.rewrite.ResponseRewriter; 31 | import com.dajudge.proxybase.ProxyApplication; 32 | import com.dajudge.proxybase.ProxyChannelFactory; 33 | import com.dajudge.proxybase.ProxyChannelFactory.ProxyChannelInitializer; 34 | import com.dajudge.proxybase.RelayingChannelInboundHandler; 35 | import com.dajudge.proxybase.certs.Filesystem; 36 | import com.dajudge.proxybase.config.DownstreamSslConfig; 37 | import com.dajudge.proxybase.config.Endpoint; 38 | import com.dajudge.proxybase.config.UpstreamSslConfig; 39 | import io.netty.channel.Channel; 40 | import io.netty.channel.ChannelHandler; 41 | import org.slf4j.Logger; 42 | import org.slf4j.LoggerFactory; 43 | 44 | import java.util.HashMap; 45 | import java.util.Map; 46 | import java.util.Optional; 47 | import java.util.function.Consumer; 48 | import java.util.function.Function; 49 | import java.util.function.Supplier; 50 | 51 | import static com.dajudge.proxybase.DownstreamSslHandlerFactory.createDownstreamSslHandler; 52 | import static com.dajudge.proxybase.UpstreamSslHandlerFactory.createUpstreamSslHandler; 53 | import static java.util.Arrays.asList; 54 | import static java.util.Collections.synchronizedMap; 55 | 56 | public class KafkaProxyApplication extends ProxyApplication { 57 | 58 | private static final Logger LOG = LoggerFactory.getLogger(KafkaProxyApplication.class); 59 | 60 | public KafkaProxyApplication( 61 | final ApplicationConfig appConfig, 62 | final Supplier clock, 63 | final Filesystem filesystem 64 | ) { 65 | super(createProxyRuntime(appConfig, clock, filesystem)); 66 | LOG.trace("Kafkaproxy init complete"); 67 | } 68 | 69 | private static Consumer createProxyRuntime( 70 | final ApplicationConfig appConfig, 71 | final Supplier clock, 72 | final Filesystem filesystem 73 | ) { 74 | final BrokerConfig brokerConfig = appConfig.require(BrokerConfig.class); 75 | final BrokerMapper brokerMapper = new BrokerMapper(brokerConfig); 76 | final Optional upstreamSslConfig = appConfig.optional(UpstreamSslConfig.class); 77 | final Optional downstreamSslConfig = appConfig.optional(DownstreamSslConfig.class); 78 | return channelFactory -> { 79 | final Map activeProxies = synchronizedMap(new HashMap<>()); 80 | final Function brokerResolver = new Function() { 81 | @Override 82 | public BrokerMapping apply(final Endpoint brokerEndpoint) { 83 | return activeProxies.computeIfAbsent(brokerEndpoint, createProxy( 84 | channelFactory, 85 | brokerMapper, 86 | this, 87 | brokerConfig.getBindAddress(), 88 | upstreamSslConfig, 89 | downstreamSslConfig, 90 | clock, 91 | filesystem 92 | )); 93 | } 94 | }; 95 | brokerMapper.getBootstrapBrokers().forEach(brokerResolver::apply); 96 | }; 97 | } 98 | 99 | private static Function createProxy( 100 | final ProxyChannelFactory channelFactory, 101 | final BrokerMapper brokerMapper, 102 | final Function brokerResolver, 103 | final String bindAddress, 104 | final Optional upstreamSslConfig, 105 | final Optional downstreamSslConfig, 106 | final Supplier clock, 107 | final Filesystem filesystem 108 | ) { 109 | return brokerEndpoint -> { 110 | final BrokerMapping mapping = brokerMapper.getBrokerMapping(brokerEndpoint); 111 | final Endpoint proxyEndpoint = mapping.getProxy(); 112 | LOG.info("Initializing proxy {} for broker {}", proxyEndpoint, brokerEndpoint); 113 | final ProxyChannelInitializer initializer = (upstreamChannel, downstreamChannel) -> { 114 | configureUpstream( 115 | upstreamSslConfig, 116 | upstreamChannel, 117 | downstreamChannel, 118 | clock, 119 | filesystem 120 | ); 121 | configureDownstream( 122 | brokerEndpoint, 123 | downstreamSslConfig, 124 | upstreamChannel, 125 | downstreamChannel, 126 | brokerResolver, 127 | clock, filesystem 128 | ); 129 | }; 130 | channelFactory.createProxyChannel( 131 | new Endpoint(bindAddress, proxyEndpoint.getPort()), 132 | brokerEndpoint, 133 | initializer 134 | ); 135 | return mapping; 136 | }; 137 | } 138 | 139 | private static void configureUpstream( 140 | final Optional sslConfig, 141 | final Channel upstreamChannel, 142 | final Channel downstreamChannel, 143 | final Supplier clock, 144 | final Filesystem filesystem 145 | ) { 146 | upstreamChannel.pipeline().addLast(new DecodingKafkaMessageInboundHandler()); 147 | upstreamChannel.pipeline().addLast(new EncodingKafkaMessageOutboundHandler()); 148 | upstreamChannel.pipeline().addLast(new RelayingChannelInboundHandler("downstream", downstreamChannel)); 149 | sslConfig.ifPresent(it -> upstreamChannel.pipeline().addAfter( 150 | LOGGING_CONTEXT_HANDLER, 151 | "SSL", 152 | createUpstreamSslHandler(it, clock, filesystem) 153 | )); 154 | } 155 | 156 | private static void configureDownstream( 157 | final Endpoint downstream, 158 | final Optional sslConfig, 159 | final Channel upstreamChannel, 160 | final Channel downstreamChannel, 161 | final Function brokerResolver, 162 | final Supplier clock, 163 | final Filesystem filesystem 164 | ) { 165 | final ResponseRewriter rewriter = new CompositeRewriter(asList( 166 | new MetadataRewriter(brokerResolver), 167 | new FindCoordinatorRewriter(brokerResolver) 168 | )); 169 | final KafkaRequestStore kafkaRequestStore = new KafkaRequestStore(rewriter); 170 | downstreamChannel.pipeline().addLast(new EncodingKafkaMessageOutboundHandler()); 171 | downstreamChannel.pipeline().addLast(new DecodingKafkaMessageInboundHandler()); 172 | downstreamChannel.pipeline().addLast(new RewritingKafkaMessageDuplexHandler(kafkaRequestStore)); 173 | downstreamChannel.pipeline().addLast(new RelayingChannelInboundHandler("upstream", upstreamChannel)); 174 | sslConfig.ifPresent(it -> { 175 | final ChannelHandler sslHandler = createDownstreamSslHandler(it, downstream, clock, filesystem) 176 | .apply(downstreamChannel.pipeline().channel()); 177 | downstreamChannel.pipeline().addAfter( 178 | LOGGING_CONTEXT_HANDLER, 179 | "SSL", 180 | sslHandler 181 | ); 182 | }); 183 | } 184 | 185 | public static KafkaProxyApplication create( 186 | final Environment environment, 187 | final Supplier clock, 188 | final Filesystem filesystem 189 | ) { 190 | return new KafkaProxyApplication(new ApplicationConfig(environment), clock, filesystem); 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /core/src/test/java/com/dajudge/kafkaproxy/roundtrip/cluster/KafkaClusterBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019-2021 The kafkaproxy developers (see CONTRIBUTORS) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package com.dajudge.kafkaproxy.roundtrip.cluster; 19 | 20 | import com.dajudge.kafkaproxy.KafkaProxyApplication; 21 | import com.dajudge.kafkaproxy.roundtrip.client.ClientFactory; 22 | import com.dajudge.kafkaproxy.roundtrip.cluster.container.KafkaContainer; 23 | import com.dajudge.kafkaproxy.roundtrip.cluster.container.ZookeeperContainer; 24 | import com.dajudge.kafkaproxy.roundtrip.comm.ClientSecurity; 25 | import com.dajudge.kafkaproxy.roundtrip.comm.ClientSslConfig; 26 | import com.dajudge.kafkaproxy.roundtrip.comm.CommunicationSetup; 27 | import com.dajudge.kafkaproxy.roundtrip.comm.ServerSecurity; 28 | import com.dajudge.kafkaproxy.roundtrip.util.PortFinder; 29 | import com.dajudge.kafkaproxy.roundtrip.util.TestEnvironment; 30 | import com.dajudge.kafkaproxy.roundtrip.util.TestFilesystem; 31 | import org.jetbrains.annotations.NotNull; 32 | import org.slf4j.Logger; 33 | import org.slf4j.LoggerFactory; 34 | import org.testcontainers.containers.Network; 35 | 36 | import java.io.File; 37 | import java.io.IOException; 38 | import java.nio.file.Files; 39 | import java.util.Collection; 40 | import java.util.List; 41 | import java.util.UUID; 42 | 43 | import static com.dajudge.kafkaproxy.roundtrip.util.Util.indent; 44 | import static com.dajudge.kafkaproxy.roundtrip.util.Util.safeToString; 45 | import static java.lang.String.valueOf; 46 | import static java.nio.charset.StandardCharsets.UTF_8; 47 | import static java.util.Arrays.stream; 48 | import static java.util.Collections.singletonList; 49 | import static java.util.stream.Collectors.joining; 50 | import static java.util.stream.Collectors.toList; 51 | import static java.util.stream.IntStream.rangeClosed; 52 | import static org.testcontainers.containers.Network.newNetwork; 53 | 54 | public class KafkaClusterBuilder { 55 | private static final Logger LOG = LoggerFactory.getLogger(KafkaClusterBuilder.class); 56 | private CommunicationSetup brokerComm; 57 | private CommunicationSetup proxyComm; 58 | 59 | public KafkaClusterBuilder withKafka(final CommunicationSetup communicationSetup) { 60 | brokerComm = communicationSetup; 61 | return this; 62 | } 63 | 64 | public KafkaClusterBuilder withProxy(final CommunicationSetup communicationSetup) { 65 | proxyComm = communicationSetup; 66 | return this; 67 | } 68 | 69 | public TestSetup build() { 70 | final KafkaCluster kafka = buildKafkaCluster(); 71 | final int proxyBootstrapPort = freePort(); 72 | @SuppressWarnings("PMD.CloseResource") 73 | final KafkaProxyApplication proxy = buildProxyApp(kafka, proxyBootstrapPort); 74 | final ClientFactory proxiedClientFactory = buildProxiedClientFactory(proxyBootstrapPort); 75 | final ClientFactory directClientFactory = buildDirectClientFactory(kafka); 76 | return new TestSetup(kafka, proxy, proxiedClientFactory, directClientFactory, proxyBootstrapPort); 77 | } 78 | 79 | private int freePort() { 80 | try (final PortFinder portFinder = new PortFinder()) { 81 | return portFinder.nextPort(); 82 | } 83 | } 84 | 85 | private ClientFactory buildProxiedClientFactory(final int bootstrapPort) { 86 | return new ClientFactory("localhost:" + bootstrapPort, proxyComm.getClientSecurity()); 87 | } 88 | 89 | private ClientFactory buildDirectClientFactory(final KafkaCluster kafka) { 90 | return new ClientFactory(kafka.getBootstrapServers(), brokerComm.getClientSecurity()); 91 | } 92 | 93 | @NotNull 94 | private KafkaProxyApplication buildProxyApp( 95 | final KafkaCluster kafka, 96 | final int bootstrapPort 97 | ) { 98 | final ClientSecurity brokerSecurity = brokerComm.getClientSecurity(); 99 | final ServerSecurity proxySecurity = proxyComm.getServerSecurity("CN=localhost"); 100 | final ClientSslConfig proxyClient = brokerSecurity.newClient("CN=proxy"); 101 | final TestEnvironment env = new TestEnvironment() 102 | .withEnv("KAFKAPROXY_BOOTSTRAP_SERVERS", kafka.getBootstrapServerList().iterator().next()) 103 | .withEnv("KAFKAPROXY_BASE_PORT", valueOf(bootstrapPort)) 104 | .withEnv("KAFKAPROXY_HOSTNAME", "localhost") 105 | .withEnv("KAFKAPROXY_KAFKA_SSL_ENABLED", valueOf("SSL".equals(brokerSecurity.getProtocol()))) 106 | .withEnv("KAFKAPROXY_KAFKA_SSL_TRUSTSTORE_LOCATION", brokerSecurity.getTrustStoreLocation()) 107 | .withEnv("KAFKAPROXY_KAFKA_SSL_TRUSTSTORE_PASSWORD", safeToString(brokerSecurity.getTrustStorePassword())) 108 | .withEnv("KAFKAPROXY_KAFKA_SSL_KEYSTORE_LOCATION", proxyClient.getKeyStoreLocation()) 109 | .withEnv("KAFKAPROXY_KAFKA_SSL_KEYSTORE_PASSWORD", safeToString(proxyClient.getKeyStorePassword())) 110 | .withEnv("KAFKAPROXY_KAFKA_SSL_KEYSTORE_TYPE", proxyClient.getKeyStoreType()) 111 | .withEnv("KAFKAPROXY_KAFKA_SSL_KEY_PASSWORD", safeToString(proxyClient.getKeyPassword())) 112 | .withEnv("KAFKAPROXY_KAFKA_SSL_CLIENT_CERT_STRATEGY", proxyClient.getProxyCertStrategy()) 113 | .withEnv("KAFKAPROXY_CLIENT_SSL_ENABLED", valueOf("SSL".equals(proxySecurity.getClientProtocol()))) 114 | .withEnv("KAFKAPROXY_CLIENT_SSL_TRUSTSTORE_LOCATION", proxySecurity.getTrustStoreLocation()) 115 | .withEnv("KAFKAPROXY_CLIENT_SSL_TRUSTSTORE_PASSWORD", safeToString(proxySecurity.getTrustStorePassword())) 116 | .withEnv("KAFKAPROXY_CLIENT_SSL_TRUSTSTORE_TYPE", proxySecurity.getTrustStoreType()) 117 | .withEnv("KAFKAPROXY_CLIENT_SSL_KEYSTORE_LOCATION", proxySecurity.getKeyStoreLocation()) 118 | .withEnv("KAFKAPROXY_CLIENT_SSL_KEY_PASSWORD", safeToString(proxySecurity.getKeyPassword())); 119 | final TestFilesystem filesystem = new TestFilesystem() 120 | .withFile(proxySecurity.getTrustStoreLocation(), proxySecurity.getTrustStore()) 121 | .withFile(proxySecurity.getKeyStoreLocation(), proxySecurity.getKeyStore()) 122 | .withFile(brokerSecurity.getTrustStoreLocation(), read(brokerSecurity.getTrustStoreLocation())) 123 | .withFile(proxyClient.getKeyStoreLocation(), read(proxyClient.getKeyStoreLocation())); 124 | passwordFile(env, filesystem, "KAFKAPROXY_CLIENT_SSL_KEYSTORE_PASSWORD_LOCATION", proxySecurity.getKeyStorePassword()); 125 | final StringBuilder buffer = new StringBuilder(); 126 | env.dump(line -> buffer.append(line + "\n")); 127 | LOG.info("ENV:\n{}", indent(4, buffer.toString())); 128 | return KafkaProxyApplication.create(env, System::currentTimeMillis, filesystem); 129 | } 130 | 131 | private void passwordFile( 132 | final TestEnvironment env, 133 | final TestFilesystem filesystem, 134 | final String varName, 135 | final char[] password 136 | ) { 137 | if (password != null) { 138 | final String path = UUID.randomUUID().toString(); 139 | env.withEnv(varName, path); 140 | filesystem.withFile(path, safeToString(password).getBytes(UTF_8)); 141 | } 142 | } 143 | 144 | private byte[] read(final String path) { 145 | if (path == null) { 146 | return null; 147 | } 148 | try { 149 | return Files.readAllBytes(new File(path).toPath()); 150 | } catch (final IOException e) { 151 | throw new AssertionError(e); 152 | } 153 | } 154 | 155 | private KafkaCluster buildKafkaCluster() { 156 | @SuppressWarnings("PMD.CloseResource") 157 | final Network network = newNetwork(); 158 | @SuppressWarnings("PMD.CloseResource") 159 | final ZookeeperContainer zookeeper = new ZookeeperContainer(network); 160 | zookeeper.start(); 161 | final List kafkaContainers = rangeClosed(1, 3) 162 | .mapToObj(i -> new KafkaContainer(zookeeper, i, network, brokerComm)) 163 | .map(StarterThread::new) 164 | .parallel() 165 | .peek(Thread::start) 166 | .peek(StarterThread::waitForStartup) 167 | .map(StarterThread::getContainer) 168 | .collect(toList()); 169 | final String bootstrapServers = kafkaContainers.stream() 170 | .map(KafkaContainer::getClientEndpoint) 171 | .collect(joining(",")); 172 | return new KafkaCluster(join(singletonList(zookeeper), kafkaContainers), bootstrapServers); 173 | } 174 | 175 | @SafeVarargs 176 | @NotNull 177 | private final List join(final List... collections) { 178 | return stream(collections).flatMap(Collection::stream).collect(toList()); 179 | } 180 | 181 | } 182 | --------------------------------------------------------------------------------