├── .dockerignore ├── .github ├── dependabot.yml ├── release-drafter.yml └── workflows │ └── ci.yml ├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── README.md ├── api ├── build.gradle └── src │ ├── main │ └── java │ │ └── com │ │ └── github │ │ └── bsideup │ │ └── liiklus │ │ ├── positions │ │ ├── GroupId.java │ │ └── PositionsStorage.java │ │ └── records │ │ ├── FiniteRecordsStorage.java │ │ ├── KeyValueExtension.java │ │ ├── LiiklusAttributes.java │ │ ├── LiiklusCloudEvent.java │ │ ├── RecordPostProcessor.java │ │ ├── RecordPreProcessor.java │ │ └── RecordsStorage.java │ └── test │ └── java │ └── com │ └── github │ └── bsideup │ └── liiklus │ ├── positions │ ├── GroupIdTest.java │ └── GroupIdValidationTest.java │ └── records │ └── LiiklusCloudEventTest.java ├── app ├── build.gradle └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── github │ │ │ └── bsideup │ │ │ └── liiklus │ │ │ ├── Application.java │ │ │ ├── config │ │ │ ├── GatewayConfiguration.java │ │ │ ├── RecordPostProcessorChain.java │ │ │ └── RecordPreProcessorChain.java │ │ │ ├── plugins │ │ │ ├── LiiklusExtensionFinder.java │ │ │ ├── LiiklusPluginLoader.java │ │ │ ├── LiiklusPluginManager.java │ │ │ └── LiiklusPluginRepository.java │ │ │ ├── service │ │ │ └── LiiklusService.java │ │ │ └── util │ │ │ └── PropertiesUtil.java │ └── resources │ │ ├── application.yml │ │ └── logback.xml │ └── test │ └── java │ └── com │ └── github │ └── bsideup │ └── liiklus │ ├── AckTest.java │ ├── CloudEventsTest.java │ ├── ConsumerGroupsTest.java │ ├── EndOffsetsTest.java │ ├── GroupVersionTest.java │ ├── PositionsTest.java │ ├── ProfilesTest.java │ ├── SmokeTest.java │ ├── config │ └── GatewayConfigurationTest.java │ └── test │ ├── AbstractIntegrationTest.java │ └── ProcessorPluginMock.java ├── build.gradle ├── client ├── build.gradle └── src │ └── main │ └── java │ └── com │ └── github │ └── bsideup │ └── liiklus │ ├── GRPCLiiklusClient.java │ ├── LiiklusClient.java │ └── RSocketLiiklusClient.java ├── examples ├── java │ ├── build.gradle │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── example │ │ │ └── Consumer.java │ │ ├── proto │ │ └── resources │ │ └── logback.xml └── plugin │ ├── build.gradle │ └── src │ ├── main │ └── java │ │ └── com │ │ └── github │ │ └── bsideup │ │ └── liiklus │ │ └── plugins │ │ └── example │ │ ├── ExampleRecordPostProcessor.java │ │ ├── ExampleRecordPreProcessor.java │ │ └── config │ │ └── ExamplePluginConfiguration.java │ └── test │ ├── java │ └── com │ │ └── github │ │ └── bsideup │ │ └── liiklus │ │ └── plugins │ │ └── example │ │ ├── SmokeTest.java │ │ └── support │ │ └── AbstractIntegrationTest.java │ └── resources │ └── logback-test.xml ├── gradle.properties ├── gradle ├── rerunTests.gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── jitpack.yml ├── plugins ├── dynamodb-positions-storage │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bsideup │ │ │ └── liiklus │ │ │ └── dynamodb │ │ │ ├── DynamoDBPositionsStorage.java │ │ │ └── config │ │ │ └── DynamoDBConfiguration.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bsideup │ │ └── liiklus │ │ └── dynamodb │ │ ├── DynamoDBPositionsStorageTest.java │ │ └── config │ │ └── DynamoDBConfigurationTest.java ├── grpc-transport-auth │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bsideup │ │ │ └── liiklus │ │ │ └── transport │ │ │ └── grpc │ │ │ ├── StaticRSAKeyProvider.java │ │ │ └── config │ │ │ ├── GRPCAuthConfiguration.java │ │ │ └── GRPCTLSConfiguration.java │ │ └── test │ │ ├── java │ │ └── com │ │ │ └── github │ │ │ └── bsideup │ │ │ └── liiklus │ │ │ └── transport │ │ │ └── grpc │ │ │ ├── GRPCAuthTest.java │ │ │ ├── GRPCTLSTest.java │ │ │ ├── StaticRSAKeyProviderTest.java │ │ │ └── config │ │ │ └── GRPCAuthConfigurationTest.java │ │ └── resources │ │ └── keys │ │ └── private_key_main_2048_pkcs8.pem ├── grpc-transport │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bsideup │ │ │ └── liiklus │ │ │ └── transport │ │ │ └── grpc │ │ │ ├── GRPCLiiklusService.java │ │ │ ├── GRPCLiiklusTransportConfigurer.java │ │ │ └── config │ │ │ └── GRPCConfiguration.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bsideup │ │ └── liiklus │ │ └── transport │ │ └── grpc │ │ └── config │ │ └── GRPCConfigurationTest.java ├── inmemory-positions-storage │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bsideup │ │ │ └── liiklus │ │ │ └── positions │ │ │ └── inmemory │ │ │ ├── InMemoryPositionsStorage.java │ │ │ └── config │ │ │ └── InMemoryPositionsConfiguration.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bsideup │ │ └── liiklus │ │ └── positions │ │ └── inmemory │ │ ├── InMemoryPositionsStorageTest.java │ │ └── config │ │ └── InMemoryPositionsConfigurationTest.java ├── inmemory-records-storage │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bsideup │ │ │ └── liiklus │ │ │ └── records │ │ │ └── inmemory │ │ │ ├── InMemoryRecordsStorage.java │ │ │ └── config │ │ │ └── InMemoryRecordsConfiguration.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bsideup │ │ └── liiklus │ │ └── records │ │ └── inmemory │ │ ├── InMemoryRecordsStorageTest.java │ │ └── config │ │ └── InMemoryRecordsConfigurationTest.java ├── kafka-records-storage │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bsideup │ │ │ └── liiklus │ │ │ └── kafka │ │ │ ├── KafkaRecordsStorage.java │ │ │ └── config │ │ │ └── KafkaRecordsStorageConfiguration.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bsideup │ │ └── liiklus │ │ └── kafka │ │ ├── KafkaRecordsStorageTest.java │ │ └── config │ │ └── KafkaRecordsStorageConfigurationTest.java ├── metrics │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bsideup │ │ │ └── liiklus │ │ │ └── metrics │ │ │ ├── MetricsCollector.java │ │ │ └── config │ │ │ └── MetricsConfiguration.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bsideup │ │ └── liiklus │ │ └── metrics │ │ └── config │ │ └── MetricsConfigurationTest.java ├── pulsar-records-storage │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ ├── com │ │ │ └── github │ │ │ │ └── bsideup │ │ │ │ └── liiklus │ │ │ │ └── pulsar │ │ │ │ ├── PulsarRecordsStorage.java │ │ │ │ └── config │ │ │ │ └── PulsarRecordsStorageConfiguration.java │ │ │ └── org │ │ │ └── apache │ │ │ └── pulsar │ │ │ └── client │ │ │ └── impl │ │ │ └── ConsumerImplAccessor.java │ │ └── test │ │ ├── java │ │ └── com │ │ │ └── github │ │ │ └── bsideup │ │ │ └── liiklus │ │ │ └── pulsar │ │ │ ├── AbstractPulsarRecordsStorageTest.java │ │ │ ├── NonPartitionedPulsarRecordsStorageTest.java │ │ │ ├── PulsarRecordsStorageTest.java │ │ │ ├── config │ │ │ └── PulsarRecordsStorageConfigurationTest.java │ │ │ └── container │ │ │ └── PulsarTlsContainer.java │ │ └── resources │ │ ├── .htpasswd │ │ └── certs │ │ ├── README.md │ │ ├── broker.cert.pem │ │ ├── broker.key-pk8.pem │ │ └── ca.cert.pem ├── redis-positions-storage │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bsideup │ │ │ └── liiklus │ │ │ └── positions │ │ │ └── redis │ │ │ ├── RedisPositionsStorage.java │ │ │ └── config │ │ │ └── RedisPositionsConfiguration.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bsideup │ │ └── liiklus │ │ └── positions │ │ └── redis │ │ ├── RedisPositionsStorageTest.java │ │ └── config │ │ └── RedisPositionsConfigurationTest.java ├── rsocket-transport │ ├── build.gradle │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── bsideup │ │ │ └── liiklus │ │ │ └── transport │ │ │ └── rsocket │ │ │ ├── RSocketLiiklusService.java │ │ │ ├── RSocketServerConfigurer.java │ │ │ └── config │ │ │ └── RSocketConfiguration.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── bsideup │ │ └── liiklus │ │ └── transport │ │ └── rsocket │ │ └── config │ │ └── RSocketConfigurationTest.java └── schema │ ├── build.gradle │ └── src │ ├── main │ └── java │ │ └── com │ │ └── github │ │ └── bsideup │ │ └── liiklus │ │ └── schema │ │ ├── JsonSchemaPreProcessor.java │ │ ├── SchemaPluginConfiguration.java │ │ └── internal │ │ └── DeprecatedKeyword.java │ └── test │ ├── java │ └── com │ │ └── github │ │ └── bsideup │ │ └── liiklus │ │ └── schema │ │ ├── JsonSchemaPreProcessorTest.java │ │ ├── SchemaPluginConfigurationTest.java │ │ └── SmokeTest.java │ └── resources │ └── schemas │ └── basic.yml ├── protocol ├── build.gradle └── src │ └── main │ └── proto │ └── LiiklusService.proto ├── settings.gradle ├── tck ├── build.gradle └── src │ ├── main │ └── java │ │ └── com │ │ └── github │ │ └── bsideup │ │ └── liiklus │ │ ├── ApplicationRunner.java │ │ ├── positions │ │ ├── PositionsStorageTestSupport.java │ │ ├── PositionsStorageTests.java │ │ └── tests │ │ │ ├── GroupsTest.java │ │ │ └── PersistenceTest.java │ │ ├── records │ │ ├── RecordStorageTestSupport.java │ │ ├── RecordStorageTests.java │ │ └── tests │ │ │ ├── BackPressureTest.java │ │ │ ├── ConsumerGroupTest.java │ │ │ ├── EndOffsetsTest.java │ │ │ ├── PublishTest.java │ │ │ └── SubscribeTest.java │ │ └── support │ │ └── DisabledUntil.java │ └── test │ └── java │ └── com │ └── github │ └── bsideup │ └── liiklus │ └── ApplicationRunnerTest.java └── testing ├── build.gradle └── src ├── main └── java │ └── com │ └── github │ └── bsideup │ └── liiklus │ └── container │ └── LiiklusContainer.java └── test ├── java └── com │ └── github │ └── bsideup │ └── liiklus │ └── container │ └── LiiklusContainerTest.java └── resources └── logback-test.xml /.dockerignore: -------------------------------------------------------------------------------- 1 | Dockerfile 2 | **/build/ 3 | 4 | .idea/ 5 | .vscode/ 6 | 7 | /protocol/generated/ 8 | /examples/** -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gradle 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "07:00" 8 | timezone: Europe/Berlin 9 | open-pull-requests-limit: 99 10 | rebase-strategy: disabled 11 | - package-ecosystem: docker 12 | directory: "/" 13 | schedule: 14 | interval: daily 15 | time: "07:00" 16 | timezone: Europe/Berlin 17 | open-pull-requests-limit: 99 18 | ignore: 19 | - dependency-name: openjdk 20 | versions: 21 | - "> 11.0.2.pre.jdk" 22 | rebase-strategy: disabled 23 | -------------------------------------------------------------------------------- /.github/release-drafter.yml: -------------------------------------------------------------------------------- 1 | name-template: $NEXT_PATCH_VERSION 2 | tag-template: $NEXT_PATCH_VERSION 3 | template: | 4 | # What's Changed 5 | 6 | $CHANGES 7 | categories: 8 | - title: 🚀 Features 9 | label: enhancement 10 | - title: 🐛 Bug Fixes 11 | label: bug 12 | - title: 📦 Dependency updates 13 | label: dependencies 14 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | strategy: 8 | matrix: 9 | tasks: 10 | - ":app:check" 11 | - "check -x :app:check" 12 | 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Set up JDK 18 | uses: actions/setup-java@v1 19 | with: 20 | java-version: 11 21 | - name: Build with Gradle 22 | run: ./gradlew ${{ matrix.tasks }} 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | 3 | .gradle/ 4 | build/ 5 | 6 | .vscode/ 7 | 8 | .idea 9 | *.iml 10 | *.ipr 11 | *.iws 12 | 13 | 14 | .classpath 15 | .project 16 | bin/ 17 | .settings/ 18 | 19 | /config/ 20 | 21 | generated/ 22 | 23 | infrastructure.yaml 24 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - openjdk11 4 | 5 | sudo: required 6 | services: 7 | - docker 8 | 9 | env: 10 | global: 11 | - JAVA_TOOL_OPTIONS=-Dhttps.protocols=TLSv1.2 12 | 13 | install: 14 | - ./gradlew build -x check 15 | 16 | jobs: 17 | include: 18 | - stage: test 19 | env: [ NAME=app ] 20 | script: ./gradlew :app:check 21 | 22 | - stage: test 23 | env: [ NAME=plugins ] 24 | script: ./gradlew check -x :app:check 25 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:11.0.2-jdk AS workspace 2 | 3 | WORKDIR /root/project 4 | COPY gradle gradle/ 5 | COPY gradlew ./ 6 | RUN ./gradlew --no-daemon --version 7 | 8 | ENV TERM=dumb 9 | 10 | COPY build.gradle ./ 11 | RUN ./gradlew --info --no-daemon --console=plain downloadDependencies 12 | 13 | COPY . . 14 | 15 | RUN ./gradlew --no-daemon --info --console=plain build -x check 16 | 17 | FROM openjdk:11-jre 18 | 19 | WORKDIR /app 20 | 21 | RUN java -Xshare:dump 22 | 23 | COPY --from=workspace /root/project/app/build/libs/app-boot.jar app.jar 24 | COPY --from=workspace /root/project/plugins/*/build/libs/*.jar plugins/ 25 | 26 | ENV JAVA_OPTS="" 27 | ENV JAVA_MEMORY_OPTS="-XX:+ExitOnOutOfMemoryError -XshowSettings:vm -noverify" 28 | 29 | CMD ["sh", "-c", "java -Xshare:on $JAVA_MEMORY_OPTS $JAVA_OPTS -jar app.jar"] 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Sergei Egorov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Liiklus 2 | > Liiklus **[li:klus]** ("traffic" in Estonian) - RSocket/gRPC-based Gateway for the event-based systems from the ones who think that Kafka is too low-level. 3 | 4 | ## Why 5 | * horizontally scalable **RSocket/gRPC streaming gateway** 6 | * supports as many client languages as RSocket+gRPC do (Java, Go, C++, Python, etc...) 7 | * reactive first 8 | * Per-partition **backpressure-aware** sources 9 | * at-least-once/at-most-once delivery guarantees 10 | * **pluggable** event storage (Kafka, Pulsar, Kinesis, etc...) 11 | * pluggable positions storage (DynamoDB, Cassandra, Redis, etc...) 12 | * WIP: cold event storage support (S3, Minio, SQL, key/value, etc...) 13 | 14 | ## Who is using 15 | * https://vivy.com/ - 25+ microservices, an abstraction in front of Kafka for the Shared Log Infrastructure (Event Sourcing / CQRS) 16 | 17 | ## Quick Start 18 | The easiest (and recommended) way to run Liiklus is with Docker: 19 | ```shell 20 | $ docker run \ 21 | -e kafka_bootstrapServers=some.kafka.host:9092 \ 22 | -e storage_positions_type=MEMORY \ # only for testing, DO NOT use in production 23 | -p 6565:6565 \ 24 | bsideup/liiklus:$LATEST_VERSION 25 | ``` 26 | Where the latest version is: 27 | [![](https://img.shields.io/github/release/bsideup/liiklus.svg)](https://github.com/bsideup/liiklus/releases/latest) 28 | 29 | Now use [LiiklusService.proto](protocol/src/main/proto/LiiklusService.proto) to generate your client. 30 | 31 | The clients must implement the following algorithm: 32 | 1. Subscribe to the assignments: 33 | ``` 34 | stub.subscribe(SubscribeRequest( 35 | topic="your-topic", 36 | group="your-consumer-group", 37 | [autoOffsetReset="earliest|latest"] 38 | )) 39 | ``` 40 | 1. For every emitted reply of `Subscribe`, using the same channel, subscribe to the records: 41 | ``` 42 | stub.receive(ReceiveRequest( 43 | assignment=reply.getAssignment() 44 | )) 45 | ``` 46 | 1. ACK records 47 | ``` 48 | stub.ack(AckRequest( 49 | assignment=reply.getAssignment(), 50 | offset=record.getOffset() 51 | )) 52 | ``` 53 | **Note 1:** If you ACK record before processing it you get at-most-once, after processing - at-least-once 54 | **Note 2:** It's recommended to ACK every n-th record, or every n seconds to reduce the load on the positions storage 55 | 56 | 57 | ## Java example: 58 | Example code using [Project Reactor](http://projectreactor.io) and [reactive-grpc](https://github.com/salesforce/reactive-grpc): 59 | ```java 60 | var stub = ReactorLiiklusServiceGrpc.newReactorStub(channel); 61 | stub 62 | .subscribe( 63 | SubscribeRequest.newBuilder() 64 | .setTopic("user-events") 65 | .setGroup("analytics") 66 | .setAutoOffsetReset(AutoOffsetReset.EARLIEST) 67 | .build() 68 | ) 69 | .flatMap(reply -> stub 70 | .receive(ReceiveRequest.newBuilder().setAssignment(reply.getAssignment()).build()) 71 | .window(1000) // ACK every 1000th records 72 | .concatMap( 73 | batch -> batch 74 | .map(ReceiveReply::getRecord) 75 | // TODO process instead of Mono.delay(), i.e. by indexing to ElasticSearch 76 | .concatMap(record -> Mono.delay(Duration.ofMillis(100))) 77 | .sample(Duration.ofSeconds(5)) // ACK every 5 seconds 78 | .onBackpressureLatest() 79 | .delayUntil(record -> stub.ack( 80 | AckRequest.newBuilder() 81 | .setAssignment(reply.getAssignment()) 82 | .setOffset(record.getOffset()) 83 | .build() 84 | )), 85 | 1 86 | ) 87 | ) 88 | .blockLast() 89 | ``` 90 | 91 | Also check [examples/java/](examples/java/) for a complete example 92 | 93 | ## Configuration 94 | The project is based on Spring Boot and uses [it's configuration system](https://docs.spring.io/spring-boot/docs/2.0.0.RELEASE/reference/html/boot-features-external-config.html) 95 | Please check [application.yml](app/src/main/resources/application.yml) for the available configuration keys. 96 | 97 | ## License 98 | 99 | See [LICENSE](LICENSE). 100 | -------------------------------------------------------------------------------- /api/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | } 4 | 5 | sourceCompatibility = targetCompatibility = 8 6 | 7 | java { 8 | withSourcesJar() 9 | } 10 | 11 | dependencies { 12 | api 'org.reactivestreams:reactive-streams' 13 | api 'io.cloudevents:cloudevents-api:1.2.0' 14 | 15 | testImplementation 'org.assertj:assertj-core' 16 | } 17 | -------------------------------------------------------------------------------- /api/src/main/java/com/github/bsideup/liiklus/positions/GroupId.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.positions; 2 | 3 | import lombok.AccessLevel; 4 | import lombok.NonNull; 5 | import lombok.RequiredArgsConstructor; 6 | import lombok.Value; 7 | 8 | import java.util.Comparator; 9 | import java.util.Optional; 10 | import java.util.regex.MatchResult; 11 | import java.util.regex.Matcher; 12 | import java.util.regex.Pattern; 13 | 14 | @Value 15 | @RequiredArgsConstructor(access = AccessLevel.PRIVATE) 16 | public class GroupId implements Comparable { 17 | 18 | public static final Comparator COMPARATOR = Comparator 19 | .comparing(GroupId::getName) 20 | .thenComparing(it -> it.getVersion().orElse(0)); 21 | 22 | public static final String VERSION_SEPARATOR = "-v"; 23 | public static final Pattern VERSION_PATTERN = Pattern.compile("^(.*)-v(\\d+)$"); 24 | 25 | public static GroupId of(String name, int version) { 26 | return of(name, Optional.of(version)); 27 | } 28 | 29 | public static GroupId of(String name, Optional version) { 30 | if (version.orElse(0) < 0) { 31 | throw new IllegalArgumentException("version must be >= 0"); 32 | } 33 | return new GroupId(name, version.filter(it -> it != 0)); 34 | } 35 | 36 | public static GroupId ofString(String str) { 37 | Matcher matcher = VERSION_PATTERN.matcher(str); 38 | 39 | if (matcher.matches()) { 40 | MatchResult result = matcher.toMatchResult(); 41 | 42 | return GroupId.of( 43 | result.group(1), 44 | Optional.ofNullable(result.group(2)).map(Integer::parseInt) 45 | ); 46 | } else { 47 | return GroupId.of( 48 | str, 49 | Optional.empty() 50 | ); 51 | } 52 | } 53 | 54 | @NonNull 55 | String name; 56 | 57 | @NonNull 58 | Optional version; 59 | 60 | public String asString() { 61 | return name + version.map(it -> VERSION_SEPARATOR + it).orElse(""); 62 | } 63 | 64 | @Override 65 | public int compareTo(GroupId other) { 66 | return COMPARATOR.compare(this, other); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /api/src/main/java/com/github/bsideup/liiklus/positions/PositionsStorage.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.positions; 2 | 3 | import lombok.Value; 4 | import org.reactivestreams.Publisher; 5 | 6 | import java.util.Map; 7 | import java.util.concurrent.CompletionStage; 8 | 9 | public interface PositionsStorage { 10 | 11 | CompletionStage update(String topic, GroupId groupId, int partition, long position); 12 | 13 | Publisher findAll(); 14 | 15 | CompletionStage> findAll(String topic, GroupId groupId); 16 | 17 | CompletionStage>> findAllVersionsByGroup(String topic, String groupName); 18 | 19 | @Value 20 | class Positions { 21 | 22 | String topic; 23 | 24 | GroupId groupId; 25 | 26 | Map values; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /api/src/main/java/com/github/bsideup/liiklus/records/FiniteRecordsStorage.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.records; 2 | 3 | import java.util.Map; 4 | import java.util.concurrent.CompletionStage; 5 | 6 | public interface FiniteRecordsStorage extends RecordsStorage { 7 | 8 | /** 9 | * Returns a {@link Map} where key is partition's number and value is the latest offset. 10 | * The offset can be zero. Offset -1 means that there is no offset for this partition. 11 | */ 12 | CompletionStage> getEndOffsets(String topic); 13 | } -------------------------------------------------------------------------------- /api/src/main/java/com/github/bsideup/liiklus/records/KeyValueExtension.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.records; 2 | 3 | import io.cloudevents.extensions.ExtensionFormat; 4 | import io.cloudevents.extensions.InMemoryFormat; 5 | import lombok.RequiredArgsConstructor; 6 | import lombok.Value; 7 | 8 | import java.util.Collections; 9 | import java.util.Map; 10 | 11 | @Value 12 | @RequiredArgsConstructor(staticName = "of") 13 | public class KeyValueExtension implements ExtensionFormat, InMemoryFormat { 14 | 15 | String key; 16 | 17 | String value; 18 | 19 | @Override 20 | public InMemoryFormat memory() { 21 | return this; 22 | } 23 | 24 | @Override 25 | public Map transport() { 26 | return Collections.singletonMap(key, value); 27 | } 28 | 29 | @Override 30 | public Class getValueType() { 31 | return String.class; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /api/src/main/java/com/github/bsideup/liiklus/records/LiiklusAttributes.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.records; 2 | 3 | import io.cloudevents.Attributes; 4 | 5 | import java.time.ZonedDateTime; 6 | 7 | public interface LiiklusAttributes extends Attributes { 8 | 9 | ZonedDateTime getTime(); 10 | 11 | } 12 | -------------------------------------------------------------------------------- /api/src/main/java/com/github/bsideup/liiklus/records/RecordPostProcessor.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.records; 2 | 3 | import com.github.bsideup.liiklus.records.RecordsStorage.Record; 4 | import org.reactivestreams.Publisher; 5 | 6 | public interface RecordPostProcessor { 7 | 8 | Publisher postProcess(Publisher records); 9 | } 10 | -------------------------------------------------------------------------------- /api/src/main/java/com/github/bsideup/liiklus/records/RecordPreProcessor.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.records; 2 | 3 | import com.github.bsideup.liiklus.records.RecordsStorage.Envelope; 4 | 5 | import java.util.concurrent.CompletionStage; 6 | 7 | public interface RecordPreProcessor { 8 | 9 | CompletionStage preProcess(Envelope envelope); 10 | } 11 | -------------------------------------------------------------------------------- /api/src/test/java/com/github/bsideup/liiklus/positions/GroupIdTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.positions; 2 | 3 | import lombok.RequiredArgsConstructor; 4 | import org.junit.jupiter.params.ParameterizedTest; 5 | import org.junit.jupiter.params.provider.MethodSource; 6 | 7 | import java.util.Arrays; 8 | import java.util.Collection; 9 | import java.util.Optional; 10 | 11 | import static org.assertj.core.api.Assertions.assertThat; 12 | 13 | @RequiredArgsConstructor 14 | class GroupIdTest { 15 | 16 | static Collection data() { 17 | return Arrays.asList(new Object[][]{ 18 | {"hello", GroupId.of("hello", Optional.empty())}, 19 | {"hello-", GroupId.of("hello-", Optional.empty())}, 20 | {"hello-v", GroupId.of("hello-v", Optional.empty())}, 21 | {"hello-v-v", GroupId.of("hello-v-v", Optional.empty())}, 22 | 23 | {"hello-v1", GroupId.of("hello", Optional.of(1))}, 24 | {"hello-v100", GroupId.of("hello", Optional.of(100))}, 25 | 26 | {"hello-v10-v5", GroupId.of("hello-v10", Optional.of(5))}, 27 | 28 | {"hello-v-1", GroupId.of("hello-v-1", Optional.empty())}, 29 | {"hello-v10-alpha", GroupId.of("hello-v10-alpha", Optional.empty())}, 30 | }); 31 | } 32 | 33 | @ParameterizedTest 34 | @MethodSource("data") 35 | void testParsing(String string, GroupId object) { 36 | assertThat(GroupId.ofString(string)).isEqualTo(object); 37 | } 38 | 39 | @ParameterizedTest 40 | @MethodSource("data") 41 | void testStringRepresentation(String string, GroupId object) { 42 | assertThat(object.asString()).isEqualTo(string); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /api/src/test/java/com/github/bsideup/liiklus/positions/GroupIdValidationTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.positions; 2 | 3 | import org.junit.jupiter.api.Test; 4 | 5 | import java.util.Optional; 6 | 7 | import static org.assertj.core.api.Assertions.assertThatThrownBy; 8 | 9 | public class GroupIdValidationTest { 10 | 11 | @Test 12 | void testWrongExplicitVersion() { 13 | assertThatThrownBy( 14 | () -> GroupId.of("test", -1) 15 | ).isInstanceOf(IllegalArgumentException.class); 16 | } 17 | 18 | @Test 19 | void testWrongExplicitOptionalVersion() { 20 | assertThatThrownBy( 21 | () -> GroupId.of("test", Optional.of(-1)) 22 | ).isInstanceOf(IllegalArgumentException.class); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /api/src/test/java/com/github/bsideup/liiklus/records/LiiklusCloudEventTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.records; 2 | 3 | import org.assertj.core.data.MapEntry; 4 | import org.junit.jupiter.api.Nested; 5 | import org.junit.jupiter.api.Test; 6 | 7 | import java.util.Map; 8 | import java.util.UUID; 9 | import java.util.stream.Collectors; 10 | import java.util.stream.Stream; 11 | 12 | import static org.assertj.core.api.Assertions.assertThat; 13 | import static org.assertj.core.api.Assertions.entry; 14 | 15 | /** 16 | * See Kafka Protocol Binding 17 | */ 18 | class LiiklusCloudEventTest { 19 | 20 | @Nested 21 | class Attributes { 22 | @Test 23 | void serialization() { 24 | LiiklusCloudEvent event = newBuilder() 25 | .mediaType("text/plain") 26 | .build(); 27 | 28 | assertThat(event.getHeaders()).containsOnly( 29 | entry("ce_specversion", "1.0"), 30 | entry("ce_id", event.getId()), 31 | entry("ce_type", event.getType()), 32 | entry("ce_source", event.getRawSource()), 33 | entry("ce_datacontenttype", "text/plain") 34 | ); 35 | } 36 | 37 | @Test 38 | void deserialization() { 39 | String id = UUID.randomUUID().toString(); 40 | Map headers = Stream.of( 41 | entry("ce_specversion", "1.0"), 42 | entry("ce_id", id), 43 | entry("ce_type", "com.example.event"), 44 | entry("ce_source", "/tests"), 45 | entry("ce_datacontenttype", "text/plain") 46 | ).collect(Collectors.toMap(MapEntry::getKey, MapEntry::getValue)); 47 | 48 | LiiklusCloudEvent event = LiiklusCloudEvent.of(null, headers); 49 | assertThat(event) 50 | .returns("1.0", LiiklusCloudEvent::getSpecversion) 51 | .returns(id, LiiklusCloudEvent::getId) 52 | .returns("com.example.event", LiiklusCloudEvent::getType) 53 | .returns("/tests", LiiklusCloudEvent::getRawSource) 54 | .returns("text/plain", LiiklusCloudEvent::getMediaTypeOrNull); 55 | } 56 | } 57 | 58 | @Nested 59 | class Extensions { 60 | 61 | @Test 62 | void serialization() { 63 | LiiklusCloudEvent event = newBuilder() 64 | .rawExtension("comexamplefoo", "foo") 65 | .rawExtension("comexamplebar", "bar") 66 | .build(); 67 | 68 | assertThat(event.getHeaders()).contains( 69 | entry("ce_comexamplefoo", "foo"), 70 | entry("ce_comexamplebar", "bar") 71 | ); 72 | } 73 | 74 | @Test 75 | void deserialization() { 76 | String id = UUID.randomUUID().toString(); 77 | Map headers = Stream.of( 78 | entry("ce_specversion", "1.0"), 79 | entry("ce_id", id), 80 | entry("ce_type", "com.example.event"), 81 | entry("ce_source", "/tests"), 82 | entry("ce_comexamplefoo", "foo"), 83 | entry("ce_comexamplebar", "bar") 84 | ).collect(Collectors.toMap(MapEntry::getKey, MapEntry::getValue)); 85 | 86 | LiiklusCloudEvent event = LiiklusCloudEvent.of(null, headers); 87 | assertThat(event.getRawExtensions()).containsOnly( 88 | entry("comexamplefoo", "foo"), 89 | entry("comexamplebar", "bar") 90 | ); 91 | } 92 | } 93 | 94 | private static LiiklusCloudEvent.LiiklusCloudEventBuilder newBuilder() { 95 | return LiiklusCloudEvent.builder() 96 | .id(UUID.randomUUID().toString()) 97 | .type("com.example.event") 98 | .rawSource("/example"); 99 | } 100 | } -------------------------------------------------------------------------------- /app/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | id 'org.springframework.boot' 4 | } 5 | 6 | java { 7 | withSourcesJar() 8 | } 9 | 10 | jar { 11 | enabled = true 12 | } 13 | 14 | bootJar { 15 | archiveClassifier = 'boot' 16 | archiveVersion = '' 17 | } 18 | 19 | dependencies { 20 | api project(":api") 21 | api project(":protocol") 22 | 23 | api 'org.springframework.boot:spring-boot-starter-webflux' 24 | api 'org.springframework.fu:spring-fu-jafu' 25 | 26 | api 'org.pf4j:pf4j' 27 | 28 | testImplementation 'org.springframework.boot:spring-boot-starter-test' 29 | testImplementation 'org.assertj:assertj-core' 30 | testImplementation 'org.awaitility:awaitility' 31 | testImplementation project(":client") 32 | } 33 | 34 | def plugins = rootProject.allprojects.findAll { it.projectDir.parentFile.name == "plugins" } 35 | 36 | tasks.test.dependsOn(plugins.collect { it.getTasksByName("jar", true) }) -------------------------------------------------------------------------------- /app/src/main/java/com/github/bsideup/liiklus/config/GatewayConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.config; 2 | 3 | import com.github.bsideup.liiklus.records.RecordPostProcessor; 4 | import com.github.bsideup.liiklus.records.RecordPreProcessor; 5 | import com.github.bsideup.liiklus.service.LiiklusService; 6 | import com.github.bsideup.liiklus.util.PropertiesUtil; 7 | import lombok.Data; 8 | import org.springframework.boot.context.properties.ConfigurationProperties; 9 | import org.springframework.context.ApplicationContextInitializer; 10 | import org.springframework.context.support.GenericApplicationContext; 11 | import org.springframework.core.env.Profiles; 12 | 13 | import java.util.Comparator; 14 | import java.util.HashMap; 15 | import java.util.Map; 16 | import java.util.stream.Collectors; 17 | 18 | public class GatewayConfiguration implements ApplicationContextInitializer { 19 | 20 | @Override 21 | public void initialize(GenericApplicationContext applicationContext) { 22 | var environment = applicationContext.getEnvironment(); 23 | 24 | if (!environment.acceptsProfiles(Profiles.of("gateway"))) { 25 | return; 26 | } 27 | 28 | var layersProperties = PropertiesUtil.bind(environment, new LayersProperties()); 29 | 30 | var comparator = Comparator 31 | .comparingInt(it -> layersProperties.getOrders().getOrDefault(it.getClass().getName(), 0)) 32 | .thenComparing(it -> it.getClass().getName()); 33 | 34 | applicationContext.registerBean(RecordPreProcessorChain.class, () -> new RecordPreProcessorChain( 35 | applicationContext.getBeansOfType(RecordPreProcessor.class).values().stream() 36 | .sorted(comparator) 37 | .collect(Collectors.toList()) 38 | )); 39 | 40 | applicationContext.registerBean(RecordPostProcessorChain.class, () -> new RecordPostProcessorChain( 41 | applicationContext.getBeansOfType(RecordPostProcessor.class).values().stream() 42 | .sorted(comparator.reversed()) 43 | .collect(Collectors.toList()) 44 | )); 45 | 46 | applicationContext.registerBean(LiiklusService.class); 47 | } 48 | 49 | @ConfigurationProperties("layers") 50 | @Data 51 | static class LayersProperties { 52 | 53 | Map orders = new HashMap<>(); 54 | 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /app/src/main/java/com/github/bsideup/liiklus/config/RecordPostProcessorChain.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.config; 2 | 3 | import com.github.bsideup.liiklus.records.RecordPostProcessor; 4 | import lombok.RequiredArgsConstructor; 5 | import lombok.experimental.FieldDefaults; 6 | 7 | import java.util.Collection; 8 | 9 | @RequiredArgsConstructor 10 | @FieldDefaults(makeFinal = true) 11 | public class RecordPostProcessorChain { 12 | 13 | Collection processors; 14 | 15 | public Iterable getAll() { 16 | return processors; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /app/src/main/java/com/github/bsideup/liiklus/config/RecordPreProcessorChain.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.config; 2 | 3 | import com.github.bsideup.liiklus.records.RecordPreProcessor; 4 | import lombok.RequiredArgsConstructor; 5 | import lombok.experimental.FieldDefaults; 6 | 7 | import java.util.Collection; 8 | 9 | @RequiredArgsConstructor 10 | @FieldDefaults(makeFinal = true) 11 | public class RecordPreProcessorChain { 12 | 13 | Collection processors; 14 | 15 | public Iterable getAll() { 16 | return processors; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /app/src/main/java/com/github/bsideup/liiklus/plugins/LiiklusExtensionFinder.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.plugins; 2 | 3 | import org.pf4j.PluginManager; 4 | import org.pf4j.ServiceProviderExtensionFinder; 5 | 6 | import java.util.Collections; 7 | import java.util.Map; 8 | import java.util.Set; 9 | import java.util.stream.Collectors; 10 | 11 | class LiiklusExtensionFinder extends ServiceProviderExtensionFinder { 12 | 13 | LiiklusExtensionFinder(PluginManager pluginManager) { 14 | super(pluginManager); 15 | } 16 | 17 | @Override 18 | public Map> readClasspathStorages() { 19 | // The app does not provide any extensions, 20 | // we can safely return an empty Map here to avoid an exception ('META-INF/services' not found) 21 | return Collections.emptyMap(); 22 | } 23 | 24 | @Override 25 | public Set findClassNames(String pluginId) { 26 | var pluginClassLoader = pluginId != null 27 | ? pluginManager.getPluginClassLoader(pluginId) 28 | : getClass().getClassLoader(); 29 | 30 | return super.findClassNames(pluginId) 31 | .stream() 32 | // We need to filter out extension definitions for classes that are not on classpath (e.g. javax CDI) 33 | .filter(it -> { 34 | try { 35 | // TODO avoid classloading 36 | pluginClassLoader.loadClass(it); 37 | return true; 38 | } catch (ClassNotFoundException | NoClassDefFoundError e) { 39 | return false; 40 | } 41 | }) 42 | .collect(Collectors.toSet()); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /app/src/main/java/com/github/bsideup/liiklus/plugins/LiiklusPluginLoader.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.plugins; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.pf4j.PluginClassLoader; 5 | import org.pf4j.PluginDescriptor; 6 | import org.pf4j.PluginLoader; 7 | import org.pf4j.PluginManager; 8 | import org.pf4j.util.FileUtils; 9 | 10 | import java.io.IOException; 11 | import java.nio.file.FileSystems; 12 | import java.nio.file.Files; 13 | import java.nio.file.Path; 14 | import java.nio.file.StandardCopyOption; 15 | 16 | @Slf4j 17 | public class LiiklusPluginLoader implements PluginLoader { 18 | 19 | private final PluginManager pluginManager; 20 | 21 | public LiiklusPluginLoader(PluginManager pluginManager) { 22 | this.pluginManager = pluginManager; 23 | } 24 | 25 | @Override 26 | public boolean isApplicable(Path pluginPath) { 27 | return Files.exists(pluginPath) && FileUtils.isJarFile(pluginPath); 28 | } 29 | 30 | @Override 31 | public ClassLoader loadPlugin(Path pluginPath, PluginDescriptor pluginDescriptor) { 32 | var pluginClassLoader = new PluginClassLoader(pluginManager, pluginDescriptor, Thread.currentThread().getContextClassLoader()); 33 | pluginClassLoader.addFile(pluginPath.toFile()); 34 | 35 | // TODO consider fat jars 36 | try (var jarFileSystem = FileSystems.newFileSystem(pluginPath, (ClassLoader) null)) { 37 | var libPath = jarFileSystem.getPath("lib"); 38 | if (Files.exists(libPath)) { 39 | try (var pathStream = Files.walk(libPath, 1)) { 40 | pathStream.filter(Files::isRegularFile).forEach(it -> { 41 | try { 42 | var tempFile = Files.createTempFile(it.getFileName().toString(), ".jar"); 43 | Files.copy(it, tempFile, StandardCopyOption.REPLACE_EXISTING); 44 | pluginClassLoader.addURL(tempFile.toUri().toURL()); 45 | } catch (Exception e) { 46 | log.error("Failed to add file from {}", it.toAbsolutePath(), e); 47 | } 48 | }); 49 | } 50 | } 51 | } catch (IOException e) { 52 | throw new RuntimeException("Failed to load JARs from " + pluginPath.toAbsolutePath(), e); 53 | } 54 | 55 | return pluginClassLoader; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /app/src/main/java/com/github/bsideup/liiklus/plugins/LiiklusPluginManager.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.plugins; 2 | 3 | import lombok.Getter; 4 | import lombok.NonNull; 5 | import lombok.experimental.Delegate; 6 | import org.pf4j.*; 7 | 8 | import java.nio.file.Path; 9 | 10 | public class LiiklusPluginManager extends DefaultPluginManager { 11 | 12 | @Getter 13 | final String pluginsPathMatcher; 14 | 15 | public LiiklusPluginManager(@NonNull Path pluginsRoot, @NonNull String pluginsPathMatcher) { 16 | super(pluginsRoot); 17 | this.pluginsPathMatcher = pluginsPathMatcher; 18 | } 19 | 20 | @Override 21 | protected VersionManager createVersionManager() { 22 | var versionManager = super.createVersionManager(); 23 | 24 | class DelegatingVersionManager implements VersionManager { 25 | @Delegate 26 | final VersionManager delegate = versionManager; 27 | } 28 | 29 | return new DelegatingVersionManager() { 30 | @Override 31 | public boolean checkVersionConstraint(String version, String constraint) { 32 | // TODO https://github.com/pf4j/pf4j/issues/367 33 | return "*".equals(constraint) || super.checkVersionConstraint(version, constraint); 34 | } 35 | }; 36 | } 37 | 38 | @Override 39 | protected PluginDescriptorFinder createPluginDescriptorFinder() { 40 | return new ManifestPluginDescriptorFinder(); 41 | } 42 | 43 | @Override 44 | protected PluginRepository createPluginRepository() { 45 | return new LiiklusPluginRepository(this); 46 | } 47 | 48 | @Override 49 | protected ExtensionFinder createExtensionFinder() { 50 | return new LiiklusExtensionFinder(this); 51 | } 52 | 53 | @Override 54 | protected PluginLoader createPluginLoader() { 55 | return new LiiklusPluginLoader(this); 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /app/src/main/java/com/github/bsideup/liiklus/plugins/LiiklusPluginRepository.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.plugins; 2 | 3 | import lombok.RequiredArgsConstructor; 4 | import lombok.SneakyThrows; 5 | import org.pf4j.PluginRepository; 6 | import org.springframework.core.io.support.PathMatchingResourcePatternResolver; 7 | 8 | import java.io.IOException; 9 | import java.nio.file.Path; 10 | import java.util.List; 11 | import java.util.stream.Collectors; 12 | import java.util.stream.Stream; 13 | 14 | @RequiredArgsConstructor 15 | public class LiiklusPluginRepository implements PluginRepository { 16 | 17 | final LiiklusPluginManager liiklusPluginManager; 18 | 19 | @Override 20 | @SneakyThrows 21 | public List getPluginPaths() { 22 | var pluginsRoot = liiklusPluginManager.getPluginsRoot(); 23 | var pathMatcher = liiklusPluginManager.getPluginsPathMatcher(); 24 | 25 | var locationPattern = "file:" + pluginsRoot.resolve(pathMatcher).toString(); 26 | return Stream.of(new PathMatchingResourcePatternResolver().getResources(locationPattern)) 27 | .map(it -> { 28 | try { 29 | return it.getFile().toPath(); 30 | } catch (IOException e) { 31 | throw new RuntimeException(e); 32 | } 33 | }) 34 | .collect(Collectors.toList()); 35 | } 36 | 37 | @Override 38 | public boolean deletePluginPath(Path pluginPath) { 39 | // TODO 40 | return false; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /app/src/main/java/com/github/bsideup/liiklus/util/PropertiesUtil.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.util; 2 | 3 | import lombok.NonNull; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | import org.springframework.boot.context.properties.bind.Bindable; 6 | import org.springframework.boot.context.properties.bind.Binder; 7 | import org.springframework.boot.context.properties.bind.validation.ValidationBindHandler; 8 | import org.springframework.core.annotation.AnnotationUtils; 9 | import org.springframework.core.env.ConfigurableEnvironment; 10 | import org.springframework.validation.beanvalidation.SpringValidatorAdapter; 11 | 12 | import javax.validation.Validation; 13 | 14 | public class PropertiesUtil { 15 | 16 | public static T bind(ConfigurableEnvironment environment, @NonNull T properties) { 17 | var configurationProperties = AnnotationUtils.findAnnotation(properties.getClass(), ConfigurationProperties.class); 18 | 19 | if (configurationProperties == null) { 20 | throw new IllegalArgumentException(properties.getClass() + " Must be annotated with @ConfigurationProperties"); 21 | } 22 | 23 | var property = configurationProperties.prefix(); 24 | 25 | var validationBindHandler = new ValidationBindHandler( 26 | new SpringValidatorAdapter(Validation.buildDefaultValidatorFactory().getValidator()) 27 | ); 28 | 29 | var bindable = Bindable.ofInstance(properties); 30 | return Binder.get(environment).bind(property, bindable, validationBindHandler).orElseGet(bindable.getValue()); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /app/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | profiles: 3 | active: exporter,gateway 4 | 5 | # Liiklus config defaults 6 | grpc: 7 | port: 6565 8 | 9 | rsocket: 10 | host: 0.0.0.0 11 | port: 8081 12 | 13 | plugins: 14 | dir: ./plugins 15 | pathMatcher: "*.jar" -------------------------------------------------------------------------------- /app/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /app/src/test/java/com/github/bsideup/liiklus/ConsumerGroupsTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus; 2 | 3 | import com.github.bsideup.liiklus.protocol.PublishRequest; 4 | import com.github.bsideup.liiklus.protocol.SubscribeRequest; 5 | import com.github.bsideup.liiklus.test.AbstractIntegrationTest; 6 | import com.google.protobuf.ByteString; 7 | import org.junit.jupiter.api.BeforeEach; 8 | import org.junit.jupiter.api.Test; 9 | import org.junit.jupiter.api.TestInfo; 10 | import reactor.core.publisher.Flux; 11 | 12 | import java.lang.reflect.Method; 13 | import java.time.Duration; 14 | import java.util.HashMap; 15 | import java.util.HashSet; 16 | import java.util.Set; 17 | 18 | class ConsumerGroupsTest extends AbstractIntegrationTest { 19 | 20 | SubscribeRequest subscribeRequest; 21 | 22 | @BeforeEach 23 | void setUpConsumerGroupsTest(TestInfo info) throws Exception { 24 | subscribeRequest = SubscribeRequest.newBuilder() 25 | .setTopic(info.getTestMethod().map(Method::getName).orElse("unknown")) 26 | .setGroup(info.getTestMethod().map(Method::getName).orElse("unknown")) 27 | .setAutoOffsetReset(SubscribeRequest.AutoOffsetReset.EARLIEST) 28 | .build(); 29 | 30 | // Will create a topic and initialize every partition 31 | Flux.fromIterable(PARTITION_UNIQUE_KEYS) 32 | .flatMap(key -> stub 33 | .publish( 34 | PublishRequest.newBuilder() 35 | .setTopic(subscribeRequest.getTopic()) 36 | .setKey(ByteString.copyFromUtf8(key)) 37 | .setLiiklusEvent(LIIKLUS_EVENT_EXAMPLE) 38 | .build() 39 | ) 40 | ) 41 | .blockLast(); 42 | } 43 | 44 | @Test 45 | void testConsumerGroups() { 46 | Flux 47 | .merge( 48 | stub.subscribe(subscribeRequest), 49 | stub.subscribe(subscribeRequest) 50 | ) 51 | .scanWith( 52 | () -> new HashMap>(), 53 | (acc, it) -> { 54 | acc.computeIfAbsent(it.getAssignment().getSessionId(), __ -> new HashSet<>()).add(it.getAssignment().getPartition()); 55 | return acc; 56 | } 57 | ) 58 | .filter(it -> it.size() == 2 && it.values().stream().noneMatch(Set::isEmpty)) 59 | .blockFirst(Duration.ofSeconds(30)); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /app/src/test/java/com/github/bsideup/liiklus/EndOffsetsTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus; 2 | 3 | import com.github.bsideup.liiklus.protocol.GetEndOffsetsRequest; 4 | import com.github.bsideup.liiklus.protocol.PublishRequest; 5 | import com.github.bsideup.liiklus.test.AbstractIntegrationTest; 6 | import com.google.protobuf.ByteString; 7 | import org.junit.jupiter.api.BeforeEach; 8 | import org.junit.jupiter.api.Test; 9 | import org.junit.jupiter.api.TestInfo; 10 | 11 | import java.lang.reflect.Method; 12 | import java.util.UUID; 13 | 14 | import static org.assertj.core.api.Assertions.assertThat; 15 | 16 | class EndOffsetsTest extends AbstractIntegrationTest { 17 | 18 | private String topic; 19 | 20 | @BeforeEach 21 | final void setUpEndOffsetsTest(TestInfo info) { 22 | topic = info.getTestMethod().map(Method::getName).orElse("unknown"); 23 | } 24 | 25 | @Test 26 | void testEndOffsets() { 27 | for (int partition = 0; partition < NUM_PARTITIONS; partition++) { 28 | for (int i = 0; i < partition + 1; i++) { 29 | stub.publish(PublishRequest.newBuilder() 30 | .setTopic(topic) 31 | .setKey(ByteString.copyFromUtf8(PARTITION_KEYS.get(partition))) 32 | .setLiiklusEvent(LIIKLUS_EVENT_EXAMPLE) 33 | .build() 34 | ).block(); 35 | } 36 | } 37 | 38 | var reply = stub.getEndOffsets(GetEndOffsetsRequest.newBuilder().setTopic(topic).build()).block(); 39 | 40 | assertThat(reply.getOffsetsMap()) 41 | .hasSize(NUM_PARTITIONS) 42 | .allSatisfy((partition, offset) -> { 43 | assertThat(offset) 44 | .as("offset of p" + partition) 45 | .isEqualTo(partition.longValue()); 46 | }); 47 | } 48 | 49 | @Test 50 | void testEndOffsets_unknownTopic() { 51 | var randomTopic = UUID.randomUUID().toString(); 52 | var reply = stub.getEndOffsets(GetEndOffsetsRequest.newBuilder().setTopic(randomTopic).build()).block(); 53 | 54 | assertThat(reply.getOffsetsMap()).isEmpty(); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /app/src/test/java/com/github/bsideup/liiklus/ProfilesTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus; 2 | 3 | import com.github.bsideup.liiklus.test.AbstractIntegrationTest; 4 | import com.google.common.collect.Sets; 5 | import org.assertj.core.api.AbstractThrowableAssert; 6 | import org.junit.jupiter.api.AfterEach; 7 | import org.junit.jupiter.api.Test; 8 | import org.springframework.context.ConfigurableApplicationContext; 9 | 10 | import java.util.Collection; 11 | import java.util.Set; 12 | import java.util.stream.Stream; 13 | 14 | import static org.assertj.core.api.Assertions.assertThatCode; 15 | 16 | class ProfilesTest extends AbstractIntegrationTest { 17 | 18 | static Set RECORDS_PROPERTIES = Sets.newHashSet( 19 | "storage.records.type=MEMORY" 20 | ); 21 | 22 | static Set POSITIONS_PROPERTIES = Sets.newHashSet( 23 | "storage.positions.type=MEMORY" 24 | ); 25 | 26 | Set commonArgs = Sets.newHashSet(Set.of( 27 | "server.port=0" 28 | )); 29 | 30 | ConfigurableApplicationContext lastApplicationContext; 31 | 32 | @AfterEach 33 | void tearDown() throws Exception { 34 | if (lastApplicationContext != null) { 35 | lastApplicationContext.close(); 36 | } 37 | } 38 | 39 | @Test 40 | void testRequired() throws Exception { 41 | assertThatAppWithProps(commonArgs) 42 | .hasMessageContaining("Required key"); 43 | 44 | assertThatAppWithProps(commonArgs, RECORDS_PROPERTIES) 45 | .hasMessageContaining("Required key"); 46 | 47 | assertThatAppWithProps(commonArgs, POSITIONS_PROPERTIES) 48 | .hasMessageContaining("Required key"); 49 | 50 | assertThatAppWithProps(commonArgs, RECORDS_PROPERTIES, POSITIONS_PROPERTIES) 51 | .doesNotThrowAnyException(); 52 | } 53 | 54 | @Test 55 | void testExporterProfile() throws Exception { 56 | commonArgs.add("spring.profiles.active=exporter"); 57 | 58 | assertThatAppWithProps(commonArgs) 59 | .hasMessageContaining("Required key"); 60 | 61 | assertThatAppWithProps(commonArgs, POSITIONS_PROPERTIES) 62 | .doesNotThrowAnyException(); 63 | } 64 | 65 | @Test 66 | void testGatewayProfile() throws Exception { 67 | commonArgs.add("spring.profiles.active=gateway"); 68 | 69 | assertThatAppWithProps(commonArgs) 70 | .hasMessageContaining("Required key"); 71 | 72 | assertThatAppWithProps(commonArgs, RECORDS_PROPERTIES) 73 | .hasMessageContaining("Required key"); 74 | 75 | assertThatAppWithProps(commonArgs, POSITIONS_PROPERTIES) 76 | .hasMessageContaining("Required key"); 77 | 78 | assertThatAppWithProps(commonArgs, RECORDS_PROPERTIES, POSITIONS_PROPERTIES) 79 | .doesNotThrowAnyException(); 80 | } 81 | 82 | @SafeVarargs 83 | protected final AbstractThrowableAssert assertThatAppWithProps(Set... props) { 84 | if (lastApplicationContext != null) { 85 | lastApplicationContext.close(); 86 | } 87 | 88 | return assertThatCode(() -> { 89 | var args = Stream.of(props) 90 | .flatMap(Collection::stream) 91 | .map(it -> "--" + it) 92 | .toArray(String[]::new); 93 | 94 | lastApplicationContext = Application.start(args); 95 | }); 96 | } 97 | 98 | } 99 | -------------------------------------------------------------------------------- /app/src/test/java/com/github/bsideup/liiklus/test/ProcessorPluginMock.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.test; 2 | 3 | import com.github.bsideup.liiklus.records.RecordPostProcessor; 4 | import com.github.bsideup.liiklus.records.RecordPreProcessor; 5 | import com.github.bsideup.liiklus.records.RecordsStorage; 6 | import lombok.Getter; 7 | import org.reactivestreams.Publisher; 8 | 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | import java.util.concurrent.CompletableFuture; 12 | import java.util.concurrent.CompletionStage; 13 | 14 | public class ProcessorPluginMock implements RecordPreProcessor, RecordPostProcessor { 15 | 16 | @Getter 17 | List preProcessors = new ArrayList<>(); 18 | 19 | @Getter 20 | List postProcessors = new ArrayList<>(); 21 | 22 | public ProcessorPluginMock() { 23 | } 24 | 25 | @Override 26 | public CompletionStage preProcess(RecordsStorage.Envelope envelope) { 27 | var future = CompletableFuture.completedFuture(envelope); 28 | for (RecordPreProcessor preProcessor : preProcessors) { 29 | future = future.thenComposeAsync(preProcessor::preProcess); 30 | } 31 | return future; 32 | } 33 | 34 | @Override 35 | public Publisher postProcess(Publisher records) { 36 | for (RecordPostProcessor postProcessor : postProcessors) { 37 | records = postProcessor.postProcess(records); 38 | } 39 | return records; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /client/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'idea' 3 | id 'java-library' 4 | id 'com.google.protobuf' 5 | } 6 | 7 | protobuf { 8 | protoc { 9 | artifact = 'com.google.protobuf:protoc' 10 | } 11 | 12 | generatedFilesBaseDir = "$projectDir/generated" 13 | 14 | plugins { 15 | grpc { 16 | artifact = 'io.grpc:protoc-gen-grpc-java' 17 | } 18 | 19 | reactor { 20 | artifact = 'com.salesforce.servicelibs:reactor-grpc:0.9.0:jdk8@jar' 21 | } 22 | 23 | rsocketRpc { 24 | artifact = 'io.rsocket.rpc:rsocket-rpc-protobuf' 25 | } 26 | } 27 | 28 | generateProtoTasks { 29 | ofSourceSet('main').each { task -> 30 | task.builtins { 31 | remove java 32 | } 33 | task.plugins { 34 | grpc { } 35 | reactor { } 36 | rsocketRpc { } 37 | } 38 | } 39 | } 40 | } 41 | 42 | clean { 43 | delete protobuf.generatedFilesBaseDir 44 | } 45 | 46 | java { 47 | withSourcesJar() 48 | } 49 | 50 | idea { 51 | module { 52 | generatedSourceDirs += file("${protobuf.generatedFilesBaseDir}/main/java") 53 | generatedSourceDirs += file("${protobuf.generatedFilesBaseDir}/main/reactor") 54 | generatedSourceDirs += file("${protobuf.generatedFilesBaseDir}/main/grpc") 55 | generatedSourceDirs += file("${protobuf.generatedFilesBaseDir}/main/rsocketRpc") 56 | } 57 | } 58 | 59 | sourceCompatibility = targetCompatibility = 8 60 | 61 | dependencies { 62 | protobuf project(":protocol") 63 | api project(":protocol") 64 | 65 | api 'io.grpc:grpc-stub' 66 | api 'io.grpc:grpc-protobuf' 67 | api 'com.salesforce.servicelibs:reactor-grpc-stub' 68 | api 'io.rsocket.rpc:rsocket-rpc-core' 69 | api 'io.rsocket:rsocket-transport-netty' 70 | api 'io.grpc:grpc-netty' 71 | api 'com.google.protobuf:protobuf-java-util' 72 | } -------------------------------------------------------------------------------- /client/src/main/java/com/github/bsideup/liiklus/GRPCLiiklusClient.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus; 2 | 3 | import com.github.bsideup.liiklus.protocol.ReactorLiiklusServiceGrpc; 4 | import com.github.bsideup.liiklus.protocol.ReactorLiiklusServiceGrpc.ReactorLiiklusServiceStub; 5 | import io.grpc.Channel; 6 | import lombok.AccessLevel; 7 | import lombok.RequiredArgsConstructor; 8 | import lombok.experimental.Delegate; 9 | import lombok.experimental.FieldDefaults; 10 | 11 | @RequiredArgsConstructor 12 | @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) 13 | public class GRPCLiiklusClient implements LiiklusClient { 14 | 15 | @Delegate 16 | ReactorLiiklusServiceStub stub; 17 | 18 | public GRPCLiiklusClient(Channel channel) { 19 | this(ReactorLiiklusServiceGrpc.newReactorStub(channel)); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /client/src/main/java/com/github/bsideup/liiklus/LiiklusClient.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus; 2 | 3 | import com.github.bsideup.liiklus.protocol.*; 4 | import com.google.protobuf.Empty; 5 | import reactor.core.publisher.Flux; 6 | import reactor.core.publisher.Mono; 7 | 8 | public interface LiiklusClient { 9 | 10 | Mono publish(PublishRequest message); 11 | 12 | Flux subscribe(SubscribeRequest message); 13 | 14 | Flux receive(ReceiveRequest message); 15 | 16 | Mono ack(AckRequest message); 17 | 18 | Mono getOffsets(GetOffsetsRequest message); 19 | 20 | Mono getEndOffsets(GetEndOffsetsRequest message); 21 | } 22 | -------------------------------------------------------------------------------- /client/src/main/java/com/github/bsideup/liiklus/RSocketLiiklusClient.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus; 2 | 3 | import com.github.bsideup.liiklus.protocol.*; 4 | import com.google.protobuf.Empty; 5 | import io.rsocket.RSocket; 6 | import lombok.AccessLevel; 7 | import lombok.RequiredArgsConstructor; 8 | import lombok.experimental.FieldDefaults; 9 | import reactor.core.publisher.Flux; 10 | import reactor.core.publisher.Mono; 11 | 12 | @RequiredArgsConstructor 13 | @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) 14 | public class RSocketLiiklusClient implements LiiklusClient { 15 | 16 | LiiklusServiceClient liiklusServiceClient; 17 | 18 | public RSocketLiiklusClient(RSocket rSocket) { 19 | this(new LiiklusServiceClient(rSocket)); 20 | } 21 | 22 | @Override 23 | public Mono publish(PublishRequest message) { 24 | return liiklusServiceClient.publish(message); 25 | } 26 | 27 | @Override 28 | public Flux subscribe(SubscribeRequest message) { 29 | return liiklusServiceClient.subscribe(message); 30 | } 31 | 32 | @Override 33 | public Flux receive(ReceiveRequest message) { 34 | return liiklusServiceClient.receive(message); 35 | } 36 | 37 | @Override 38 | public Mono ack(AckRequest message) { 39 | return liiklusServiceClient.ack(message); 40 | } 41 | 42 | @Override 43 | public Mono getOffsets(GetOffsetsRequest message) { 44 | return liiklusServiceClient.getOffsets(message); 45 | } 46 | 47 | @Override 48 | public Mono getEndOffsets(GetEndOffsetsRequest message) { 49 | return liiklusServiceClient.getEndOffsets(message); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /examples/java/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | id 'idea' 4 | id 'com.google.protobuf' 5 | id 'io.franzbecker.gradle-lombok' 6 | } 7 | 8 | protobuf { 9 | protoc { 10 | artifact = 'com.google.protobuf:protoc:3.10.1' 11 | } 12 | 13 | generatedFilesBaseDir = "$projectDir/generated" 14 | 15 | plugins { 16 | grpc { 17 | artifact = 'io.grpc:protoc-gen-grpc-java:1.24.1' 18 | } 19 | 20 | reactor { 21 | artifact = "com.salesforce.servicelibs:reactor-grpc:0.8.2:jdk8@jar" 22 | } 23 | } 24 | 25 | generateProtoTasks { 26 | ofSourceSet('main')*.plugins { 27 | grpc { } 28 | reactor { } 29 | } 30 | } 31 | } 32 | 33 | clean { 34 | delete protobuf.generatedFilesBaseDir 35 | } 36 | 37 | idea { 38 | module { 39 | generatedSourceDirs += file("${protobuf.generatedFilesBaseDir}/main/java") 40 | generatedSourceDirs += file("${protobuf.generatedFilesBaseDir}/main/reactor") 41 | generatedSourceDirs += file("${protobuf.generatedFilesBaseDir}/main/grpc") 42 | } 43 | } 44 | 45 | repositories { 46 | mavenCentral() 47 | } 48 | 49 | dependencies { 50 | implementation 'org.testcontainers:kafka:1.15.3' 51 | 52 | implementation 'org.apache.commons:commons-math3:3.6.1' 53 | 54 | implementation 'com.google.protobuf:protobuf-java:3.10.0' 55 | 56 | implementation 'io.grpc:grpc-netty:1.24.1' 57 | implementation 'io.grpc:grpc-protobuf:1.24.1' 58 | implementation 'com.salesforce.servicelibs:reactor-grpc-stub:0.10.0' 59 | 60 | runtimeOnly 'ch.qos.logback:logback-classic:1.2.4-groovyless' 61 | } -------------------------------------------------------------------------------- /examples/java/src/main/proto: -------------------------------------------------------------------------------- 1 | ../../../../protocol/src/main/proto/ -------------------------------------------------------------------------------- /examples/java/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{HH:mm:ss.SSS} %-5level %36(%logger{36}) - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /examples/plugin/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | id 'io.spring.dependency-management' 4 | id 'io.franzbecker.gradle-lombok' 5 | } 6 | 7 | sourceCompatibility = targetCompatibility = 8 8 | 9 | jar { 10 | archiveBaseName = "example-plugin" 11 | manifest { 12 | attributes( 13 | 'Plugin-Id': "example-plugin", 14 | 'Plugin-Version': "1.0.0", 15 | ) 16 | } 17 | 18 | into('lib') { 19 | from configurations.runtimeClasspath - configurations.compileOnlyClasspath 20 | } 21 | } 22 | 23 | configurations { 24 | compileOnlyClasspath { 25 | extendsFrom compileOnly 26 | canBeConsumed false 27 | canBeResolved true 28 | } 29 | } 30 | 31 | processTestResources.dependsOn(jar) 32 | test.dependsOn(jar) 33 | 34 | sourceSets { 35 | test { 36 | resources.srcDir jar.outputs.files.singleFile.parentFile 37 | } 38 | } 39 | 40 | repositories { 41 | mavenCentral() 42 | maven { url 'https://jitpack.io' } 43 | } 44 | 45 | dependencyManagement { 46 | overriddenByDependencies = false 47 | 48 | imports { 49 | mavenBom org.springframework.boot.gradle.plugin.SpringBootPlugin.BOM_COORDINATES 50 | mavenBom 'org.testcontainers:testcontainers-bom:1.16.0' 51 | } 52 | 53 | dependencies { 54 | dependencySet(group: "com.github.bsideup.liiklus", version: "0.9.0") { 55 | entry "api" 56 | entry "client" 57 | entry "testing" 58 | } 59 | 60 | dependency 'com.google.auto.service:auto-service:1.0' 61 | } 62 | } 63 | 64 | dependencies { 65 | compileOnly "com.google.auto.service:auto-service" 66 | annotationProcessor "com.google.auto.service:auto-service" 67 | 68 | compileOnly "com.github.bsideup.liiklus:api" 69 | compileOnly 'org.springframework.boot:spring-boot-starter' 70 | compileOnly 'io.projectreactor:reactor-core' 71 | 72 | implementation 'org.apache.commons:commons-lang3:3.9' 73 | 74 | testImplementation 'org.testcontainers:kafka' 75 | testImplementation 'ch.qos.logback:logback-classic:' 76 | testImplementation "com.github.bsideup.liiklus:client" 77 | testImplementation "com.github.bsideup.liiklus:testing" 78 | testImplementation 'org.assertj:assertj-core' 79 | } 80 | -------------------------------------------------------------------------------- /examples/plugin/src/main/java/com/github/bsideup/liiklus/plugins/example/ExampleRecordPostProcessor.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.plugins.example; 2 | 3 | import com.github.bsideup.liiklus.records.RecordPostProcessor; 4 | import com.github.bsideup.liiklus.records.RecordsStorage.Record; 5 | import org.reactivestreams.Publisher; 6 | import reactor.core.publisher.Flux; 7 | 8 | import java.nio.ByteBuffer; 9 | import java.nio.charset.StandardCharsets; 10 | 11 | public class ExampleRecordPostProcessor implements RecordPostProcessor { 12 | @Override 13 | public Publisher postProcess(Publisher publisher) { 14 | return Flux.from(publisher) 15 | .map(record -> { 16 | String key = StandardCharsets.UTF_8.decode(record.getEnvelope().getKey().duplicate()).toString(); 17 | if ("maskMe".equals(key)) { 18 | return new Record( 19 | record.getEnvelope().withValue(ByteBuffer.wrap("**masked**".getBytes())), 20 | record.getTimestamp(), 21 | record.getPartition(), 22 | record.getOffset() 23 | ); 24 | } else { 25 | return record; 26 | } 27 | }); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /examples/plugin/src/main/java/com/github/bsideup/liiklus/plugins/example/ExampleRecordPreProcessor.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.plugins.example; 2 | 3 | import com.github.bsideup.liiklus.records.RecordPreProcessor; 4 | import com.github.bsideup.liiklus.records.RecordsStorage.Envelope; 5 | import org.apache.commons.lang3.StringUtils; 6 | 7 | import java.nio.ByteBuffer; 8 | import java.nio.charset.StandardCharsets; 9 | import java.util.concurrent.CompletableFuture; 10 | import java.util.concurrent.CompletionStage; 11 | 12 | public class ExampleRecordPreProcessor implements RecordPreProcessor { 13 | 14 | @Override 15 | public CompletionStage preProcess(Envelope envelope) { 16 | String value = StandardCharsets.UTF_8.decode(envelope.getValue()).toString(); 17 | 18 | String reversed = StringUtils.reverse(value); 19 | return CompletableFuture.completedFuture( 20 | envelope.withValue(ByteBuffer.wrap(reversed.getBytes())) 21 | ); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /examples/plugin/src/main/java/com/github/bsideup/liiklus/plugins/example/config/ExamplePluginConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.plugins.example.config; 2 | 3 | import com.github.bsideup.liiklus.plugins.example.ExampleRecordPostProcessor; 4 | import com.github.bsideup.liiklus.plugins.example.ExampleRecordPreProcessor; 5 | import com.google.auto.service.AutoService; 6 | import org.springframework.context.ApplicationContextInitializer; 7 | import org.springframework.context.support.GenericApplicationContext; 8 | 9 | @AutoService(ApplicationContextInitializer.class) 10 | public class ExamplePluginConfiguration implements ApplicationContextInitializer { 11 | 12 | @Override 13 | public void initialize(GenericApplicationContext applicationContext) { 14 | applicationContext.registerBean(ExampleRecordPreProcessor.class); 15 | applicationContext.registerBean(ExampleRecordPostProcessor.class); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /examples/plugin/src/test/java/com/github/bsideup/liiklus/plugins/example/SmokeTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.plugins.example; 2 | 3 | import com.github.bsideup.liiklus.protocol.ReceiveReply.Record; 4 | import com.github.bsideup.liiklus.plugins.example.support.AbstractIntegrationTest; 5 | import org.junit.jupiter.api.Test; 6 | 7 | import java.time.Duration; 8 | import java.util.UUID; 9 | 10 | import static org.assertj.core.api.Assertions.assertThat; 11 | 12 | class SmokeTest extends AbstractIntegrationTest { 13 | 14 | @Test 15 | void testPreProcessor() { 16 | String key = UUID.randomUUID().toString(); 17 | 18 | publishRecord(key, "Hello!"); 19 | 20 | Record record = receiveRecords(key).blockFirst(Duration.ofSeconds(10)); 21 | 22 | assertThat(record).isNotNull().satisfies(it -> { 23 | assertThat(it.getValue().toStringUtf8()).isEqualTo("!olleH"); 24 | }); 25 | } 26 | 27 | @Test 28 | void testPostProcessor() { 29 | String key = "maskMe"; 30 | 31 | publishRecord(key, "Hello!"); 32 | 33 | Record record = receiveRecords(key).blockFirst(Duration.ofSeconds(10)); 34 | 35 | assertThat(record).isNotNull().satisfies(it -> { 36 | assertThat(it.getValue().toStringUtf8()).isEqualTo("**masked**"); 37 | }); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /examples/plugin/src/test/java/com/github/bsideup/liiklus/plugins/example/support/AbstractIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.plugins.example.support; 2 | 3 | import com.github.bsideup.liiklus.GRPCLiiklusClient; 4 | import com.github.bsideup.liiklus.LiiklusClient; 5 | import com.github.bsideup.liiklus.container.LiiklusContainer; 6 | import com.github.bsideup.liiklus.protocol.*; 7 | import com.github.bsideup.liiklus.protocol.ReceiveReply.Record; 8 | import com.google.protobuf.ByteString; 9 | import io.grpc.netty.NettyChannelBuilder; 10 | import org.testcontainers.containers.BindMode; 11 | import org.testcontainers.containers.output.OutputFrame; 12 | import org.testcontainers.containers.output.ToStringConsumer; 13 | import reactor.core.publisher.Flux; 14 | 15 | import java.time.Duration; 16 | import java.time.ZonedDateTime; 17 | import java.time.format.DateTimeFormatter; 18 | import java.util.UUID; 19 | 20 | public abstract class AbstractIntegrationTest { 21 | 22 | protected static final LiiklusClient client; 23 | 24 | static { 25 | LiiklusContainer liiklus = new LiiklusContainer("0.9.3") 26 | .withEnv("storage_records_type", "MEMORY") 27 | .withClasspathResourceMapping("/example-plugin.jar", "/app/plugins/example-plugin.jar", BindMode.READ_ONLY) 28 | .withLogConsumer(new ToStringConsumer() { 29 | @Override 30 | public void accept(OutputFrame outputFrame) { 31 | System.out.print("\uD83D\uDEA6 " + outputFrame.getUtf8String()); 32 | } 33 | }); 34 | 35 | liiklus.start(); 36 | 37 | client = new GRPCLiiklusClient( 38 | NettyChannelBuilder.forTarget(liiklus.getTarget()) 39 | .usePlaintext() 40 | .build() 41 | ); 42 | } 43 | 44 | protected String topic = "test-topic-" + UUID.randomUUID(); 45 | 46 | protected PublishReply publishRecord(String key, String value) { 47 | return client.publish(PublishRequest.newBuilder() 48 | .setTopic(topic) 49 | .setKey(ByteString.copyFromUtf8(key)) 50 | .setValue(ByteString.copyFromUtf8(value)) 51 | .build() 52 | ).block(Duration.ofSeconds(10)); 53 | } 54 | 55 | protected Flux receiveRecords(String key) { 56 | return client 57 | .subscribe(SubscribeRequest.newBuilder() 58 | .setTopic(topic) 59 | .setGroup(UUID.randomUUID().toString()) 60 | .setAutoOffsetReset(SubscribeRequest.AutoOffsetReset.EARLIEST) 61 | .build() 62 | ) 63 | .flatMap(it -> client.receive(ReceiveRequest.newBuilder().setAssignment(it.getAssignment()).build())) 64 | .map(ReceiveReply::getRecord) 65 | .filter(it -> key.equals(it.getKey().toStringUtf8())); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /examples/plugin/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | org.gradle.caching=true 2 | -------------------------------------------------------------------------------- /gradle/rerunTests.gradle: -------------------------------------------------------------------------------- 1 | project.afterEvaluate { 2 | tasks.withType(Test) { originalTask -> 3 | if (name.endsWith("Rerun")) { 4 | // Avoid recursion 5 | return 6 | } 7 | 8 | def rerunTask = tasks.register("${name}Rerun", Test) { 9 | // Enabled only when there are failures (see `afterTest`) 10 | enabled = false 11 | failFast = true // ¯\_(ツ)_/¯ 12 | outputs.upToDateWhen { false } 13 | 14 | beforeTest { desc -> logger.warn("Re-running ${desc.className}.${desc.name}") } 15 | 16 | useTestFramework(originalTask.getTestFramework()) 17 | classpath = originalTask.classpath 18 | testClassesDirs = originalTask.testClassesDirs 19 | 20 | testLogging { 21 | displayGranularity 1 22 | showStackTraces = true 23 | exceptionFormat = 'full' 24 | events "STARTED", "PASSED", "FAILED", "SKIPPED" 25 | } 26 | } 27 | 28 | ignoreFailures = true 29 | finalizedBy(rerunTask) 30 | 31 | afterTest { desc, result -> 32 | if (TestResult.ResultType.FAILURE == result.resultType) { 33 | rerunTask.configure { 34 | enabled = true 35 | filter.includeTestsMatching("${desc.className}.${desc.name}") 36 | } 37 | } 38 | } 39 | } 40 | } -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bsideup/liiklus/9a5bb00fa8c4d35e9985fe535a6dc84d41420f41/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-all.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto init 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto init 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :init 68 | @rem Get command-line arguments, handling Windows variants 69 | 70 | if not "%OS%" == "Windows_NT" goto win9xME_args 71 | 72 | :win9xME_args 73 | @rem Slurp the command line arguments. 74 | set CMD_LINE_ARGS= 75 | set _SKIP=2 76 | 77 | :win9xME_args_slurp 78 | if "x%~1" == "x" goto execute 79 | 80 | set CMD_LINE_ARGS=%* 81 | 82 | :execute 83 | @rem Setup the command line 84 | 85 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 86 | 87 | @rem Execute Gradle 88 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 89 | 90 | :end 91 | @rem End local scope for the variables with windows NT shell 92 | if "%ERRORLEVEL%"=="0" goto mainEnd 93 | 94 | :fail 95 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 96 | rem the _cmd.exe /c_ return code! 97 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 98 | exit /b 1 99 | 100 | :mainEnd 101 | if "%OS%"=="Windows_NT" endlocal 102 | 103 | :omega 104 | -------------------------------------------------------------------------------- /jitpack.yml: -------------------------------------------------------------------------------- 1 | jdk: 2 | - openjdk11 3 | -------------------------------------------------------------------------------- /plugins/dynamodb-positions-storage/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | } 4 | 5 | jar { 6 | manifest { 7 | attributes( 8 | 'Plugin-Id': "${project.name}", 9 | 'Plugin-Version': "${project.version}", 10 | ) 11 | } 12 | 13 | into('lib') { 14 | from(configurations.runtimeClasspath - configurations.compileOnlyClasspath) 15 | } 16 | } 17 | 18 | tasks.test.dependsOn( 19 | jar, 20 | rootProject.project(":plugins:inmemory-records-storage").getTasksByName("jar", false) 21 | ) 22 | 23 | dependencies { 24 | compileOnly 'com.google.auto.service:auto-service' 25 | annotationProcessor 'com.google.auto.service:auto-service' 26 | 27 | compileOnly project(":app") 28 | 29 | api 'software.amazon.awssdk:dynamodb:2.3.0' 30 | 31 | testImplementation project(":tck") 32 | testImplementation 'org.springframework.boot:spring-boot-test' 33 | testImplementation 'org.testcontainers:localstack' 34 | testImplementation 'com.amazonaws:aws-java-sdk-dynamodb:1.11.475' 35 | } 36 | -------------------------------------------------------------------------------- /plugins/dynamodb-positions-storage/src/main/java/com/github/bsideup/liiklus/dynamodb/config/DynamoDBConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.dynamodb.config; 2 | 3 | import com.github.bsideup.liiklus.dynamodb.DynamoDBPositionsStorage; 4 | import com.github.bsideup.liiklus.util.PropertiesUtil; 5 | import com.google.auto.service.AutoService; 6 | import lombok.Data; 7 | import lombok.extern.slf4j.Slf4j; 8 | import org.springframework.boot.context.properties.ConfigurationProperties; 9 | import org.springframework.context.ApplicationContextInitializer; 10 | import org.springframework.context.support.GenericApplicationContext; 11 | import org.springframework.validation.annotation.Validated; 12 | import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; 13 | import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition; 14 | import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest; 15 | import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement; 16 | import software.amazon.awssdk.services.dynamodb.model.KeyType; 17 | import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput; 18 | import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; 19 | 20 | import javax.validation.constraints.NotEmpty; 21 | import java.net.URI; 22 | import java.util.Optional; 23 | 24 | @Slf4j 25 | @AutoService(ApplicationContextInitializer.class) 26 | public class DynamoDBConfiguration implements ApplicationContextInitializer { 27 | 28 | @Override 29 | public void initialize(GenericApplicationContext applicationContext) { 30 | var environment = applicationContext.getEnvironment(); 31 | 32 | if (!"DYNAMODB".equals(environment.getRequiredProperty("storage.positions.type"))) { 33 | return; 34 | } 35 | 36 | var dynamoDBProperties = PropertiesUtil.bind(environment, new DynamoDBProperties()); 37 | 38 | applicationContext.registerBean(DynamoDBPositionsStorage.class, () -> { 39 | var builder = DynamoDbAsyncClient.builder(); 40 | 41 | dynamoDBProperties.getEndpoint() 42 | .map(URI::create) 43 | .ifPresent(builder::endpointOverride); 44 | 45 | var dynamoDB = builder 46 | .build(); 47 | 48 | if (dynamoDBProperties.isAutoCreateTable()) { 49 | log.info("Going to automatically create a table with name '{}'", dynamoDBProperties.getPositionsTable()); 50 | var request = CreateTableRequest.builder() 51 | .keySchema( 52 | KeySchemaElement.builder().attributeName(DynamoDBPositionsStorage.HASH_KEY_FIELD).keyType(KeyType.HASH).build(), 53 | KeySchemaElement.builder().attributeName(DynamoDBPositionsStorage.RANGE_KEY_FIELD).keyType(KeyType.RANGE).build() 54 | ) 55 | .attributeDefinitions( 56 | AttributeDefinition.builder().attributeName(DynamoDBPositionsStorage.HASH_KEY_FIELD).attributeType(ScalarAttributeType.S).build(), 57 | AttributeDefinition.builder().attributeName(DynamoDBPositionsStorage.RANGE_KEY_FIELD).attributeType(ScalarAttributeType.S).build() 58 | ) 59 | .tableName(dynamoDBProperties.getPositionsTable()) 60 | .provisionedThroughput(ProvisionedThroughput.builder().readCapacityUnits(10L).writeCapacityUnits(10L).build()) 61 | .build(); 62 | 63 | try { 64 | dynamoDB.createTable(request).get(); 65 | } catch (Exception e) { 66 | throw new IllegalStateException("Can't create positions dynamodb table", e); 67 | } 68 | } 69 | 70 | return new DynamoDBPositionsStorage( 71 | dynamoDB, 72 | dynamoDBProperties.getPositionsTable() 73 | ); 74 | }); 75 | } 76 | 77 | @ConfigurationProperties("dynamodb") 78 | @Data 79 | @Validated 80 | public static class DynamoDBProperties { 81 | 82 | Optional endpoint = Optional.empty(); 83 | 84 | boolean autoCreateTable = false; 85 | 86 | @NotEmpty 87 | String positionsTable; 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /plugins/dynamodb-positions-storage/src/test/java/com/github/bsideup/liiklus/dynamodb/DynamoDBPositionsStorageTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.dynamodb; 2 | 3 | import com.github.bsideup.liiklus.ApplicationRunner; 4 | import com.github.bsideup.liiklus.positions.PositionsStorage; 5 | import com.github.bsideup.liiklus.positions.PositionsStorageTests; 6 | import lombok.Getter; 7 | import org.springframework.context.ApplicationContext; 8 | import org.testcontainers.containers.localstack.LocalStackContainer; 9 | import org.testcontainers.containers.localstack.LocalStackContainer.Service; 10 | import org.testcontainers.utility.DockerImageName; 11 | 12 | import java.util.UUID; 13 | 14 | class DynamoDBPositionsStorageTest implements PositionsStorageTests { 15 | 16 | private static final LocalStackContainer localstack = new LocalStackContainer(DockerImageName.parse("localstack/localstack:0.11.2")) 17 | .withServices(Service.DYNAMODB); 18 | 19 | static final ApplicationContext applicationContext; 20 | 21 | static { 22 | localstack.start(); 23 | var endpointConfiguration = localstack.getEndpointConfiguration(Service.DYNAMODB); 24 | System.setProperty("aws.region", endpointConfiguration.getSigningRegion()); 25 | var credentials = localstack.getDefaultCredentialsProvider().getCredentials(); 26 | System.setProperty("aws.accessKeyId", credentials.getAWSAccessKeyId()); 27 | System.setProperty("aws.secretAccessKey", credentials.getAWSSecretKey()); 28 | 29 | applicationContext = new ApplicationRunner("MEMORY", "DYNAMODB") 30 | .withProperty("dynamodb.autoCreateTable", "true") 31 | .withProperty("dynamodb.positionsTable", "positions-" + UUID.randomUUID()) 32 | .withProperty("dynamodb.endpoint", endpointConfiguration.getServiceEndpoint()) 33 | .run(); 34 | } 35 | 36 | @Getter 37 | PositionsStorage storage = applicationContext.getBean(PositionsStorage.class); 38 | } -------------------------------------------------------------------------------- /plugins/dynamodb-positions-storage/src/test/java/com/github/bsideup/liiklus/dynamodb/config/DynamoDBConfigurationTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.dynamodb.config; 2 | 3 | import com.github.bsideup.liiklus.dynamodb.DynamoDBPositionsStorage; 4 | import com.github.bsideup.liiklus.positions.PositionsStorage; 5 | import org.junit.jupiter.api.Test; 6 | import org.springframework.beans.BeansException; 7 | import org.springframework.boot.context.properties.bind.validation.BindValidationException; 8 | import org.springframework.boot.test.context.runner.ApplicationContextRunner; 9 | import org.springframework.context.ApplicationContextInitializer; 10 | import org.springframework.context.support.StaticApplicationContext; 11 | 12 | import static org.assertj.core.api.Assertions.assertThat; 13 | 14 | class DynamoDBConfigurationTest { 15 | 16 | ApplicationContextRunner applicationContextRunner = new ApplicationContextRunner(() -> new StaticApplicationContext() { 17 | @Override 18 | public void refresh() throws BeansException, IllegalStateException { 19 | } 20 | }) 21 | .withInitializer((ApplicationContextInitializer) new DynamoDBConfiguration()); 22 | 23 | @Test 24 | void shouldSkipWhenNotDynamoDB() { 25 | applicationContextRunner = applicationContextRunner.withPropertyValues( 26 | "storage.positions.type: FOO" 27 | ); 28 | applicationContextRunner.run(context -> { 29 | assertThat(context).doesNotHaveBean(PositionsStorage.class); 30 | }); 31 | } 32 | 33 | @Test 34 | void shouldValidateProperties() { 35 | applicationContextRunner = applicationContextRunner.withPropertyValues( 36 | "storage.positions.type: DYNAMODB" 37 | ); 38 | applicationContextRunner.run(context -> { 39 | assertThat(context) 40 | .getFailure() 41 | .hasCauseInstanceOf(BindValidationException.class); 42 | }); 43 | applicationContextRunner = applicationContextRunner.withPropertyValues( 44 | "dynamodb.positionsTable: foo" 45 | ); 46 | applicationContextRunner.run(context -> { 47 | assertThat(context).hasNotFailed(); 48 | }); 49 | } 50 | 51 | @Test 52 | void shouldRegisterWhenDynamoDB() { 53 | applicationContextRunner = applicationContextRunner.withPropertyValues( 54 | "storage.positions.type: DYNAMODB", 55 | "dynamodb.positionsTable: foo" 56 | ); 57 | applicationContextRunner.run(context -> { 58 | assertThat(context).hasSingleBean(DynamoDBPositionsStorage.class); 59 | }); 60 | } 61 | 62 | } -------------------------------------------------------------------------------- /plugins/grpc-transport-auth/build.gradle: -------------------------------------------------------------------------------- 1 | jar { 2 | manifest { 3 | attributes( 4 | 'Plugin-Id': "${project.name}", 5 | 'Plugin-Version': "${project.version}", 6 | 'Plugin-Dependencies': [ 7 | project(":plugins:grpc-transport").name 8 | ].join(","), 9 | ) 10 | } 11 | 12 | into('lib') { 13 | from(configurations.runtimeClasspath - configurations.compileOnlyClasspath) 14 | } 15 | } 16 | 17 | tasks.test.dependsOn(jar) 18 | tasks.test.dependsOn( 19 | [":plugins:inmemory-records-storage", ":plugins:inmemory-positions-storage", ":plugins:grpc-transport"].collect { 20 | project(it).getTasksByName("jar", true) 21 | } 22 | ) 23 | 24 | dependencies { 25 | compileOnly 'com.google.auto.service:auto-service' 26 | annotationProcessor 'com.google.auto.service:auto-service' 27 | 28 | compileOnly project(":app") 29 | compileOnly project(":plugins:grpc-transport") 30 | 31 | implementation 'com.auth0:java-jwt:3.9.0' 32 | implementation 'com.avast.grpc.jwt:grpc-java-jwt:0.2.0' 33 | 34 | // see https://github.com/grpc/grpc-java/blob/master/SECURITY.md#netty for the compatibility version 35 | runtimeOnly 'io.netty:netty-tcnative-boringssl-static:2.0.25.Final' 36 | testRuntimeOnly 'io.netty:netty-tcnative-boringssl-static:2.0.25.Final' 37 | 38 | testImplementation project(":tck") 39 | testImplementation (project(":client")) { 40 | exclude group: 'io.rsocket.rpc', module: 'rsocket-rpc-core' 41 | exclude group: 'io.rsocket', module: 'rsocket-transport-netty' 42 | } 43 | testImplementation 'org.springframework.boot:spring-boot-starter-test' 44 | testImplementation 'org.bouncycastle:bcpkix-jdk15on:1.66' 45 | testImplementation project(":plugins:grpc-transport") 46 | } 47 | -------------------------------------------------------------------------------- /plugins/grpc-transport-auth/src/main/java/com/github/bsideup/liiklus/transport/grpc/StaticRSAKeyProvider.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.transport.grpc; 2 | 3 | import com.auth0.jwt.interfaces.RSAKeyProvider; 4 | import lombok.SneakyThrows; 5 | 6 | import java.security.KeyFactory; 7 | import java.security.NoSuchAlgorithmException; 8 | import java.security.interfaces.RSAPrivateKey; 9 | import java.security.interfaces.RSAPublicKey; 10 | import java.security.spec.InvalidKeySpecException; 11 | import java.security.spec.X509EncodedKeySpec; 12 | import java.util.Base64; 13 | import java.util.Map; 14 | import java.util.NoSuchElementException; 15 | import java.util.stream.Collectors; 16 | 17 | public class StaticRSAKeyProvider implements RSAKeyProvider { 18 | private Map keys; 19 | 20 | public StaticRSAKeyProvider(Map keys) { 21 | this.keys = keys.entrySet() 22 | .stream() 23 | .collect(Collectors.toMap( 24 | Map.Entry::getKey, 25 | key -> { 26 | try { 27 | return parsePubKey(key.getValue()); 28 | } catch (InvalidKeySpecException e) { 29 | throw new IllegalArgumentException(String.format("Invalid RSA pubkey with id %s", key.getKey()), e); 30 | } 31 | } 32 | )); 33 | } 34 | 35 | @Override 36 | public RSAPublicKey getPublicKeyById(String keyId) { 37 | if (!keys.containsKey(keyId)) { 38 | throw new NoSuchElementException(String.format("KeyId %s is not defined to authorize GRPC requests", keyId)); 39 | } 40 | return keys.get(keyId); 41 | } 42 | 43 | @Override 44 | public RSAPrivateKey getPrivateKey() { 45 | return null; // we don't sign anything 46 | } 47 | 48 | @Override 49 | public String getPrivateKeyId() { 50 | return null; // we don't sign anything 51 | } 52 | 53 | /** 54 | * Standard "ssh-rsa AAAAB3Nza..." pubkey representation could be converted to a proper format with 55 | * `ssh-keygen -f id_rsa.pub -e -m pkcs8` 56 | * 57 | * This method will work the same if you strip beginning, as well as line breaks on your own 58 | * 59 | * @param key X509 encoded (with -----BEGIN PUBLIC KEY----- lines) 60 | * @return parsed string 61 | */ 62 | @SneakyThrows(NoSuchAlgorithmException.class) 63 | static RSAPublicKey parsePubKey(String key) throws InvalidKeySpecException { 64 | String keyContent = key.replaceAll("\\n", "") 65 | .replace("-----BEGIN PUBLIC KEY-----", "") 66 | .replace("-----END PUBLIC KEY-----", ""); 67 | 68 | byte[] byteKey = Base64.getDecoder().decode(keyContent); 69 | var x509EncodedKeySpec = new X509EncodedKeySpec(byteKey); 70 | 71 | return (RSAPublicKey) KeyFactory.getInstance("RSA").generatePublic(x509EncodedKeySpec); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /plugins/grpc-transport-auth/src/main/java/com/github/bsideup/liiklus/transport/grpc/config/GRPCTLSConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.transport.grpc.config; 2 | 3 | import com.github.bsideup.liiklus.transport.grpc.GRPCLiiklusTransportConfigurer; 4 | import com.github.bsideup.liiklus.util.PropertiesUtil; 5 | import com.google.auto.service.AutoService; 6 | import io.grpc.netty.GrpcSslContexts; 7 | import io.grpc.netty.NettyServerBuilder; 8 | import io.netty.handler.ssl.ClientAuth; 9 | import io.netty.handler.ssl.SslContext; 10 | import io.netty.handler.ssl.SslContextBuilder; 11 | import lombok.Data; 12 | import lombok.SneakyThrows; 13 | import lombok.Value; 14 | import lombok.extern.slf4j.Slf4j; 15 | import org.springframework.boot.context.properties.ConfigurationProperties; 16 | import org.springframework.context.ApplicationContextInitializer; 17 | import org.springframework.context.support.GenericApplicationContext; 18 | import org.springframework.core.io.Resource; 19 | 20 | import java.io.File; 21 | 22 | @Slf4j 23 | @AutoService(ApplicationContextInitializer.class) 24 | public class GRPCTLSConfiguration implements ApplicationContextInitializer { 25 | 26 | @Override 27 | public void initialize(GenericApplicationContext applicationContext) { 28 | var environment = applicationContext.getEnvironment(); 29 | 30 | var tlsProperties = PropertiesUtil.bind(environment, new GRPCTLSProperties()); 31 | 32 | if (tlsProperties.getKey() == null) { 33 | return; 34 | } 35 | 36 | log.info("GRPC {}TLS ENABLED", tlsProperties.getTrustCert() != null ? "mutual " : ""); 37 | 38 | applicationContext.registerBean( 39 | TLSGRPCTransportConfigurer.class, 40 | () -> new TLSGRPCTransportConfigurer(tlsProperties) 41 | ); 42 | } 43 | 44 | @Value 45 | static class TLSGRPCTransportConfigurer implements GRPCLiiklusTransportConfigurer { 46 | 47 | GRPCTLSProperties properties; 48 | 49 | @Override 50 | public void apply(NettyServerBuilder builder) { 51 | SslContext ctx = createSSLContext( 52 | properties.getKey(), 53 | properties.getKeyPassword(), 54 | properties.getKeyCertChain(), 55 | properties.getTrustCert() 56 | ); 57 | 58 | builder.sslContext(ctx); 59 | } 60 | 61 | /** 62 | * Mostly copy of the https://github.com/grpc/grpc-java/tree/master/examples/example-tls 63 | * and https://github.com/grpc/grpc-java/blob/master/SECURITY.md 64 | * 65 | * Refer to {@link io.netty.handler.ssl.SslContextBuilder#forServer(File keyCertChainFile, File keyFile, String keyPassword)} 66 | * for more details. 67 | * 68 | * @param key a PKCS#8 private key file in PEM format 69 | * @param keyPassword the password of the key or null if not protected 70 | * @param keyCertChain an X.509 certificate chain file in PEM format 71 | * @param trustCert file should contain an X.509 certificate collection in PEM format 72 | * @return ready-to-use ssl context. 73 | */ 74 | @SneakyThrows 75 | SslContext createSSLContext(Resource key, String keyPassword, Resource keyCertChain, Resource trustCert) { 76 | SslContextBuilder sslClientContextBuilder = SslContextBuilder.forServer( 77 | keyCertChain.getInputStream(), 78 | key.getInputStream(), 79 | keyPassword 80 | ); 81 | if (trustCert != null) { 82 | sslClientContextBuilder.trustManager(trustCert.getInputStream()); 83 | sslClientContextBuilder.clientAuth(ClientAuth.REQUIRE); 84 | } 85 | return GrpcSslContexts.configure(sslClientContextBuilder).build(); 86 | } 87 | } 88 | 89 | @ConfigurationProperties("grpc.tls") 90 | @Data 91 | static class GRPCTLSProperties { 92 | 93 | Resource key; 94 | 95 | String keyPassword; 96 | 97 | Resource keyCertChain; 98 | 99 | Resource trustCert; 100 | 101 | } 102 | } -------------------------------------------------------------------------------- /plugins/grpc-transport-auth/src/test/java/com/github/bsideup/liiklus/transport/grpc/StaticRSAKeyProviderTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.transport.grpc; 2 | 3 | 4 | import org.junit.jupiter.api.Test; 5 | import org.junit.jupiter.params.ParameterizedTest; 6 | import org.junit.jupiter.params.provider.ValueSource; 7 | 8 | import java.security.spec.InvalidKeySpecException; 9 | import java.util.Map; 10 | import java.util.NoSuchElementException; 11 | 12 | import static org.assertj.core.api.Assertions.assertThat; 13 | import static org.assertj.core.api.Assertions.assertThatThrownBy; 14 | 15 | class StaticRSAKeyProviderTest { 16 | 17 | private static final String STRIPPED_4096 = "MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEApkuZHC50VVyc6mkqWMXl" + 18 | "+fuhBmXhw8N8w6A0mlxRDHltdsPYvxE5n/Id4xUDCISZfjXIuSVyq/a7K3esbEqc" + 19 | "fs6/dm6PQuLMsaxEzS3Gxn+QxELJ41IyKiT0DFAhorSoFChfzkkS7whHm+O8wVDI" + 20 | "Z0Aj5TjY5t0/1CvU7wKeMDjVqOR3usEb37/5qu4ps0RbgQzBKjoJ3LSo/tt4tZw+" + 21 | "V3dT2lEVCKCA9OA0I5UXFUwUyMH8NudSlEpExGcmNHM4sEW4NK4Y7RW9tyDT0RQR" + 22 | "ydUIP8rXkjqyxMyHnwNUuzxJHqIXAdEhzw2xGLBSxr87wfmK09TEfSjmMemHfCfF" + 23 | "Ht+esDSy7zRB68hCS/chyN57xyBWG3BeaKeJm34gLU6gt+9Bhvq90a0RXA7TXK7y" + 24 | "QwhDQQwNPhUQshE036l/jCDxmgJZPNkvpweAeROsoEDf5o0TRaybXbyQh+jn+iJP" + 25 | "ve7K2bTixmjlQKOWB4HZ+1YWyTzUabpdeuHVokKuVFzpKqi5oid3Bz17XU4fN36e" + 26 | "M9CSV1urnlgdVwKwYttFwuerstwpB2rOT1UmamQhPwfDGy9x2d2vghSi+ELzKkKv" + 27 | "yAlkIdeK/WLIi3l/R4pCFC1JfAGagXS+Jtvr9+PkiD3bG220HpW1ry68CZcsO91z" + 28 | "7UCJcQMxXdt1gk3K+EbWaDUCAwEAAQ=="; 29 | 30 | private static final String FULL_2048 = "-----BEGIN PUBLIC KEY-----\n" + 31 | "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6b/6nQLIQQ8fHT4PcSyb\n" + 32 | "hOLUE/237dgicbjsE7/Z/uPffuc36NTMJ122ppz6dWYnCrQ6CeTgAde4hlLE7Kvv\n" + 33 | "aFiUbe5XKwSL8KV292XqrwRZhMI58TTTygcrBodYGzHy0Yytv703rz+9Qt5HO5BF\n" + 34 | "02/+sM+Z0wlH6aXl3K3/2HfSOfitqnArBGaAs+PRNX2jlVKD1c9Cb7vo5L0X7q+6\n" + 35 | "55uBErEoN7IHbj1u33qI/xEvPSycIiT2RXMGZkvDZH6mTsALel4aP4Qpp1NcE+kD\n" + 36 | "itoBYAPTGgR4gBQveXZmD10yUVgJl2icINY3FvT9oJB6wgCY9+iTvufPppT1RPFH\n" + 37 | "dQIDAQAB\n" + 38 | "-----END PUBLIC KEY-----\n"; 39 | 40 | @ParameterizedTest 41 | @ValueSource(strings = { 42 | STRIPPED_4096, 43 | FULL_2048 44 | }) 45 | void shouldParseX509(String pubkey) throws InvalidKeySpecException { 46 | var parsed = StaticRSAKeyProvider.parsePubKey(pubkey); 47 | assertThat(parsed).isNotNull(); 48 | assertThat(parsed.getAlgorithm()).isEqualTo("RSA"); 49 | } 50 | 51 | @Test 52 | void shouldThrowExceptionOnInvalid() { 53 | assertThatThrownBy(() -> StaticRSAKeyProvider.parsePubKey("")).isInstanceOf(InvalidKeySpecException.class); 54 | } 55 | 56 | @Test 57 | void shouldCreateProviderInstance() { 58 | new StaticRSAKeyProvider(Map.of( 59 | "valid", STRIPPED_4096 60 | )); 61 | } 62 | 63 | @Test 64 | void shouldHandleValidAndInvalidWithExceptionInConstructor() { 65 | assertThatThrownBy(() -> new StaticRSAKeyProvider(Map.of( 66 | "valid", STRIPPED_4096, 67 | "invalid", "" 68 | ))) 69 | .isInstanceOf(IllegalArgumentException.class); 70 | } 71 | 72 | @Test 73 | void shouldComplainOnNotFoundKey() { 74 | assertThatThrownBy(() -> new StaticRSAKeyProvider(Map.of( 75 | "valid", STRIPPED_4096 76 | )).getPublicKeyById("unknown")) 77 | .isInstanceOf(NoSuchElementException.class); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /plugins/grpc-transport-auth/src/test/java/com/github/bsideup/liiklus/transport/grpc/config/GRPCAuthConfigurationTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.transport.grpc.config; 2 | 3 | import com.github.bsideup.liiklus.transport.grpc.config.GRPCAuthConfiguration.JWTAuthGRPCTransportConfigurer; 4 | import org.junit.jupiter.api.Test; 5 | import org.springframework.beans.BeansException; 6 | import org.springframework.boot.context.properties.bind.validation.BindValidationException; 7 | import org.springframework.boot.test.context.runner.ApplicationContextRunner; 8 | import org.springframework.context.ApplicationContextInitializer; 9 | import org.springframework.context.support.StaticApplicationContext; 10 | 11 | import static org.assertj.core.api.Assertions.assertThat; 12 | 13 | class GRPCAuthConfigurationTest { 14 | private static final String FULL_2048 = "-----BEGIN PUBLIC KEY-----\n" + 15 | "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6b/6nQLIQQ8fHT4PcSyb\n" + 16 | "hOLUE/237dgicbjsE7/Z/uPffuc36NTMJ122ppz6dWYnCrQ6CeTgAde4hlLE7Kvv\n" + 17 | "aFiUbe5XKwSL8KV292XqrwRZhMI58TTTygcrBodYGzHy0Yytv703rz+9Qt5HO5BF\n" + 18 | "02/+sM+Z0wlH6aXl3K3/2HfSOfitqnArBGaAs+PRNX2jlVKD1c9Cb7vo5L0X7q+6\n" + 19 | "55uBErEoN7IHbj1u33qI/xEvPSycIiT2RXMGZkvDZH6mTsALel4aP4Qpp1NcE+kD\n" + 20 | "itoBYAPTGgR4gBQveXZmD10yUVgJl2icINY3FvT9oJB6wgCY9+iTvufPppT1RPFH\n" + 21 | "dQIDAQAB\n" + 22 | "-----END PUBLIC KEY-----\n"; 23 | 24 | ApplicationContextRunner applicationContextRunner = new ApplicationContextRunner(() -> new StaticApplicationContext() { 25 | @Override 26 | public void refresh() throws BeansException, IllegalStateException { 27 | } 28 | }) 29 | .withInitializer((ApplicationContextInitializer) new GRPCAuthConfiguration()); 30 | 31 | @Test 32 | void shouldRequireAlg() { 33 | applicationContextRunner = applicationContextRunner.withPropertyValues( 34 | "spring.profiles.active: not_gateway", 35 | "grpc.auth.alg: NONE" 36 | ); 37 | applicationContextRunner.run(context -> { 38 | assertThat(context).doesNotHaveBean(JWTAuthGRPCTransportConfigurer.class); 39 | }); 40 | } 41 | 42 | @Test 43 | void shouldAddWithHmac512() { 44 | applicationContextRunner = applicationContextRunner.withPropertyValues( 45 | "spring.profiles.active: not_gateway", 46 | "grpc.auth.alg: HMAC512", 47 | "grpc.auth.secret: secret" 48 | ); 49 | applicationContextRunner.run(context -> { 50 | assertThat(context) 51 | .hasSingleBean(JWTAuthGRPCTransportConfigurer.class); 52 | }); 53 | } 54 | 55 | @Test 56 | void shouldAddWithRsa512() { 57 | applicationContextRunner = applicationContextRunner.withPropertyValues( 58 | "spring.profiles.active: not_gateway", 59 | "grpc.auth.alg: RSA512", 60 | "grpc.auth.keys.key: " + FULL_2048 61 | ); 62 | applicationContextRunner.run(context -> { 63 | assertThat(context) 64 | .hasSingleBean(JWTAuthGRPCTransportConfigurer.class); 65 | }); 66 | } 67 | 68 | @Test 69 | void shouldValidateParametersHmac512() { 70 | applicationContextRunner = applicationContextRunner.withPropertyValues( 71 | "spring.profiles.active: not_gateway", 72 | "grpc.auth.alg: HMAC512" 73 | ); 74 | applicationContextRunner.run(context -> { 75 | assertThat(context) 76 | .getFailure() 77 | .hasCauseInstanceOf(BindValidationException.class); 78 | }); 79 | } 80 | 81 | @Test 82 | void shouldValidateParametersRsa512() { 83 | applicationContextRunner = applicationContextRunner.withPropertyValues( 84 | "spring.profiles.active: not_gateway", 85 | "grpc.auth.alg: RSA512" 86 | ); 87 | applicationContextRunner.run(context -> { 88 | assertThat(context) 89 | .getFailure() 90 | .hasCauseInstanceOf(BindValidationException.class); 91 | }); 92 | } 93 | } -------------------------------------------------------------------------------- /plugins/grpc-transport-auth/src/test/resources/keys/private_key_main_2048_pkcs8.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDpv/qdAshBDx8d 3 | Pg9xLJuE4tQT/bft2CJxuOwTv9n+499+5zfo1MwnXbamnPp1ZicKtDoJ5OAB17iG 4 | UsTsq+9oWJRt7lcrBIvwpXb3ZeqvBFmEwjnxNNPKBysGh1gbMfLRjK2/vTevP71C 5 | 3kc7kEXTb/6wz5nTCUfppeXcrf/Yd9I5+K2qcCsEZoCz49E1faOVUoPVz0Jvu+jk 6 | vRfur7rnm4ESsSg3sgduPW7feoj/ES89LJwiJPZFcwZmS8NkfqZOwAt6Xho/hCmn 7 | U1wT6QOK2gFgA9MaBHiAFC95dmYPXTJRWAmXaJwg1jcW9P2gkHrCAJj36JO+58+m 8 | lPVE8Ud1AgMBAAECggEBAKeqljhDg6LvFtFh76+tYIxsK9V/G4yWiPZrv6LW7aZg 9 | i7K6Zacz5JCkLtzKIvlM4dpStoLcUjXgJ7Lp8ekV1y9Qwn8sBAiORVbDSVdiGnmZ 10 | tCB/NRKoYvY6OAmB0ZgINvVKZGLxddzV6orpZ8z4yq1EWzs2Xk87DAMzhXLKuIbq 11 | 5KCCoSlckDPCf4azfDgnxee3W5Tb3sH3CWi3msz7cu+2AzLT0OAjE4XYCTiR7vnT 12 | MtpFNX1AR3HiwE1a/8YosOjqKFcjoMSwqjfOcShrS5ePvm4p2agXeJPz924PXB0J 13 | +HXF3uS7D61HOQ5KGnnyN3+l1j3KTXN8jqC2Xzo/qu0CgYEA/ATiy1l4iIf2pue4 14 | kllv9/sPZqIkcu5QAecnQaYe29bdh8j7rT3wWMXzVDpBXi5xYVK356BuEtRtdw1A 15 | pdjs1jWngvC8dVGnHBVLLy8NYOKd8uDJSFsysyAJ+Zirw+F3o9L0xoGIWgxEA+4m 16 | d44UbcdJtvOBaZ+kremDpHA44p8CgYEA7XE3WwLF6AbZHw5COL72UCeVc9Ist0if 17 | jw2+KP2U8JVj7E/A7dJVvlujJFoWVUmr+NIKzBLOpP/bjkTcGfYFWCxsKDpGoTJz 18 | Vx5OSjAAaRtqiUniyNEI6UKsYLmozxU4sylUGlTeD67yvZUgqdnB+M8Y8J/xJeW4 19 | 4/GoS2MiEWsCgYEAq7zYoCJUVRXyK0L1MCXqe16G3DXaCMgFlYZj5gTCOqVtST7Y 20 | 4vG2e4hJjTg1m2yiruOjlyBoYkSIY/yP9XSh0Ee34y8R/hCqhCSum3TA9Sj44a30 21 | /G3JWu+WXJSBWHapBOaZDzzuIg8BunvrksUrfrOztAy0P7oeirT6lHA1E5kCgYAt 22 | LADP+7MS9VqRIfFPQmUx0pYINs/y/on8eSzYN4YCTyl3Z6TYmc9eK6jZ3ZmqGB3z 23 | dGJBeMJ/eX2Xj1ogRkG2CJ16+bs+J47x3/4c9wzc8i5OeBQBCGOdnOWWcTvASdVD 24 | oHUznTmx2iKsFpbkOV1BrISeIo+KGi4Wj37o+K8eiQKBgHQOPd/5G7ZEyiV+l8rB 25 | Z1c7DpyqN1yj3cEO9k3Pw485e6GxYJOWHOuqYu7AWhiFi3zN2OuyGQfykjwAs5v3 26 | B6JIhIk2bGv17b54skr4KgZTzeseWwtN918G8iXz0LgZGaafdLywMkgTM1lyqO0i 27 | DJbKe/M6FGmMmbz7j5NRdE/6 28 | -----END PRIVATE KEY----- -------------------------------------------------------------------------------- /plugins/grpc-transport/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'idea' 3 | id 'java-library' 4 | id 'com.google.protobuf' 5 | } 6 | 7 | protobuf { 8 | protoc { 9 | artifact = 'com.google.protobuf:protoc' 10 | } 11 | 12 | generatedFilesBaseDir = "$projectDir/generated" 13 | 14 | plugins { 15 | grpc { 16 | artifact = 'io.grpc:protoc-gen-grpc-java' 17 | } 18 | 19 | reactor { 20 | artifact = "com.salesforce.servicelibs:reactor-grpc:0.9.0:jdk8@jar" 21 | } 22 | } 23 | 24 | generateProtoTasks { 25 | ofSourceSet('main').each { task -> 26 | task.builtins { 27 | remove java 28 | } 29 | task.plugins { 30 | grpc { } 31 | reactor { } 32 | } 33 | } 34 | } 35 | } 36 | 37 | clean { 38 | delete protobuf.generatedFilesBaseDir 39 | } 40 | 41 | idea { 42 | module { 43 | generatedSourceDirs += file("${protobuf.generatedFilesBaseDir}/main/java") 44 | generatedSourceDirs += file("${protobuf.generatedFilesBaseDir}/main/grpc") 45 | generatedSourceDirs += file("${protobuf.generatedFilesBaseDir}/main/reactor") 46 | } 47 | } 48 | 49 | jar { 50 | manifest { 51 | attributes( 52 | 'Plugin-Id': "${project.name}", 53 | 'Plugin-Version': "${project.version}", 54 | ) 55 | } 56 | 57 | into('lib') { 58 | from(configurations.runtimeClasspath - configurations.compileOnlyClasspath) 59 | } 60 | } 61 | 62 | dependencies { 63 | compileOnly 'com.google.auto.service:auto-service' 64 | annotationProcessor 'com.google.auto.service:auto-service' 65 | 66 | compileOnly project(":app") 67 | 68 | protobuf project(":protocol") 69 | 70 | api 'com.salesforce.servicelibs:reactor-grpc-stub' 71 | api 'io.grpc:grpc-stub' 72 | api 'io.grpc:grpc-protobuf' 73 | api 'io.grpc:grpc-netty' 74 | api 'io.grpc:grpc-services' 75 | 76 | testImplementation project(":app") 77 | testImplementation 'org.springframework.boot:spring-boot-test' 78 | testImplementation 'org.assertj:assertj-core' 79 | testImplementation 'org.mockito:mockito-core' 80 | } 81 | -------------------------------------------------------------------------------- /plugins/grpc-transport/src/main/java/com/github/bsideup/liiklus/transport/grpc/GRPCLiiklusService.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.transport.grpc; 2 | 3 | import com.github.bsideup.liiklus.protocol.AckRequest; 4 | import com.github.bsideup.liiklus.protocol.GetEndOffsetsReply; 5 | import com.github.bsideup.liiklus.protocol.GetEndOffsetsRequest; 6 | import com.github.bsideup.liiklus.protocol.GetOffsetsReply; 7 | import com.github.bsideup.liiklus.protocol.GetOffsetsRequest; 8 | import com.github.bsideup.liiklus.protocol.PublishReply; 9 | import com.github.bsideup.liiklus.protocol.PublishRequest; 10 | import com.github.bsideup.liiklus.protocol.ReactorLiiklusServiceGrpc; 11 | import com.github.bsideup.liiklus.protocol.ReceiveReply; 12 | import com.github.bsideup.liiklus.protocol.ReceiveRequest; 13 | import com.github.bsideup.liiklus.protocol.SubscribeReply; 14 | import com.github.bsideup.liiklus.protocol.SubscribeRequest; 15 | import com.github.bsideup.liiklus.service.LiiklusService; 16 | import com.google.protobuf.Empty; 17 | import io.grpc.Status; 18 | import lombok.AccessLevel; 19 | import lombok.RequiredArgsConstructor; 20 | import lombok.experimental.FieldDefaults; 21 | import reactor.core.publisher.Flux; 22 | import reactor.core.publisher.Mono; 23 | 24 | @RequiredArgsConstructor 25 | @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) 26 | public class GRPCLiiklusService extends ReactorLiiklusServiceGrpc.LiiklusServiceImplBase { 27 | 28 | LiiklusService liiklusService; 29 | 30 | @Override 31 | public Mono publish(Mono request) { 32 | return liiklusService.publish(request) 33 | .onErrorMap(e -> Status.INTERNAL.withCause(e).withDescription(e.getMessage()).asException()); 34 | } 35 | 36 | @Override 37 | public Flux subscribe(Mono request) { 38 | return liiklusService.subscribe(request) 39 | .onErrorMap(e -> Status.INTERNAL.withCause(e).withDescription(e.getMessage()).asException()); 40 | } 41 | 42 | @Override 43 | public Flux receive(Mono request) { 44 | return liiklusService.receive(request) 45 | .onErrorMap(e -> Status.INTERNAL.withCause(e).withDescription(e.getMessage()).asException()); 46 | } 47 | 48 | @Override 49 | public Mono ack(Mono request) { 50 | return liiklusService.ack(request) 51 | .onErrorMap(e -> Status.INTERNAL.withCause(e).withDescription(e.getMessage()).asException()); 52 | } 53 | 54 | @Override 55 | public Mono getOffsets(Mono request) { 56 | return liiklusService.getOffsets(request) 57 | .onErrorMap(e -> Status.INTERNAL.withCause(e).withDescription(e.getMessage()).asException()); 58 | } 59 | 60 | @Override 61 | public Mono getEndOffsets(Mono request) { 62 | return liiklusService.getEndOffsets(request) 63 | .onErrorMap(e -> Status.INTERNAL.withCause(e).withDescription(e.getMessage()).asException()); 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /plugins/grpc-transport/src/main/java/com/github/bsideup/liiklus/transport/grpc/GRPCLiiklusTransportConfigurer.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.transport.grpc; 2 | 3 | import io.grpc.netty.NettyServerBuilder; 4 | 5 | @FunctionalInterface 6 | public interface GRPCLiiklusTransportConfigurer { 7 | void apply(NettyServerBuilder builder); 8 | } 9 | -------------------------------------------------------------------------------- /plugins/inmemory-positions-storage/build.gradle: -------------------------------------------------------------------------------- 1 | jar { 2 | manifest { 3 | attributes( 4 | 'Plugin-Id': "${project.name}", 5 | 'Plugin-Version': "${project.version}", 6 | ) 7 | } 8 | 9 | into('lib') { 10 | from(configurations.runtimeClasspath - configurations.compileOnlyClasspath) 11 | } 12 | } 13 | 14 | tasks.test.dependsOn( 15 | jar, 16 | rootProject.project(":plugins:inmemory-records-storage").getTasksByName("jar", false) 17 | ) 18 | 19 | dependencies { 20 | compileOnly 'com.google.auto.service:auto-service' 21 | annotationProcessor 'com.google.auto.service:auto-service' 22 | 23 | compileOnly project(':app') 24 | 25 | testImplementation project(':tck') 26 | testImplementation 'org.springframework.boot:spring-boot-test' 27 | } 28 | -------------------------------------------------------------------------------- /plugins/inmemory-positions-storage/src/main/java/com/github/bsideup/liiklus/positions/inmemory/InMemoryPositionsStorage.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.positions.inmemory; 2 | 3 | import com.github.bsideup.liiklus.positions.GroupId; 4 | import com.github.bsideup.liiklus.positions.PositionsStorage; 5 | import lombok.RequiredArgsConstructor; 6 | import lombok.Value; 7 | import lombok.experimental.FieldDefaults; 8 | import org.reactivestreams.Publisher; 9 | import reactor.core.publisher.Flux; 10 | 11 | import java.util.Map; 12 | import java.util.concurrent.CompletableFuture; 13 | import java.util.concurrent.CompletionStage; 14 | import java.util.concurrent.ConcurrentHashMap; 15 | import java.util.concurrent.ConcurrentMap; 16 | 17 | /** 18 | * WARNING: this storage type should only be used for testing and NOT in production 19 | */ 20 | @RequiredArgsConstructor 21 | @FieldDefaults(makeFinal = true) 22 | public class InMemoryPositionsStorage implements PositionsStorage { 23 | 24 | ConcurrentMap> storage = new ConcurrentHashMap<>(); 25 | 26 | @Override 27 | public Publisher findAll() { 28 | return Flux.fromIterable(storage.entrySet()) 29 | .map(entry -> new Positions( 30 | entry.getKey().getTopic(), 31 | GroupId.ofString(entry.getKey().getGroupId()), 32 | entry.getValue() 33 | )); 34 | } 35 | 36 | @Override 37 | public CompletionStage> findAll(String topic, GroupId groupId) { 38 | return CompletableFuture.completedFuture(storage.get(Key.of(topic, groupId.asString()))); 39 | } 40 | 41 | @Override 42 | public CompletionStage>> findAllVersionsByGroup(String topic, String groupName) { 43 | return Flux.fromIterable(storage.entrySet()) 44 | .filter(it -> topic.equals(it.getKey().getTopic())) 45 | .filter(it -> groupName.equals(GroupId.ofString(it.getKey().getGroupId()).getName())) 46 | .>collectMap( 47 | it -> GroupId.ofString(it.getKey().getGroupId()).getVersion().orElse(0), 48 | Map.Entry::getValue 49 | ) 50 | .toFuture(); 51 | } 52 | 53 | @Override 54 | public CompletionStage update(String topic, GroupId groupId, int partition, long position) { 55 | storage.computeIfAbsent(Key.of(topic, groupId.asString()), __ -> new ConcurrentHashMap<>()).put(partition, position); 56 | 57 | return CompletableFuture.completedFuture(null); 58 | } 59 | 60 | @Value 61 | @RequiredArgsConstructor(staticName = "of") 62 | private static class Key { 63 | 64 | String topic; 65 | 66 | String groupId; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /plugins/inmemory-positions-storage/src/main/java/com/github/bsideup/liiklus/positions/inmemory/config/InMemoryPositionsConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.positions.inmemory.config; 2 | 3 | import com.github.bsideup.liiklus.positions.inmemory.InMemoryPositionsStorage; 4 | import com.google.auto.service.AutoService; 5 | import lombok.extern.slf4j.Slf4j; 6 | import org.springframework.context.ApplicationContextInitializer; 7 | import org.springframework.context.support.GenericApplicationContext; 8 | 9 | @AutoService(ApplicationContextInitializer.class) 10 | @Slf4j 11 | public class InMemoryPositionsConfiguration implements ApplicationContextInitializer { 12 | 13 | @Override 14 | public void initialize(GenericApplicationContext applicationContext) { 15 | var type = applicationContext.getEnvironment().getRequiredProperty("storage.positions.type"); 16 | 17 | if (!"MEMORY".equals(type)) { 18 | return; 19 | } 20 | 21 | log.warn("\n" + 22 | String.format("%0106d", 0).replace("0", "=") + "\n" + 23 | String.format("%0106d", 0).replace("0", "=") + "\n" + 24 | String.format("%0106d", 0).replace("0", "=") + "\n" + 25 | "=== In-memory position storage is used. Please, DO NOT run it in production if you ACK your positions. ===\n" + 26 | String.format("%0106d", 0).replace("0", "=") + "\n" + 27 | String.format("%0106d", 0).replace("0", "=") + "\n" + 28 | String.format("%0106d", 0).replace("0", "=") 29 | ); 30 | applicationContext.registerBean(InMemoryPositionsStorage.class); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /plugins/inmemory-positions-storage/src/test/java/com/github/bsideup/liiklus/positions/inmemory/InMemoryPositionsStorageTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.positions.inmemory; 2 | 3 | import com.github.bsideup.liiklus.ApplicationRunner; 4 | import com.github.bsideup.liiklus.positions.PositionsStorage; 5 | import com.github.bsideup.liiklus.positions.PositionsStorageTests; 6 | import lombok.Getter; 7 | import org.springframework.context.ApplicationContext; 8 | 9 | class InMemoryPositionsStorageTest implements PositionsStorageTests { 10 | 11 | static final ApplicationContext applicationContext = new ApplicationRunner("MEMORY", "MEMORY").run(); 12 | 13 | @Getter 14 | PositionsStorage storage = applicationContext.getBean(PositionsStorage.class); 15 | 16 | } -------------------------------------------------------------------------------- /plugins/inmemory-positions-storage/src/test/java/com/github/bsideup/liiklus/positions/inmemory/config/InMemoryPositionsConfigurationTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.positions.inmemory.config; 2 | 3 | import com.github.bsideup.liiklus.positions.PositionsStorage; 4 | import com.github.bsideup.liiklus.positions.inmemory.InMemoryPositionsStorage; 5 | import org.junit.jupiter.api.Test; 6 | import org.springframework.beans.BeansException; 7 | import org.springframework.boot.test.context.runner.ApplicationContextRunner; 8 | import org.springframework.context.ApplicationContextInitializer; 9 | import org.springframework.context.support.StaticApplicationContext; 10 | 11 | import static org.assertj.core.api.Assertions.assertThat; 12 | 13 | class InMemoryPositionsConfigurationTest { 14 | 15 | ApplicationContextRunner applicationContextRunner = new ApplicationContextRunner(() -> new StaticApplicationContext() { 16 | @Override 17 | public void refresh() throws BeansException, IllegalStateException { 18 | } 19 | }) 20 | .withInitializer((ApplicationContextInitializer) new InMemoryPositionsConfiguration()); 21 | 22 | @Test 23 | void shouldSkipWhenNotInMemory() { 24 | applicationContextRunner = applicationContextRunner.withPropertyValues( 25 | "storage.positions.type: FOO" 26 | ); 27 | applicationContextRunner.run(context -> { 28 | assertThat(context).doesNotHaveBean(PositionsStorage.class); 29 | }); 30 | } 31 | 32 | @Test 33 | void shouldRegisterWhenInMemory() { 34 | applicationContextRunner = applicationContextRunner.withPropertyValues( 35 | "storage.positions.type: MEMORY" 36 | ); 37 | applicationContextRunner.run(context -> { 38 | assertThat(context).hasSingleBean(InMemoryPositionsStorage.class); 39 | }); 40 | } 41 | } -------------------------------------------------------------------------------- /plugins/inmemory-records-storage/build.gradle: -------------------------------------------------------------------------------- 1 | jar { 2 | manifest { 3 | attributes( 4 | 'Plugin-Id': "${project.name}", 5 | 'Plugin-Version': "${project.version}", 6 | ) 7 | } 8 | 9 | into('lib') { 10 | from(configurations.runtimeClasspath - configurations.compileOnlyClasspath) 11 | } 12 | } 13 | 14 | tasks.test.dependsOn( 15 | jar, 16 | rootProject.project(":plugins:inmemory-positions-storage").getTasksByName("jar", false) 17 | ) 18 | 19 | dependencies { 20 | compileOnly 'com.google.auto.service:auto-service' 21 | annotationProcessor 'com.google.auto.service:auto-service' 22 | 23 | compileOnly project(":app") 24 | 25 | testImplementation project(":tck") 26 | testImplementation 'org.springframework.boot:spring-boot-test' 27 | } 28 | -------------------------------------------------------------------------------- /plugins/inmemory-records-storage/src/main/java/com/github/bsideup/liiklus/records/inmemory/config/InMemoryRecordsConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.records.inmemory.config; 2 | 3 | import com.github.bsideup.liiklus.records.inmemory.InMemoryRecordsStorage; 4 | import com.google.auto.service.AutoService; 5 | import lombok.extern.slf4j.Slf4j; 6 | import org.springframework.context.ApplicationContextInitializer; 7 | import org.springframework.context.support.GenericApplicationContext; 8 | import org.springframework.core.env.Profiles; 9 | 10 | @AutoService(ApplicationContextInitializer.class) 11 | @Slf4j 12 | public class InMemoryRecordsConfiguration implements ApplicationContextInitializer { 13 | 14 | public static final int NUMBER_OF_PARTITIONS = 32; 15 | 16 | @Override 17 | public void initialize(GenericApplicationContext applicationContext) { 18 | var environment = applicationContext.getEnvironment(); 19 | if (!environment.acceptsProfiles(Profiles.of("gateway"))) { 20 | return; 21 | } 22 | 23 | String type = environment.getRequiredProperty("storage.records.type"); 24 | if (!"MEMORY".equals(type)) { 25 | return; 26 | } 27 | 28 | log.warn("\n" + 29 | String.format("%0106d", 0).replace("0", "=") + "\n" + 30 | String.format("%0106d", 0).replace("0", "=") + "\n" + 31 | String.format("%0106d", 0).replace("0", "=") + "\n" + 32 | "=== In-memory records storage is used. Please, DO NOT run it in production. ===\n" + 33 | String.format("%0106d", 0).replace("0", "=") + "\n" + 34 | String.format("%0106d", 0).replace("0", "=") + "\n" + 35 | String.format("%0106d", 0).replace("0", "=") 36 | ); 37 | applicationContext.registerBean(InMemoryRecordsStorage.class, () -> new InMemoryRecordsStorage(NUMBER_OF_PARTITIONS)); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /plugins/inmemory-records-storage/src/test/java/com/github/bsideup/liiklus/records/inmemory/InMemoryRecordsStorageTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.records.inmemory; 2 | 3 | import com.github.bsideup.liiklus.ApplicationRunner; 4 | import com.github.bsideup.liiklus.records.RecordStorageTests; 5 | import com.github.bsideup.liiklus.records.RecordsStorage; 6 | import com.github.bsideup.liiklus.records.inmemory.config.InMemoryRecordsConfiguration; 7 | import lombok.Getter; 8 | import org.springframework.context.ApplicationContext; 9 | import reactor.core.publisher.Mono; 10 | 11 | import java.time.Duration; 12 | import java.util.HashMap; 13 | import java.util.Map; 14 | import java.util.UUID; 15 | 16 | class InMemoryRecordsStorageTest implements RecordStorageTests { 17 | 18 | private static final int NUM_OF_PARTITIONS = InMemoryRecordsConfiguration.NUMBER_OF_PARTITIONS; 19 | 20 | // Generate a set of keys where each key goes to unique partition 21 | public static Map PARTITION_KEYS = Mono.fromCallable(() -> UUID.randomUUID().toString()) 22 | .repeat() 23 | .scanWith( 24 | () -> new HashMap(), 25 | (acc, it) -> { 26 | acc.put(InMemoryRecordsStorage.partitionByKey(it, NUM_OF_PARTITIONS), it); 27 | return acc; 28 | } 29 | ) 30 | .filter(it -> it.size() == NUM_OF_PARTITIONS) 31 | .blockFirst(Duration.ofSeconds(10)); 32 | 33 | static final ApplicationContext applicationContext = new ApplicationRunner("MEMORY", "MEMORY").run(); 34 | 35 | @Getter 36 | RecordsStorage target = applicationContext.getBean(RecordsStorage.class); 37 | 38 | @Getter 39 | String topic = UUID.randomUUID().toString(); 40 | 41 | @Override 42 | public String keyByPartition(int partition) { 43 | return PARTITION_KEYS.get(partition); 44 | } 45 | 46 | @Override 47 | public int getNumberOfPartitions() { 48 | return NUM_OF_PARTITIONS; 49 | } 50 | } -------------------------------------------------------------------------------- /plugins/inmemory-records-storage/src/test/java/com/github/bsideup/liiklus/records/inmemory/config/InMemoryRecordsConfigurationTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.records.inmemory.config; 2 | 3 | import com.github.bsideup.liiklus.positions.PositionsStorage; 4 | import com.github.bsideup.liiklus.records.inmemory.InMemoryRecordsStorage; 5 | import org.junit.jupiter.api.Test; 6 | import org.springframework.beans.BeansException; 7 | import org.springframework.boot.test.context.runner.ApplicationContextRunner; 8 | import org.springframework.context.ApplicationContextInitializer; 9 | import org.springframework.context.support.StaticApplicationContext; 10 | 11 | import static org.assertj.core.api.Assertions.assertThat; 12 | 13 | class InMemoryRecordsConfigurationTest { 14 | 15 | ApplicationContextRunner applicationContextRunner = new ApplicationContextRunner(() -> new StaticApplicationContext() { 16 | @Override 17 | public void refresh() throws BeansException, IllegalStateException { 18 | } 19 | }) 20 | .withInitializer((ApplicationContextInitializer) new InMemoryRecordsConfiguration()); 21 | 22 | @Test 23 | void shouldSkipWhenNotInMemory() { 24 | applicationContextRunner.run(context -> { 25 | assertThat(context).doesNotHaveBean(PositionsStorage.class); 26 | }); 27 | } 28 | 29 | @Test 30 | void shouldSkipIfNotGateway() { 31 | applicationContextRunner = applicationContextRunner.withPropertyValues( 32 | "spring.profiles.active: not_gateway", 33 | "storage.records.type: MEMORY" 34 | ); 35 | applicationContextRunner.run(context -> { 36 | assertThat(context).doesNotHaveBean(PositionsStorage.class); 37 | }); 38 | } 39 | 40 | @Test 41 | void shouldRegisterWhenInMemory() { 42 | applicationContextRunner = applicationContextRunner.withPropertyValues( 43 | "spring.profiles.active: gateway", 44 | "storage.records.type: MEMORY" 45 | ); 46 | applicationContextRunner.run(context -> { 47 | assertThat(context).hasSingleBean(InMemoryRecordsStorage.class); 48 | }); 49 | } 50 | 51 | } -------------------------------------------------------------------------------- /plugins/kafka-records-storage/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | } 4 | 5 | jar { 6 | manifest { 7 | attributes( 8 | 'Plugin-Id': "${project.name}", 9 | 'Plugin-Version': "${project.version}", 10 | ) 11 | } 12 | 13 | into('lib') { 14 | from(configurations.runtimeClasspath - configurations.compileOnlyClasspath) 15 | } 16 | } 17 | 18 | tasks.test.dependsOn( 19 | jar, 20 | rootProject.project(":plugins:inmemory-positions-storage").getTasksByName("jar", false) 21 | ) 22 | 23 | dependencies { 24 | compileOnly 'com.google.auto.service:auto-service' 25 | annotationProcessor 'com.google.auto.service:auto-service' 26 | 27 | compileOnly project(":app") 28 | 29 | api 'org.apache.kafka:kafka-clients' 30 | 31 | testImplementation project(":tck") 32 | testImplementation 'org.testcontainers:kafka' 33 | testImplementation 'org.springframework.boot:spring-boot-test' 34 | } -------------------------------------------------------------------------------- /plugins/kafka-records-storage/src/main/java/com/github/bsideup/liiklus/kafka/config/KafkaRecordsStorageConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.kafka.config; 2 | 3 | import com.github.bsideup.liiklus.kafka.KafkaRecordsStorage; 4 | import com.github.bsideup.liiklus.records.RecordsStorage; 5 | import com.github.bsideup.liiklus.util.PropertiesUtil; 6 | import com.google.auto.service.AutoService; 7 | import lombok.Data; 8 | import org.springframework.boot.context.properties.ConfigurationProperties; 9 | import org.springframework.context.ApplicationContextInitializer; 10 | import org.springframework.context.support.GenericApplicationContext; 11 | import org.springframework.core.env.Profiles; 12 | import org.springframework.validation.annotation.Validated; 13 | 14 | import javax.validation.constraints.NotEmpty; 15 | import java.util.HashMap; 16 | import java.util.Map; 17 | 18 | @AutoService(ApplicationContextInitializer.class) 19 | public class KafkaRecordsStorageConfiguration implements ApplicationContextInitializer { 20 | 21 | @Override 22 | public void initialize(GenericApplicationContext applicationContext) { 23 | var environment = applicationContext.getEnvironment(); 24 | 25 | if (!environment.acceptsProfiles(Profiles.of("gateway"))) { 26 | return; 27 | } 28 | 29 | if (!"KAFKA".equals(environment.getRequiredProperty("storage.records.type"))) { 30 | return; 31 | } 32 | 33 | var kafkaProperties = PropertiesUtil.bind(environment, new KafkaProperties()); 34 | 35 | applicationContext.registerBean(KafkaRecordsStorage.class, () -> { 36 | return new KafkaRecordsStorage(kafkaProperties.getBootstrapServers(), kafkaProperties.getProperties()); 37 | }); 38 | } 39 | 40 | @ConfigurationProperties("kafka") 41 | @Data 42 | @Validated 43 | public static class KafkaProperties { 44 | 45 | @NotEmpty 46 | String bootstrapServers; 47 | 48 | Map properties = new HashMap<>(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /plugins/kafka-records-storage/src/test/java/com/github/bsideup/liiklus/kafka/KafkaRecordsStorageTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.kafka; 2 | 3 | import com.github.bsideup.liiklus.ApplicationRunner; 4 | import com.github.bsideup.liiklus.records.RecordStorageTests; 5 | import com.github.bsideup.liiklus.records.RecordsStorage; 6 | import lombok.Getter; 7 | import org.apache.kafka.common.utils.Utils; 8 | import org.springframework.context.ApplicationContext; 9 | import org.testcontainers.containers.KafkaContainer; 10 | import org.testcontainers.utility.DockerImageName; 11 | import reactor.core.publisher.Mono; 12 | 13 | import java.time.Duration; 14 | import java.util.HashMap; 15 | import java.util.Map; 16 | import java.util.UUID; 17 | 18 | public class KafkaRecordsStorageTest implements RecordStorageTests { 19 | 20 | private static final int NUM_OF_PARTITIONS = 4; 21 | 22 | // Generate a set of keys where each key goes to unique partition 23 | public static Map PARTITION_KEYS = Mono.fromCallable(() -> UUID.randomUUID().toString()) 24 | .repeat() 25 | .scanWith( 26 | () -> new HashMap(), 27 | (acc, it) -> { 28 | acc.put(Utils.toPositive(Utils.murmur2(it.getBytes())) % NUM_OF_PARTITIONS, it); 29 | return acc; 30 | } 31 | ) 32 | .filter(it -> it.size() == NUM_OF_PARTITIONS) 33 | .blockFirst(Duration.ofSeconds(10)); 34 | 35 | private static final KafkaContainer kafka = new KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:5.4.3")) 36 | .withEnv("KAFKA_NUM_PARTITIONS", NUM_OF_PARTITIONS + ""); 37 | 38 | static final ApplicationContext applicationContext; 39 | 40 | static { 41 | kafka.start(); 42 | 43 | applicationContext = new ApplicationRunner("KAFKA", "MEMORY") 44 | .withProperty("kafka.bootstrapServers", kafka.getBootstrapServers()) 45 | .run(); 46 | } 47 | 48 | @Getter 49 | RecordsStorage target = applicationContext.getBean(RecordsStorage.class); 50 | 51 | @Getter 52 | String topic = UUID.randomUUID().toString(); 53 | 54 | @Override 55 | public int getNumberOfPartitions() { 56 | return NUM_OF_PARTITIONS; 57 | } 58 | 59 | @Override 60 | public String keyByPartition(int partition) { 61 | return PARTITION_KEYS.get(partition); 62 | } 63 | } -------------------------------------------------------------------------------- /plugins/kafka-records-storage/src/test/java/com/github/bsideup/liiklus/kafka/config/KafkaRecordsStorageConfigurationTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.kafka.config; 2 | 3 | import com.github.bsideup.liiklus.kafka.KafkaRecordsStorage; 4 | import com.github.bsideup.liiklus.positions.PositionsStorage; 5 | import org.junit.jupiter.api.Test; 6 | import org.springframework.beans.BeansException; 7 | import org.springframework.boot.context.properties.bind.validation.BindValidationException; 8 | import org.springframework.boot.test.context.runner.ApplicationContextRunner; 9 | import org.springframework.context.ApplicationContextInitializer; 10 | import org.springframework.context.support.StaticApplicationContext; 11 | 12 | import static org.assertj.core.api.Assertions.assertThat; 13 | 14 | class KafkaRecordsStorageConfigurationTest { 15 | 16 | ApplicationContextRunner applicationContextRunner = new ApplicationContextRunner(() -> new StaticApplicationContext() { 17 | @Override 18 | public void refresh() throws BeansException, IllegalStateException { 19 | } 20 | }) 21 | .withInitializer((ApplicationContextInitializer) new KafkaRecordsStorageConfiguration()); 22 | 23 | @Test 24 | void shouldSkipWhenNotKafka() { 25 | applicationContextRunner.run(context -> { 26 | assertThat(context).doesNotHaveBean(PositionsStorage.class); 27 | }); 28 | } 29 | 30 | @Test 31 | void shouldSkipIfNotGateway() { 32 | applicationContextRunner = applicationContextRunner.withPropertyValues( 33 | "spring.profiles.active: not_gateway", 34 | "storage.records.type: KAFKA" 35 | ); 36 | applicationContextRunner.run(context -> { 37 | assertThat(context).doesNotHaveBean(PositionsStorage.class); 38 | }); 39 | } 40 | 41 | @Test 42 | void shouldValidateProperties() { 43 | applicationContextRunner = applicationContextRunner.withPropertyValues( 44 | "spring.profiles.active: gateway", 45 | "storage.records.type: KAFKA" 46 | ); 47 | applicationContextRunner.run(context -> { 48 | assertThat(context) 49 | .getFailure() 50 | .hasCauseInstanceOf(BindValidationException.class); 51 | }); 52 | applicationContextRunner = applicationContextRunner.withPropertyValues( 53 | "kafka.bootstrapServers: host:9092" 54 | ); 55 | applicationContextRunner.run(context -> { 56 | assertThat(context).hasSingleBean(KafkaRecordsStorage.class); 57 | }); 58 | } 59 | 60 | @Test 61 | void shouldRegisterWhenKafka() { 62 | applicationContextRunner = applicationContextRunner.withPropertyValues( 63 | "spring.profiles.active: gateway", 64 | "storage.records.type: KAFKA", 65 | "kafka.bootstrapServers: host:9092" 66 | ); 67 | applicationContextRunner.run(context -> { 68 | assertThat(context).hasSingleBean(KafkaRecordsStorage.class); 69 | }); 70 | } 71 | 72 | } -------------------------------------------------------------------------------- /plugins/metrics/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | } 4 | 5 | jar { 6 | manifest { 7 | attributes( 8 | 'Plugin-Id': "${project.name}", 9 | 'Plugin-Version': "${project.version}", 10 | ) 11 | } 12 | 13 | into('lib') { 14 | from(configurations.runtimeClasspath - configurations.compileOnlyClasspath) 15 | } 16 | } 17 | 18 | dependencies { 19 | compileOnly 'com.google.auto.service:auto-service' 20 | annotationProcessor 'com.google.auto.service:auto-service' 21 | 22 | compileOnly project(":app") 23 | 24 | api 'io.prometheus:simpleclient_common:0.8.0' 25 | 26 | testImplementation project(":app") 27 | testImplementation 'org.springframework.boot:spring-boot-test' 28 | testImplementation 'org.assertj:assertj-core' 29 | } 30 | -------------------------------------------------------------------------------- /plugins/metrics/src/main/java/com/github/bsideup/liiklus/metrics/MetricsCollector.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.metrics; 2 | 3 | import com.github.bsideup.liiklus.positions.PositionsStorage; 4 | import com.github.bsideup.liiklus.positions.PositionsStorage.Positions; 5 | import io.prometheus.client.Collector.MetricFamilySamples; 6 | import io.prometheus.client.GaugeMetricFamily; 7 | import lombok.RequiredArgsConstructor; 8 | import lombok.experimental.FieldDefaults; 9 | import lombok.extern.slf4j.Slf4j; 10 | import reactor.core.publisher.Flux; 11 | 12 | import java.util.Arrays; 13 | import java.util.Comparator; 14 | 15 | @RequiredArgsConstructor 16 | @FieldDefaults(makeFinal = true) 17 | @Slf4j 18 | public class MetricsCollector { 19 | 20 | PositionsStorage positionsStorage; 21 | 22 | public Flux collect() { 23 | return Flux.from(positionsStorage.findAll()) 24 | .groupBy(it -> it.getGroupId().getName()) 25 | .flatMap(it -> it 26 | .sort(Comparator.comparing(Positions::getGroupId).reversed()) 27 | .index() 28 | ) 29 | .map(tuple -> { 30 | var isLatest = tuple.getT1() == 0; 31 | var positions = tuple.getT2(); 32 | 33 | var gauge = new GaugeMetricFamily("liiklus_topic_position", "", Arrays.asList("topic", "groupName", "groupVersion", "isLatest", "partition")); 34 | 35 | for (var entry : positions.getValues().entrySet()) { 36 | gauge.addMetric( 37 | Arrays.asList( 38 | positions.getTopic(), 39 | positions.getGroupId().getName(), 40 | Integer.toString(positions.getGroupId().getVersion().orElse(0)), 41 | Boolean.toString(isLatest), 42 | entry.getKey().toString() 43 | ), 44 | entry.getValue().doubleValue() 45 | ); 46 | } 47 | 48 | return gauge; 49 | }); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /plugins/metrics/src/main/java/com/github/bsideup/liiklus/metrics/config/MetricsConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.metrics.config; 2 | 3 | import com.github.bsideup.liiklus.metrics.MetricsCollector; 4 | import com.google.auto.service.AutoService; 5 | import io.prometheus.client.exporter.common.TextFormat; 6 | import org.springframework.context.ApplicationContextInitializer; 7 | import org.springframework.context.support.GenericApplicationContext; 8 | import org.springframework.core.env.Profiles; 9 | import org.springframework.http.HttpStatus; 10 | import org.springframework.http.MediaType; 11 | import org.springframework.web.reactive.function.server.RouterFunction; 12 | import org.springframework.web.reactive.function.server.RouterFunctions; 13 | import org.springframework.web.reactive.function.server.ServerResponse; 14 | 15 | import java.io.IOException; 16 | import java.io.StringWriter; 17 | import java.util.Collections; 18 | 19 | @AutoService(ApplicationContextInitializer.class) 20 | public class MetricsConfiguration implements ApplicationContextInitializer { 21 | 22 | @Override 23 | public void initialize(GenericApplicationContext applicationContext) { 24 | var environment = applicationContext.getEnvironment(); 25 | 26 | if (!environment.acceptsProfiles(Profiles.of("exporter"))) { 27 | return; 28 | } 29 | 30 | applicationContext.registerBean(MetricsCollector.class); 31 | 32 | applicationContext.registerBean("prometheus", RouterFunction.class, () -> { 33 | var metricsCollector = applicationContext.getBean(MetricsCollector.class); 34 | return RouterFunctions.route() 35 | .GET("/prometheus", __ -> { 36 | return metricsCollector.collect() 37 | .collectList() 38 | .flatMap(metrics -> { 39 | try { 40 | var writer = new StringWriter(); 41 | TextFormat.write004(writer, Collections.enumeration(metrics)); 42 | return ServerResponse.ok() 43 | .contentType(MediaType.valueOf(TextFormat.CONTENT_TYPE_004)) 44 | .bodyValue(writer.toString()); 45 | } catch (IOException e) { 46 | return ServerResponse.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); 47 | } 48 | }); 49 | }) 50 | .build(); 51 | 52 | }); 53 | } 54 | } -------------------------------------------------------------------------------- /plugins/metrics/src/test/java/com/github/bsideup/liiklus/metrics/config/MetricsConfigurationTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.metrics.config; 2 | 3 | import com.github.bsideup.liiklus.metrics.MetricsCollector; 4 | import org.junit.jupiter.api.Test; 5 | import org.springframework.beans.BeansException; 6 | import org.springframework.boot.test.context.runner.ApplicationContextRunner; 7 | import org.springframework.context.ApplicationContextInitializer; 8 | import org.springframework.context.support.StaticApplicationContext; 9 | 10 | import static org.assertj.core.api.Assertions.assertThat; 11 | 12 | class MetricsConfigurationTest { 13 | 14 | ApplicationContextRunner applicationContextRunner = new ApplicationContextRunner(() -> new StaticApplicationContext() { 15 | @Override 16 | public void refresh() throws BeansException, IllegalStateException { 17 | } 18 | }) 19 | .withInitializer((ApplicationContextInitializer) new MetricsConfiguration()); 20 | 21 | @Test 22 | void shouldNotBeEnabledByDefault() { 23 | applicationContextRunner.run(context -> { 24 | assertThat(context).doesNotHaveBean(MetricsCollector.class); 25 | }); 26 | } 27 | 28 | @Test 29 | void shouldRequireExporterProfile() { 30 | applicationContextRunner = applicationContextRunner.withPropertyValues( 31 | "spring.profiles.active: not_exporter" 32 | ); 33 | applicationContextRunner.run(context -> { 34 | assertThat(context).doesNotHaveBean(MetricsCollector.class); 35 | }); 36 | } 37 | 38 | @Test 39 | void shouldRegisterWhenExporter() { 40 | applicationContextRunner = applicationContextRunner.withPropertyValues( 41 | "spring.profiles.active: exporter" 42 | ); 43 | applicationContextRunner.run(context -> { 44 | assertThat(context).hasSingleBean(MetricsCollector.class); 45 | }); 46 | } 47 | } -------------------------------------------------------------------------------- /plugins/pulsar-records-storage/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | } 4 | 5 | jar { 6 | manifest { 7 | attributes( 8 | 'Plugin-Id': "${project.name}", 9 | 'Plugin-Version': "${project.version}", 10 | ) 11 | } 12 | 13 | into('lib') { 14 | from(configurations.runtimeClasspath - configurations.compileOnlyClasspath) 15 | } 16 | } 17 | 18 | tasks.test.dependsOn( 19 | jar, 20 | rootProject.project(":plugins:inmemory-positions-storage").getTasksByName("jar", false) 21 | ) 22 | 23 | dependencies { 24 | compileOnly 'com.google.auto.service:auto-service' 25 | annotationProcessor 'com.google.auto.service:auto-service' 26 | 27 | compileOnly project(":app") 28 | 29 | api 'org.apache.pulsar:pulsar-client-original:2.4.0' 30 | 31 | testImplementation project(":tck") 32 | testImplementation 'org.testcontainers:pulsar' 33 | testImplementation 'org.apache.pulsar:pulsar-client-admin-original:2.4.0' 34 | testImplementation 'org.springframework.boot:spring-boot-test' 35 | } 36 | -------------------------------------------------------------------------------- /plugins/pulsar-records-storage/src/main/java/com/github/bsideup/liiklus/pulsar/config/PulsarRecordsStorageConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.pulsar.config; 2 | 3 | import com.github.bsideup.liiklus.pulsar.PulsarRecordsStorage; 4 | import com.github.bsideup.liiklus.records.RecordsStorage; 5 | import com.github.bsideup.liiklus.util.PropertiesUtil; 6 | import com.google.auto.service.AutoService; 7 | import lombok.Data; 8 | import lombok.SneakyThrows; 9 | import org.apache.pulsar.client.api.PulsarClient; 10 | import org.apache.pulsar.client.api.PulsarClientException; 11 | import org.springframework.boot.context.properties.ConfigurationProperties; 12 | import org.springframework.context.ApplicationContextInitializer; 13 | import org.springframework.context.support.GenericApplicationContext; 14 | import org.springframework.core.env.Profiles; 15 | import org.springframework.validation.annotation.Validated; 16 | 17 | import javax.validation.constraints.NotEmpty; 18 | import java.util.Map; 19 | import java.util.Optional; 20 | 21 | @AutoService(ApplicationContextInitializer.class) 22 | public class PulsarRecordsStorageConfiguration implements ApplicationContextInitializer { 23 | 24 | @Override 25 | public void initialize(GenericApplicationContext applicationContext) { 26 | var environment = applicationContext.getEnvironment(); 27 | 28 | if (!environment.acceptsProfiles(Profiles.of("gateway"))) { 29 | return; 30 | } 31 | 32 | if (!"PULSAR".equals(environment.getRequiredProperty("storage.records.type"))) { 33 | return; 34 | } 35 | 36 | var pulsarProperties = PropertiesUtil.bind(environment, new PulsarProperties()); 37 | 38 | applicationContext.registerBean(PulsarRecordsStorage.class, () -> { 39 | return new PulsarRecordsStorage(createClient(pulsarProperties)); 40 | }); 41 | } 42 | 43 | @SneakyThrows 44 | PulsarClient createClient(PulsarProperties pulsarProperties) { 45 | var clientBuilder = PulsarClient.builder() 46 | .serviceUrl(pulsarProperties.getServiceUrl()); 47 | 48 | pulsarProperties.getTlsTrustCertsFilePath().ifPresent(clientBuilder::tlsTrustCertsFilePath); 49 | pulsarProperties.getAuthPluginClassName().ifPresent(authClass -> { 50 | try { 51 | clientBuilder.authentication(authClass, pulsarProperties.getAuthPluginParams()); 52 | } catch (PulsarClientException.UnsupportedAuthenticationException e) { 53 | throw new IllegalStateException(e); 54 | } 55 | }); 56 | 57 | return clientBuilder.build(); 58 | } 59 | 60 | @ConfigurationProperties("pulsar") 61 | @Data 62 | @Validated 63 | static class PulsarProperties { 64 | 65 | @NotEmpty 66 | String serviceUrl; 67 | 68 | Optional tlsTrustCertsFilePath = Optional.empty(); 69 | 70 | Optional authPluginClassName = Optional.empty(); 71 | 72 | Map authPluginParams = Map.of(); 73 | 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /plugins/pulsar-records-storage/src/main/java/org/apache/pulsar/client/impl/ConsumerImplAccessor.java: -------------------------------------------------------------------------------- 1 | package org.apache.pulsar.client.impl; 2 | 3 | import org.apache.pulsar.client.api.Consumer; 4 | import org.apache.pulsar.client.api.MessageId; 5 | 6 | import java.util.concurrent.CompletableFuture; 7 | 8 | public class ConsumerImplAccessor { 9 | 10 | public static CompletableFuture getLastMessageIdAsync(Consumer consumer) { 11 | return ((ConsumerImpl) consumer).getLastMessageIdAsync(); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /plugins/pulsar-records-storage/src/test/java/com/github/bsideup/liiklus/pulsar/AbstractPulsarRecordsStorageTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.pulsar; 2 | 3 | import com.github.bsideup.liiklus.ApplicationRunner; 4 | import com.github.bsideup.liiklus.records.RecordStorageTests; 5 | import com.github.bsideup.liiklus.records.RecordsStorage; 6 | import com.github.bsideup.liiklus.support.DisabledUntil; 7 | import lombok.Getter; 8 | import org.apache.pulsar.client.api.PulsarClient; 9 | import org.apache.pulsar.client.impl.MessageIdImpl; 10 | import org.junit.jupiter.api.Test; 11 | import org.springframework.context.ApplicationContext; 12 | import org.testcontainers.containers.PulsarContainer; 13 | import org.testcontainers.utility.DockerImageName; 14 | 15 | import java.time.Duration; 16 | import java.time.Instant; 17 | import java.time.temporal.ChronoUnit; 18 | import java.util.UUID; 19 | 20 | import static org.assertj.core.api.Assertions.assertThat; 21 | 22 | abstract class AbstractPulsarRecordsStorageTest implements RecordStorageTests { 23 | 24 | static final PulsarContainer pulsar = new PulsarContainer(DockerImageName.parse("apachepulsar/pulsar:2.5.0")) 25 | .withReuse(true); 26 | 27 | private static final ApplicationContext applicationContext; 28 | 29 | static { 30 | pulsar.start(); 31 | 32 | applicationContext = new ApplicationRunner("PULSAR", "MEMORY") 33 | .withProperty("pulsar.serviceUrl", pulsar.getPulsarBrokerUrl()) 34 | .run(); 35 | } 36 | 37 | @Getter 38 | RecordsStorage target = applicationContext.getBean(RecordsStorage.class); 39 | 40 | @Getter 41 | String topic = UUID.randomUUID().toString(); 42 | 43 | @Override 44 | @Test 45 | @DisabledUntil(value = "2021-09-01", comment = "#180 - Pulsar should fix the way seek works, not disconnecting consumers (apache/pulsar/pull/5022)") 46 | public void shouldAlwaysUseEarliestOffsetOnEmptyOffsetsInTheInitialProvider() { 47 | RecordStorageTests.super.shouldAlwaysUseEarliestOffsetOnEmptyOffsetsInTheInitialProvider(); 48 | } 49 | 50 | @Test 51 | void shouldPreferEventTimeOverPublishTime() throws Exception { 52 | var topic = getTopic(); 53 | var eventTimestamp = Instant.now().minusSeconds(1000).truncatedTo(ChronoUnit.MILLIS); 54 | 55 | int partition; 56 | try ( 57 | var pulsarClient = PulsarClient.builder() 58 | .serviceUrl(pulsar.getPulsarBrokerUrl()) 59 | .build() 60 | ) { 61 | var messageId = pulsarClient.newProducer() 62 | .topic(topic) 63 | .create() 64 | .newMessage() 65 | .value("hello".getBytes()) 66 | .eventTime(eventTimestamp.toEpochMilli()) 67 | .send(); 68 | 69 | partition = ((MessageIdImpl) messageId).getPartitionIndex(); 70 | } 71 | 72 | var record = subscribeToPartition(partition) 73 | .flatMap(RecordsStorage.PartitionSource::getPublisher) 74 | .blockFirst(Duration.ofSeconds(10)); 75 | 76 | assertThat(record).satisfies(it -> { 77 | assertThat(it.getTimestamp()).isEqualTo(eventTimestamp); 78 | }); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /plugins/pulsar-records-storage/src/test/java/com/github/bsideup/liiklus/pulsar/NonPartitionedPulsarRecordsStorageTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.pulsar; 2 | 3 | import lombok.SneakyThrows; 4 | import org.apache.pulsar.client.admin.PulsarAdmin; 5 | 6 | public class NonPartitionedPulsarRecordsStorageTest extends AbstractPulsarRecordsStorageTest { 7 | 8 | @SneakyThrows 9 | public NonPartitionedPulsarRecordsStorageTest() { 10 | PulsarAdmin pulsarAdmin = PulsarAdmin.builder() 11 | .serviceHttpUrl(pulsar.getHttpServiceUrl()) 12 | .build(); 13 | 14 | pulsarAdmin.topics().createNonPartitionedTopic(topic); 15 | } 16 | 17 | @Override 18 | public String keyByPartition(int partition) { 19 | return "foo"; 20 | } 21 | 22 | @Override 23 | public int getNumberOfPartitions() { 24 | return 1; 25 | } 26 | } -------------------------------------------------------------------------------- /plugins/pulsar-records-storage/src/test/java/com/github/bsideup/liiklus/pulsar/PulsarRecordsStorageTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.pulsar; 2 | 3 | import lombok.SneakyThrows; 4 | import org.apache.pulsar.client.admin.PulsarAdmin; 5 | import org.apache.pulsar.client.impl.Murmur3_32Hash; 6 | import org.apache.pulsar.client.util.MathUtils; 7 | import reactor.core.publisher.Mono; 8 | 9 | import java.time.Duration; 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | import java.util.UUID; 13 | 14 | public class PulsarRecordsStorageTest extends AbstractPulsarRecordsStorageTest { 15 | 16 | private static final int NUM_OF_PARTITIONS = 4; 17 | 18 | // Generate a set of keys where each key goes to unique partition 19 | public static Map PARTITION_KEYS = Mono.fromCallable(() -> UUID.randomUUID().toString()) 20 | .repeat() 21 | .scanWith( 22 | () -> new HashMap(), 23 | (acc, it) -> { 24 | acc.put(MathUtils.signSafeMod(Murmur3_32Hash.getInstance().makeHash(it), NUM_OF_PARTITIONS), it); 25 | return acc; 26 | } 27 | ) 28 | .filter(it -> it.size() == NUM_OF_PARTITIONS) 29 | .blockFirst(Duration.ofSeconds(10)); 30 | 31 | @SneakyThrows 32 | public PulsarRecordsStorageTest() { 33 | PulsarAdmin pulsarAdmin = PulsarAdmin.builder() 34 | .serviceHttpUrl(pulsar.getHttpServiceUrl()) 35 | .build(); 36 | 37 | pulsarAdmin.topics().createPartitionedTopic(topic, getNumberOfPartitions()); 38 | } 39 | 40 | @Override 41 | public String keyByPartition(int partition) { 42 | return PARTITION_KEYS.get(partition); 43 | } 44 | 45 | @Override 46 | public int getNumberOfPartitions() { 47 | return NUM_OF_PARTITIONS; 48 | } 49 | } -------------------------------------------------------------------------------- /plugins/pulsar-records-storage/src/test/java/com/github/bsideup/liiklus/pulsar/container/PulsarTlsContainer.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.pulsar.container; 2 | 3 | import org.testcontainers.containers.PulsarContainer; 4 | import org.testcontainers.containers.output.Slf4jLogConsumer; 5 | import org.testcontainers.containers.wait.strategy.Wait; 6 | import org.testcontainers.utility.DockerImageName; 7 | import org.testcontainers.utility.MountableFile; 8 | 9 | import java.nio.file.Path; 10 | import java.nio.file.Paths; 11 | 12 | public class PulsarTlsContainer extends PulsarContainer { 13 | 14 | public static final int BROKER_TLS_PORT = 6651; 15 | 16 | public PulsarTlsContainer() { 17 | super(DockerImageName.parse("apachepulsar/pulsar:2.5.0")); 18 | withExposedPorts(BROKER_TLS_PORT, BROKER_HTTP_PORT); 19 | withEnv("PULSAR_PREFIX_brokerServicePortTls", BROKER_TLS_PORT + ""); 20 | withEnv("PULSAR_PREFIX_tlsEnabled", "true"); 21 | withEnv("PULSAR_PREFIX_tlsCertificateFilePath", "/pulsar/broker.cert.pem"); 22 | withEnv("PULSAR_PREFIX_tlsKeyFilePath", "/pulsar/broker.key-pk8.pem"); 23 | withEnv("PULSAR_PREFIX_tlsTrustCertsFilePath", "/pulsar/ca.cert.pem"); 24 | 25 | withCopyFileToContainer(MountableFile.forClasspathResource("certs/"), "/pulsar/"); 26 | 27 | setCommand( 28 | "/bin/bash", 29 | "-c", 30 | "bin/apply-config-from-env.py conf/standalone.conf && " + 31 | "bin/apply-config-from-env.py conf/proxy.conf && " + 32 | "bin/pulsar standalone --no-functions-worker -nss" 33 | ); 34 | 35 | waitingFor(Wait.forLogMessage(".*Created namespace public\\/default.*", 1)); 36 | } 37 | 38 | public Path getCaCert() { 39 | return Paths.get("src", "test", "resources", "certs").resolve("ca.cert.pem"); 40 | } 41 | 42 | @Override 43 | public String getPulsarBrokerUrl() { 44 | return String.format("pulsar+ssl://%s:%s", getContainerIpAddress(), getMappedPort(BROKER_TLS_PORT)); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /plugins/pulsar-records-storage/src/test/resources/.htpasswd: -------------------------------------------------------------------------------- 1 | super:$apr1$6s/xhL9h$Ow3i695wwrEc1mYq/BG2q/ -------------------------------------------------------------------------------- /plugins/pulsar-records-storage/src/test/resources/certs/README.md: -------------------------------------------------------------------------------- 1 | Files are taken from Pulsar's example: 2 | https://github.com/apache/pulsar/tree/master/tests/docker-images/latest-version-image/ssl -------------------------------------------------------------------------------- /plugins/pulsar-records-storage/src/test/resources/certs/broker.cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIEkDCCAnigAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGZm9v 3 | YmFyMCAXDTE4MDYyMjA4NTUzMloYDzIyOTIwNDA2MDg1NTMyWjAjMSEwHwYDVQQD 4 | DBhicm9rZXIucHVsc2FyLmFwYWNoZS5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IB 5 | DwAwggEKAoIBAQDQouKhZah4hMCqmg4aS5RhQG/Y1gA+yP9DGF9mlw35tfhfWs63 6 | EvNjEK4L/ZWSEV45L/wc6YV14RmM6bJ0V/0vXo4xmISbqptND/2kRIspkLZQ5F0O 7 | OQXVicqZLOc6igZQhRg8ANDYdTJUTF65DqauX4OJt3YMhF2FSt7jQtlj06IQBa01 8 | +ARO9OotMJtBY+vIU5bV6JydfgkhQH9rIDI7AMeY5j02gGkJJrelfm+WoOsUez+X 9 | aqTN3/tF8+MBcFB3G04s1qc2CJPJM3YGxvxEtHqTGI14t9J8p5O7X9JHpcY8X00s 10 | bxa4FGbKgfDobbkJ+GgblWCkAcLN95sKTqtHAgMBAAGjgd0wgdowCQYDVR0TBAIw 11 | ADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2Vu 12 | ZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUaxFvJrkEGqk8azTA 13 | DyVyTyTbJAIwQQYDVR0jBDowOIAUVwvpyyPov0c+UHo/RX6hGEOdFSehFaQTMBEx 14 | DzANBgNVBAMMBmZvb2JhcoIJANfih0+geeIMMA4GA1UdDwEB/wQEAwIFoDATBgNV 15 | HSUEDDAKBggrBgEFBQcDATANBgkqhkiG9w0BAQsFAAOCAgEA35QDGclHzQtHs3yQ 16 | ZzNOSKisg5srTiIoQgRzfHrXfkthNFCnBzhKjBxqk3EIasVtvyGuk0ThneC1ai3y 17 | ZK3BivnMZfm1SfyvieFoqWetsxohWfcpOSVkpvO37P6v/NmmaTIGkBN3gxKCx0QN 18 | zqApLQyNTM++X3wxetYH/afAGUrRmBGWZuJheQpB9yZ+FB6BRp8YuYIYBzANJyW9 19 | spvXW03TpqX2AIoRBoGMLzK72vbhAbLWiCIfEYREhbZVRkP+yvD338cWrILlOEur 20 | x/n8L/FTmbf7mXzHg4xaQ3zg/5+0OCPMDPUBE4xWDBAbZ82hgOcTqfVjwoPgo2V0 21 | fbbx6redq44J3Vn5d9Xhi59fkpqEjHpX4xebr5iMikZsNTJMeLh0h3uf7DstuO9d 22 | mfnF5j+yDXCKb9XzCsTSvGCN+spmUh6RfSrbkw8/LrRvBUpKVEM0GfKSnaFpOaSS 23 | efM4UEi72FRjszzHEkdvpiLhYvihINLJmDXszhc3fCi42be/DGmUhuhTZWynOPmp 24 | 0N0V/8/sGT5gh4fGEtGzS/8xEvZwO9uDlccJiG8Pi+aO0/K9urB9nppd/xKWXv3C 25 | cib/QrW0Qow4TADWC1fnGYCpFzzaZ2esPL2MvzOYXnW4/AbEqmb6Weatluai64ZK 26 | 3N2cGJWRyvpvvmbP2hKCa4eLgEc= 27 | -----END CERTIFICATE----- 28 | -------------------------------------------------------------------------------- /plugins/pulsar-records-storage/src/test/resources/certs/broker.key-pk8.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDQouKhZah4hMCq 3 | mg4aS5RhQG/Y1gA+yP9DGF9mlw35tfhfWs63EvNjEK4L/ZWSEV45L/wc6YV14RmM 4 | 6bJ0V/0vXo4xmISbqptND/2kRIspkLZQ5F0OOQXVicqZLOc6igZQhRg8ANDYdTJU 5 | TF65DqauX4OJt3YMhF2FSt7jQtlj06IQBa01+ARO9OotMJtBY+vIU5bV6Jydfgkh 6 | QH9rIDI7AMeY5j02gGkJJrelfm+WoOsUez+XaqTN3/tF8+MBcFB3G04s1qc2CJPJ 7 | M3YGxvxEtHqTGI14t9J8p5O7X9JHpcY8X00sbxa4FGbKgfDobbkJ+GgblWCkAcLN 8 | 95sKTqtHAgMBAAECggEBALE1eMtfnk3nbAI74bih84D7C0Ug14p8jJv/qqBnsx4j 9 | WrgbWDMVrJa7Rym2FQHBMMfgIwKnso0iSeJvaPz683j1lk833YKe0VQOPgD1m0IN 10 | wV1J6mQ3OOZcKDIcerY1IBHqSmBEzR7dxIbnaxlCAX9gb0hdBK6zCwA5TMG5OQ5Y 11 | 3cGOmevK5i2PiejhpruA8h7E48P1ATaGHUZif9YD724oi6AcilQ8H/DlOjZTvlmK 12 | r4aJ30f72NwGM8Ecet5CE2wyflAGtY0k+nChYkPRfy54u64Z/T9B53AvneFaj8jv 13 | yFepZgRTs2cWhEl0KQGuBHQ4+IeOfMt2LebhvjWW8YkCgYEA7BXVsnqPHKRDd8wP 14 | eNkolY4Fjdq4wu9ad+DaFiZcJuv7ugr+Kplltq6e4aU36zEdBYdPp/6KM/HGE/Xj 15 | bo0CELNUKs/Ny9H/UJc8DDbVEmoF3XGiIbKKq1T8NTXTETFnwrGkBFD8nl7YTsOF 16 | M4FZmSok0MhhkpEULAqxBS6YpQsCgYEA4jxM1egTVSWjTreg2UdYo2507jKa7maP 17 | PRtoPsNJzWNbOpfj26l3/8pd6oYKWck6se6RxIUxUrk3ywhNJIIOvWEC7TaOH1c9 18 | T4NQNcweqBW9+A1x5gyzT14gDaBfl45gs82vI+kcpVv/w2N3HZOQZX3yAUqWpfw2 19 | yw1uQDXtgDUCgYEAiYPWbBXTkp1j5z3nrT7g0uxc89n5USLWkYlZvxktCEbg4+dP 20 | UUT06EoipdD1F3wOKZA9p98uZT9pX2sUxOpBz7SFTEKq3xQ9IZZWFc9CoW08aVat 21 | V++FsnLYTa5CeXtLsy6CGTmLTDx2xrpAtlWb+QmBVFPD8fmrxFOd9STFKS0CgYAt 22 | 6ztVN3OlFqyc75yQPXD6SxMkvdTAisSMDKIOCylRrNb5f5baIP2gR3zkeyxiqPtm 23 | 3htsHfSy67EtXpP50wQW4Dft2eLi7ZweJXMEWFfomfEjBeeWYAGNHHe5DFIauuVZ 24 | 2WexDEGqNpAlIm0s7aSjVPrn1DHbouOkNyenlMqN+QKBgQDVYVhk9widShSnCmUA 25 | G30moXDgj3eRqCf5T7NEr9GXD1QBD/rQSPh5agnDV7IYLpV7/wkYLI7l9x7mDwu+ 26 | I9mRXkyAmTVEctLTdXQHt0jdJa5SfUaVEDUzQbr0fUjkmythTvqZ809+d3ELPeLI 27 | 5qJ7jxgksHWji4lYfL4r4J6Zaw== 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /plugins/pulsar-records-storage/src/test/resources/certs/ca.cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFCDCCAvCgAwIBAgIJANfih0+geeIMMA0GCSqGSIb3DQEBCwUAMBExDzANBgNV 3 | BAMMBmZvb2JhcjAeFw0xODA2MjIwODQ2MjFaFw0zODA2MTcwODQ2MjFaMBExDzAN 4 | BgNVBAMMBmZvb2JhcjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOVU 5 | UpTPeXCeyfUiQS824l9s9krZd4R6TA4D97eQ9EWm2D7ppV4gPApHO8j5f+joo/b6 6 | Iso4aFlHpJ8VV2a5Ol7rjQw43MJHaBgwDxB1XWgsNdfoI7ebtp/BWg2nM3r8wm+Z 7 | gKenf9d1/1Ol+6yFUehkLkIXUvldiVegmmje8FnwhcDNE1eTrh66XqSJXEXqgBKu 8 | NqsoYcVak72OyOO1/N8CESoSdyBkbSiH5vJyo0AUCjn7tULga7fxojmqBZDog9Pg 9 | e5Fi/hbCrdinbxBrMgIxQ7wqXw2sw6iOWu4FU8Ih/CuF4xaQy2YP7MEk4Ff0LCY0 10 | KMhFMWU7550r/fz/C2l7fKhREyCQPa/bVE+dfxgZ/gCZ+p7vQ154hCCjpd+5bECv 11 | SN1bcVIPG6ngQu4vMXa7QRBi/Od40jSVGVJXYY6kXvrYatad7035w2GGGGkvMsQm 12 | y53yh4tqQfH7ulHqB0J5LebTQRp6nRizWigVCLjNkxJYI+Dj51qvT1zdyWEegKr1 13 | CthBfYzXlfjeH3xri1f0UABeC12n24Wkacd9af7zs7S3rYntEK444w/3fB0F62Lh 14 | SESfMLAmUH0dF5plRShrFUXz23nUeS8EYgWmnGkpf/HDzB67vdfAK0tfJEtmmY78 15 | q06OSgMr+AOOqaomh4Ez2ZQG592bS71G8MrE7r2/AgMBAAGjYzBhMB0GA1UdDgQW 16 | BBRXC+nLI+i/Rz5Qej9FfqEYQ50VJzAfBgNVHSMEGDAWgBRXC+nLI+i/Rz5Qej9F 17 | fqEYQ50VJzAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG 18 | 9w0BAQsFAAOCAgEAYd2PxdV+YOaWcmMG1fK7CGwSzDOGsgC7hi4gWPiNsVbz6fwQ 19 | m5Ac7Zw76dzin8gzOPKST7B8WIoc7ZWrMnyh3G6A3u29Ec8iWahqGa91NPA3bOIl 20 | 0ldXnXfa416+JL/Q5utpiV6W2XDaB53v9GqpMk4rOTS9kCFOiuH5ZU8P69jp9mq6 21 | 7pI/+hWFr+21ibmXH6ANxRLd/5+AqojRUYowAu2997Z+xmbpwx/2Svciq3LNY/Vz 22 | s9DudUHCBHj/DPgNxsEUt8QNohjQkRbFTY0a1aXodJ/pm0Ehk2kf9KwYYYduR7ak 23 | 6UmPIPrZg6FePNahxwMZ0RtgX7EXmpiiIH1q9BsulddWkrFQclevsWO3ONQVrDs2 24 | gwY0HQuCRCJ+xgS2cyGiGohW5MkIsg1aI0i0j5GIUSppCIYgirAGCairARbCjhcx 25 | pbMe8RTuBhCqO3R2wZ0wXu7P7/ArI/Ltm1dU6IeHUAUmeneVj5ie0SdA19mHTS2o 26 | lG77N0jy6eq2zyEwJE6tuS/tyP1xrxdzXCYY7f6X9aNfsuPVQTcnrFajvDv8R6uD 27 | YnRStVCdS6fZEP0JzsLrqp9bgLIRRsiqsVVBCgJdK1I/X59qk2EyCLXWSgk8T9XZ 28 | iux8LlPpskt30YYt1KhlWB9zVz7k0uYAwits5foU6RfCRDPAyOa1q/QOXk0= 29 | -----END CERTIFICATE----- 30 | -------------------------------------------------------------------------------- /plugins/redis-positions-storage/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | } 4 | 5 | jar { 6 | manifest { 7 | attributes( 8 | 'Plugin-Id': "${project.name}", 9 | 'Plugin-Version': "${project.version}", 10 | ) 11 | } 12 | 13 | into('lib') { 14 | from(configurations.runtimeClasspath - configurations.compileOnlyClasspath) 15 | } 16 | } 17 | 18 | tasks.test.dependsOn( 19 | jar, 20 | rootProject.project(":plugins:inmemory-records-storage").getTasksByName("jar", false) 21 | ) 22 | 23 | dependencies { 24 | compileOnly 'com.google.auto.service:auto-service' 25 | annotationProcessor 'com.google.auto.service:auto-service' 26 | 27 | compileOnly project(":app") 28 | 29 | api 'io.lettuce:lettuce-core' 30 | 31 | testImplementation project(":tck") 32 | testImplementation 'org.springframework.boot:spring-boot-test' 33 | testImplementation 'org.testcontainers:testcontainers' 34 | } -------------------------------------------------------------------------------- /plugins/redis-positions-storage/src/main/java/com/github/bsideup/liiklus/positions/redis/config/RedisPositionsConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.positions.redis.config; 2 | 3 | import com.github.bsideup.liiklus.positions.redis.RedisPositionsStorage; 4 | import com.github.bsideup.liiklus.util.PropertiesUtil; 5 | import com.google.auto.service.AutoService; 6 | import io.lettuce.core.RedisClient; 7 | import io.lettuce.core.RedisURI; 8 | import io.lettuce.core.codec.StringCodec; 9 | import lombok.Data; 10 | import lombok.extern.slf4j.Slf4j; 11 | import org.springframework.boot.context.properties.ConfigurationProperties; 12 | import org.springframework.context.ApplicationContextInitializer; 13 | import org.springframework.context.support.GenericApplicationContext; 14 | import org.springframework.validation.annotation.Validated; 15 | import reactor.core.publisher.Mono; 16 | 17 | import javax.validation.constraints.Min; 18 | import javax.validation.constraints.NotEmpty; 19 | 20 | @Slf4j 21 | @AutoService(ApplicationContextInitializer.class) 22 | public class RedisPositionsConfiguration implements ApplicationContextInitializer { 23 | 24 | @Override 25 | public void initialize(GenericApplicationContext applicationContext) { 26 | var environment = applicationContext.getEnvironment(); 27 | 28 | var type = environment.getRequiredProperty("storage.positions.type"); 29 | if(!"REDIS".equals(type)) { 30 | return; 31 | } 32 | 33 | var redisProperties = PropertiesUtil.bind(environment, new RedisProperties()); 34 | 35 | applicationContext.registerBean(RedisPositionsStorage.class, () -> { 36 | var redisURI = RedisURI.builder() 37 | .withHost(redisProperties.getHost()) 38 | .withPort(redisProperties.getPort()) 39 | .build(); 40 | 41 | return new RedisPositionsStorage( 42 | Mono 43 | .fromCompletionStage(() -> RedisClient.create().connectAsync(StringCodec.UTF8, redisURI)) 44 | .cache(), 45 | redisProperties.getPositionsProperties().getPrefix() 46 | ); 47 | }); 48 | } 49 | 50 | @ConfigurationProperties("redis") 51 | @Data 52 | @Validated 53 | public static class RedisProperties { 54 | 55 | @NotEmpty 56 | String host; 57 | 58 | @Min(1) 59 | int port = -1; 60 | 61 | PositionsProperties positionsProperties = new PositionsProperties(); 62 | 63 | @Data 64 | @Validated 65 | static class PositionsProperties { 66 | 67 | @NotEmpty 68 | String prefix = "liiklus:positions:"; 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /plugins/redis-positions-storage/src/test/java/com/github/bsideup/liiklus/positions/redis/RedisPositionsStorageTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.positions.redis; 2 | 3 | import com.github.bsideup.liiklus.ApplicationRunner; 4 | import com.github.bsideup.liiklus.positions.GroupId; 5 | import com.github.bsideup.liiklus.positions.PositionsStorage; 6 | import com.github.bsideup.liiklus.positions.PositionsStorage.Positions; 7 | import com.github.bsideup.liiklus.positions.PositionsStorageTests; 8 | import io.lettuce.core.RedisClient; 9 | import io.lettuce.core.RedisURI; 10 | import io.lettuce.core.api.sync.RedisCommands; 11 | import lombok.Getter; 12 | import org.junit.jupiter.api.Test; 13 | import org.junit.jupiter.api.TestInfo; 14 | import org.springframework.context.ApplicationContext; 15 | import org.testcontainers.containers.GenericContainer; 16 | import reactor.core.publisher.Flux; 17 | 18 | import java.time.Duration; 19 | 20 | import static org.assertj.core.api.Assertions.assertThat; 21 | 22 | class RedisPositionsStorageTest implements PositionsStorageTests { 23 | 24 | static final GenericContainer redis = new GenericContainer("redis:3.0.6") 25 | .withExposedPorts(6379); 26 | 27 | static final ApplicationContext applicationContext; 28 | 29 | static { 30 | redis.start(); 31 | 32 | applicationContext = new ApplicationRunner("MEMORY", "REDIS") 33 | .withProperty("redis.host", redis.getContainerIpAddress()) 34 | .withProperty("redis.port", redis.getMappedPort(6379) + "") 35 | .run(); 36 | } 37 | 38 | static final RedisClient redisClient = RedisClient.create( 39 | RedisURI.builder() 40 | .withHost(redis.getContainerIpAddress()) 41 | .withPort(redis.getMappedPort(6379)) 42 | .build() 43 | ); 44 | 45 | static final RedisCommands redisCommands = redisClient.connect().sync(); 46 | 47 | @Getter 48 | PositionsStorage storage = applicationContext.getBean(PositionsStorage.class); 49 | 50 | @Test 51 | void should_skip_keys_without_prefix(TestInfo testInfo) { 52 | // This test assumes an empty DB 53 | redisCommands.flushall(); 54 | 55 | var topicName = getTopicName(testInfo); 56 | redisCommands.set("CORRUPTED_KEY", "SUSPICIOUS_VALUE"); 57 | storage.update(topicName, GroupId.ofString("mygroup-v1"), 0, 0); 58 | 59 | var positions = Flux.from(storage.findAll()) 60 | .collectList() 61 | .block(Duration.ofSeconds(5)); 62 | 63 | assertThat(positions) 64 | .extracting(Positions::getTopic) 65 | .containsOnly(topicName); 66 | } 67 | 68 | private String getTopicName(TestInfo testInfo) { 69 | return testInfo.getTestMethod().get().getName() + "topic_name"; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /plugins/redis-positions-storage/src/test/java/com/github/bsideup/liiklus/positions/redis/config/RedisPositionsConfigurationTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.positions.redis.config; 2 | 3 | import com.github.bsideup.liiklus.positions.PositionsStorage; 4 | import com.github.bsideup.liiklus.positions.redis.RedisPositionsStorage; 5 | import org.junit.jupiter.api.Test; 6 | import org.springframework.beans.BeansException; 7 | import org.springframework.boot.context.properties.bind.validation.BindValidationException; 8 | import org.springframework.boot.test.context.runner.ApplicationContextRunner; 9 | import org.springframework.context.ApplicationContextInitializer; 10 | import org.springframework.context.support.StaticApplicationContext; 11 | 12 | import static org.assertj.core.api.Assertions.assertThat; 13 | 14 | class RedisPositionsConfigurationTest { 15 | 16 | ApplicationContextRunner applicationContextRunner = new ApplicationContextRunner(() -> new StaticApplicationContext() { 17 | @Override 18 | public void refresh() throws BeansException, IllegalStateException { 19 | } 20 | }) 21 | .withInitializer((ApplicationContextInitializer) new RedisPositionsConfiguration()); 22 | 23 | @Test 24 | void should_skip_when_position_storage_not_redis() { 25 | applicationContextRunner = applicationContextRunner.withPropertyValues( 26 | "storage.positions.type: FOO" 27 | ); 28 | applicationContextRunner.run(context -> { 29 | assertThat(context).doesNotHaveBean(PositionsStorage.class); 30 | }); 31 | } 32 | 33 | @Test 34 | void should_validate_properties() { 35 | applicationContextRunner = applicationContextRunner.withPropertyValues( 36 | "storage.positions.type: REDIS" 37 | ); 38 | applicationContextRunner.run(context -> { 39 | assertThat(context) 40 | .getFailure() 41 | .hasCauseInstanceOf(BindValidationException.class); 42 | }); 43 | applicationContextRunner = applicationContextRunner.withPropertyValues( 44 | "redis.host: host" 45 | ); 46 | applicationContextRunner.run(context -> { 47 | assertThat(context) 48 | .getFailure() 49 | .hasCauseInstanceOf(BindValidationException.class); 50 | }); 51 | applicationContextRunner = applicationContextRunner.withPropertyValues( 52 | "redis.port: 8888" 53 | ); 54 | applicationContextRunner.run(context -> { 55 | assertThat(context).hasNotFailed(); 56 | }); 57 | } 58 | 59 | @Test 60 | void should_register_positions_storage_bean_when_type_is_redis() { 61 | applicationContextRunner = applicationContextRunner.withPropertyValues( 62 | "storage.positions.type: REDIS", 63 | "redis.host: host", 64 | "redis.port: 8888" 65 | ); 66 | applicationContextRunner.run(context -> { 67 | assertThat(context).hasSingleBean(RedisPositionsStorage.class); 68 | }); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /plugins/rsocket-transport/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'idea' 3 | id 'java-library' 4 | id 'com.google.protobuf' 5 | } 6 | 7 | protobuf { 8 | protoc { 9 | artifact = 'com.google.protobuf:protoc' 10 | } 11 | 12 | generatedFilesBaseDir = "$projectDir/generated" 13 | 14 | plugins { 15 | rsocketRpc { 16 | artifact = 'io.rsocket.rpc:rsocket-rpc-protobuf' 17 | } 18 | } 19 | 20 | generateProtoTasks { 21 | ofSourceSet('main').each { task -> 22 | task.builtins { 23 | remove java 24 | } 25 | task.plugins { 26 | rsocketRpc { } 27 | } 28 | } 29 | } 30 | } 31 | 32 | clean { 33 | delete protobuf.generatedFilesBaseDir 34 | } 35 | 36 | idea { 37 | module { 38 | generatedSourceDirs += file("${protobuf.generatedFilesBaseDir}/main/java") 39 | generatedSourceDirs += file("${protobuf.generatedFilesBaseDir}/main/rsocketRpc") 40 | } 41 | } 42 | 43 | jar { 44 | manifest { 45 | attributes( 46 | 'Plugin-Id': "${project.name}", 47 | 'Plugin-Version': "${project.version}", 48 | ) 49 | } 50 | 51 | into('lib') { 52 | from(configurations.runtimeClasspath - configurations.compileOnlyClasspath) 53 | } 54 | } 55 | 56 | dependencies { 57 | compileOnly 'com.google.auto.service:auto-service' 58 | annotationProcessor 'com.google.auto.service:auto-service' 59 | 60 | protobuf project(":protocol") 61 | 62 | compileOnly project(":app") 63 | 64 | api 'io.rsocket.rpc:rsocket-rpc-core' 65 | api ('io.rsocket:rsocket-transport-netty') { 66 | exclude group: "io.projectreactor.netty", module: "reactor-netty-core" 67 | exclude group: "io.projectreactor.netty", module: "reactor-netty-http" 68 | } 69 | 70 | testImplementation project(":app") 71 | testImplementation 'org.springframework.boot:spring-boot-test' 72 | testImplementation 'org.assertj:assertj-core' 73 | testImplementation 'org.mockito:mockito-core' 74 | } 75 | -------------------------------------------------------------------------------- /plugins/rsocket-transport/src/main/java/com/github/bsideup/liiklus/transport/rsocket/RSocketLiiklusService.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.transport.rsocket; 2 | 3 | import com.github.bsideup.liiklus.protocol.AckRequest; 4 | import com.github.bsideup.liiklus.protocol.GetEndOffsetsReply; 5 | import com.github.bsideup.liiklus.protocol.GetEndOffsetsRequest; 6 | import com.github.bsideup.liiklus.protocol.GetOffsetsReply; 7 | import com.github.bsideup.liiklus.protocol.GetOffsetsRequest; 8 | import com.github.bsideup.liiklus.protocol.PublishReply; 9 | import com.github.bsideup.liiklus.protocol.PublishRequest; 10 | import com.github.bsideup.liiklus.protocol.ReceiveReply; 11 | import com.github.bsideup.liiklus.protocol.ReceiveRequest; 12 | import com.github.bsideup.liiklus.protocol.SubscribeReply; 13 | import com.github.bsideup.liiklus.protocol.SubscribeRequest; 14 | import com.github.bsideup.liiklus.service.LiiklusService; 15 | import com.google.protobuf.Empty; 16 | import io.netty.buffer.ByteBuf; 17 | import lombok.AccessLevel; 18 | import lombok.RequiredArgsConstructor; 19 | import lombok.experimental.FieldDefaults; 20 | import reactor.core.publisher.Flux; 21 | import reactor.core.publisher.Mono; 22 | 23 | @RequiredArgsConstructor 24 | @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) 25 | public class RSocketLiiklusService implements com.github.bsideup.liiklus.protocol.LiiklusService { 26 | 27 | LiiklusService liiklusService; 28 | 29 | @Override 30 | public Mono publish(PublishRequest message, ByteBuf metadata) { 31 | return liiklusService.publish(Mono.just(message)); 32 | } 33 | 34 | @Override 35 | public Flux subscribe(SubscribeRequest message, ByteBuf metadata) { 36 | return liiklusService.subscribe(Mono.just(message)); 37 | } 38 | 39 | @Override 40 | public Flux receive(ReceiveRequest message, ByteBuf metadata) { 41 | return liiklusService.receive(Mono.just(message)); 42 | } 43 | 44 | @Override 45 | public Mono ack(AckRequest message, ByteBuf metadata) { 46 | return liiklusService.ack(Mono.just(message)); 47 | } 48 | 49 | @Override 50 | public Mono getOffsets(GetOffsetsRequest message, ByteBuf metadata) { 51 | return liiklusService.getOffsets(Mono.just(message)); 52 | } 53 | 54 | @Override 55 | public Mono getEndOffsets(GetEndOffsetsRequest message, ByteBuf metadata) { 56 | return liiklusService.getEndOffsets(Mono.just(message)); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /plugins/rsocket-transport/src/main/java/com/github/bsideup/liiklus/transport/rsocket/RSocketServerConfigurer.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.transport.rsocket; 2 | 3 | import io.rsocket.core.RSocketServer; 4 | 5 | public interface RSocketServerConfigurer { 6 | 7 | void apply(RSocketServer server); 8 | } 9 | -------------------------------------------------------------------------------- /plugins/rsocket-transport/src/test/java/com/github/bsideup/liiklus/transport/rsocket/config/RSocketConfigurationTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.transport.rsocket.config; 2 | 3 | import com.github.bsideup.liiklus.transport.rsocket.RSocketLiiklusService; 4 | import org.junit.jupiter.api.Test; 5 | import org.springframework.beans.BeansException; 6 | import org.springframework.boot.context.properties.bind.validation.BindValidationException; 7 | import org.springframework.boot.test.context.runner.ApplicationContextRunner; 8 | import org.springframework.context.ApplicationContextInitializer; 9 | import org.springframework.context.support.StaticApplicationContext; 10 | 11 | import static org.assertj.core.api.Assertions.assertThat; 12 | 13 | class RSocketConfigurationTest { 14 | 15 | ApplicationContextRunner applicationContextRunner = new ApplicationContextRunner(() -> new StaticApplicationContext() { 16 | @Override 17 | public void refresh() throws BeansException, IllegalStateException { 18 | } 19 | }) 20 | .withInitializer((ApplicationContextInitializer) new RSocketConfiguration()); 21 | 22 | @Test 23 | void shouldRequireGatewayProfile() { 24 | applicationContextRunner = applicationContextRunner.withPropertyValues( 25 | "spring.profiles.active: not_gateway" 26 | ); 27 | applicationContextRunner.run(context -> { 28 | assertThat(context).doesNotHaveBean(RSocketLiiklusService.class); 29 | }); 30 | applicationContextRunner = applicationContextRunner.withPropertyValues( 31 | "spring.profiles.active: gateway" 32 | ); 33 | applicationContextRunner.run(context -> { 34 | assertThat(context) 35 | .getFailure() 36 | .hasCauseInstanceOf(BindValidationException.class); 37 | }); 38 | } 39 | 40 | @Test 41 | void shouldBeDisableable() { 42 | applicationContextRunner = applicationContextRunner.withPropertyValues( 43 | "spring.profiles.active: gateway", 44 | "rsocket.enabled: false" 45 | ); 46 | applicationContextRunner.run(context -> { 47 | assertThat(context).doesNotHaveBean(RSocketLiiklusService.class); 48 | }); 49 | } 50 | 51 | @Test 52 | void shouldValidateParameters() { 53 | applicationContextRunner = applicationContextRunner.withPropertyValues( 54 | "spring.profiles.active: gateway" 55 | ); 56 | applicationContextRunner.run(context -> { 57 | assertThat(context) 58 | .getFailure() 59 | .hasCauseInstanceOf(BindValidationException.class); 60 | }); 61 | applicationContextRunner = applicationContextRunner.withPropertyValues( 62 | "rsocket.host: localhost" 63 | ); 64 | applicationContextRunner.run(context -> { 65 | assertThat(context) 66 | .getFailure() 67 | .hasCauseInstanceOf(BindValidationException.class); 68 | }); 69 | applicationContextRunner = applicationContextRunner.withPropertyValues( 70 | "rsocket.port: 0" 71 | ); 72 | applicationContextRunner.run(context -> { 73 | assertThat(context) 74 | .hasNotFailed(); 75 | }); 76 | } 77 | } -------------------------------------------------------------------------------- /plugins/schema/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | } 4 | 5 | jar { 6 | manifest { 7 | attributes( 8 | 'Plugin-Id': "${project.name}", 9 | 'Plugin-Version': "${project.version}", 10 | ) 11 | } 12 | 13 | into('lib') { 14 | from(configurations.runtimeClasspath - configurations.compileOnlyClasspath) 15 | } 16 | } 17 | 18 | tasks.test.dependsOn(jar) 19 | 20 | dependencies { 21 | compileOnly 'com.google.auto.service:auto-service' 22 | annotationProcessor 'com.google.auto.service:auto-service' 23 | 24 | compileOnly project(":app") 25 | 26 | api 'com.networknt:json-schema-validator:0.1.19' 27 | 28 | api 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' 29 | 30 | testImplementation project(":app") 31 | testImplementation project(":tck") 32 | testImplementation 'org.springframework.boot:spring-boot-test' 33 | testImplementation 'org.assertj:assertj-core' 34 | } 35 | -------------------------------------------------------------------------------- /plugins/schema/src/main/java/com/github/bsideup/liiklus/schema/SchemaPluginConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.schema; 2 | 3 | import com.fasterxml.jackson.core.JsonPointer; 4 | import com.github.bsideup.liiklus.records.RecordPreProcessor; 5 | import com.github.bsideup.liiklus.util.PropertiesUtil; 6 | import com.google.auto.service.AutoService; 7 | import lombok.Data; 8 | import org.hibernate.validator.group.GroupSequenceProvider; 9 | import org.hibernate.validator.spi.group.DefaultGroupSequenceProvider; 10 | import org.springframework.boot.context.properties.ConfigurationProperties; 11 | import org.springframework.context.ApplicationContextInitializer; 12 | import org.springframework.context.support.GenericApplicationContext; 13 | import org.springframework.core.env.Profiles; 14 | import org.springframework.validation.annotation.Validated; 15 | 16 | import javax.validation.constraints.NotEmpty; 17 | import javax.validation.constraints.NotNull; 18 | import java.net.URL; 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | @AutoService(ApplicationContextInitializer.class) 23 | public class SchemaPluginConfiguration implements ApplicationContextInitializer { 24 | 25 | @Override 26 | public void initialize(GenericApplicationContext applicationContext) { 27 | var environment = applicationContext.getEnvironment(); 28 | if (!environment.acceptsProfiles(Profiles.of("gateway"))) { 29 | return; 30 | } 31 | 32 | var schemaProperties = PropertiesUtil.bind(environment, new SchemaProperties()); 33 | 34 | if (!schemaProperties.isEnabled()) { 35 | return; 36 | } 37 | 38 | applicationContext.registerBean(RecordPreProcessor.class, () -> { 39 | return new JsonSchemaPreProcessor( 40 | schemaProperties.getSchemaURL(), 41 | JsonPointer.compile(schemaProperties.getEventTypeJsonPointer()), 42 | schemaProperties.isAllowDeprecatedProperties() 43 | ); 44 | }); 45 | } 46 | 47 | @ConfigurationProperties("schema") 48 | @Data 49 | @Validated 50 | @GroupSequenceProvider(SchemaProperties.EnabledSequenceProvider.class) 51 | static class SchemaProperties { 52 | 53 | boolean enabled = false; 54 | 55 | @NotNull(groups = Enabled.class) 56 | SchemaType type = SchemaType.JSON; 57 | 58 | @NotNull(groups = Enabled.class) 59 | URL schemaURL; 60 | 61 | @NotEmpty(groups = Enabled.class) 62 | String eventTypeJsonPointer = "/eventType"; 63 | 64 | boolean allowDeprecatedProperties = false; 65 | 66 | enum SchemaType { 67 | JSON, 68 | ; 69 | } 70 | 71 | interface Enabled {} 72 | 73 | public static class EnabledSequenceProvider implements DefaultGroupSequenceProvider { 74 | 75 | @Override 76 | public List> getValidationGroups(SchemaProperties object) { 77 | var sequence = new ArrayList>(); 78 | sequence.add(SchemaProperties.class); 79 | if (object != null && object.isEnabled()) { 80 | sequence.add(Enabled.class); 81 | } 82 | return sequence; 83 | } 84 | } 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /plugins/schema/src/main/java/com/github/bsideup/liiklus/schema/internal/DeprecatedKeyword.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.schema.internal; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import com.networknt.schema.*; 5 | import lombok.Getter; 6 | 7 | import java.text.MessageFormat; 8 | import java.util.Set; 9 | 10 | public class DeprecatedKeyword extends AbstractKeyword { 11 | 12 | public DeprecatedKeyword() { 13 | super("deprecated"); 14 | } 15 | 16 | @Override 17 | public JsonValidator newValidator(String schemaPath, JsonNode schemaNode, JsonSchema parentSchema, ValidationContext validationContext) { 18 | return new AbstractJsonValidator(getValue()) { 19 | @Override 20 | public Set validate(JsonNode node, JsonNode rootNode, String at) { 21 | if (schemaNode.asBoolean()) { 22 | var message = new ErrorMessageType() { 23 | 24 | @Getter 25 | String errorCode = "deprecated"; 26 | 27 | @Getter 28 | MessageFormat messageFormat = new MessageFormat("{0}: is deprecated"); 29 | }; 30 | 31 | return fail(message, at); 32 | } else { 33 | return pass(); 34 | } 35 | } 36 | }; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /plugins/schema/src/test/java/com/github/bsideup/liiklus/schema/SchemaPluginConfigurationTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.schema; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.beans.BeansException; 5 | import org.springframework.boot.context.properties.bind.BindException; 6 | import org.springframework.boot.test.context.runner.ApplicationContextRunner; 7 | import org.springframework.context.ApplicationContextInitializer; 8 | import org.springframework.context.support.StaticApplicationContext; 9 | 10 | import java.net.URL; 11 | 12 | import static org.assertj.core.api.Assertions.assertThat; 13 | 14 | class SchemaPluginConfigurationTest { 15 | 16 | ApplicationContextRunner applicationContextRunner = new ApplicationContextRunner(() -> new StaticApplicationContext() { 17 | @Override 18 | public void refresh() throws BeansException, IllegalStateException { 19 | } 20 | }) 21 | .withInitializer((ApplicationContextInitializer) new SchemaPluginConfiguration()); 22 | 23 | @Test 24 | void shouldNotBeEnabledByDefault() { 25 | applicationContextRunner.run(context -> { 26 | assertThat(context).doesNotHaveBean(JsonSchemaPreProcessor.class); 27 | }); 28 | } 29 | 30 | @Test 31 | void shouldRequireGatewayProfile() { 32 | applicationContextRunner = applicationContextRunner.withPropertyValues( 33 | "spring.profiles.active: not_gateway", 34 | "schema.enabled: true" 35 | ); 36 | applicationContextRunner.run(context -> { 37 | assertThat(context).doesNotHaveBean(JsonSchemaPreProcessor.class); 38 | }); 39 | } 40 | 41 | @Test 42 | void shouldValidateProperties() { 43 | applicationContextRunner = applicationContextRunner.withPropertyValues( 44 | "spring.profiles.active: gateway", 45 | "schema.enabled: true" 46 | ); 47 | applicationContextRunner.run(context -> { 48 | assertThat(context) 49 | .getFailure() 50 | .isInstanceOf(BindException.class); 51 | }); 52 | applicationContextRunner = applicationContextRunner.withPropertyValues( 53 | "schema.schemaURL: " + getSchemaURL() 54 | ); 55 | applicationContextRunner.run(context -> { 56 | assertThat(context).hasNotFailed(); 57 | }); 58 | } 59 | 60 | static URL getSchemaURL() { 61 | return Thread.currentThread().getContextClassLoader().getResource("schemas/basic.yml"); 62 | } 63 | } -------------------------------------------------------------------------------- /plugins/schema/src/test/java/com/github/bsideup/liiklus/schema/SmokeTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.schema; 2 | 3 | import com.fasterxml.jackson.core.JsonParseException; 4 | import com.github.bsideup.liiklus.ApplicationRunner; 5 | import com.github.bsideup.liiklus.protocol.LiiklusEvent; 6 | import com.github.bsideup.liiklus.protocol.PublishRequest; 7 | import com.github.bsideup.liiklus.service.LiiklusService; 8 | import com.google.protobuf.ByteString; 9 | import org.junit.jupiter.api.AfterAll; 10 | import org.junit.jupiter.api.BeforeAll; 11 | import org.junit.jupiter.api.Test; 12 | import org.springframework.context.ConfigurableApplicationContext; 13 | import reactor.core.publisher.Mono; 14 | 15 | import java.util.UUID; 16 | 17 | import static com.github.bsideup.liiklus.schema.SchemaPluginConfigurationTest.getSchemaURL; 18 | import static org.assertj.core.api.Assertions.assertThatThrownBy; 19 | 20 | public class SmokeTest { 21 | 22 | private static ConfigurableApplicationContext APP; 23 | 24 | @BeforeAll 25 | static void beforeAll() { 26 | APP = new ApplicationRunner("MEMORY", "MEMORY") 27 | .withProperty("schema.enabled", true) 28 | .withProperty("schema.schemaURL", getSchemaURL()) 29 | .run(); 30 | } 31 | 32 | @AfterAll 33 | static void afterAll() { 34 | APP.close(); 35 | } 36 | 37 | @Test 38 | void validEvent() { 39 | var event = LiiklusEvent.newBuilder() 40 | .setId(UUID.randomUUID().toString()) 41 | .setType("com.example.cloudevent") 42 | .setSource("/tests") 43 | .setDataContentType("application/json") 44 | .setData(ByteString.copyFromUtf8("{\"foo\":\"Hello!\"}")) 45 | .build(); 46 | 47 | send(event); 48 | } 49 | 50 | @Test 51 | void missingBody() { 52 | var event = LiiklusEvent.newBuilder() 53 | .setId(UUID.randomUUID().toString()) 54 | .setType("com.example.cloudevent") 55 | .setSource("/tests") 56 | .setDataContentType("application/json") 57 | .build(); 58 | 59 | assertThatThrownBy(() -> send(event)).hasMessageContaining("object expected"); 60 | } 61 | 62 | @Test 63 | void invalidData() { 64 | var event = LiiklusEvent.newBuilder() 65 | .setId(UUID.randomUUID().toString()) 66 | .setType("com.example.cloudevent") 67 | .setSource("/tests") 68 | .setDataContentType("application/json") 69 | .setData(ByteString.copyFromUtf8("wtf")) 70 | .build(); 71 | 72 | assertThatThrownBy(() -> send(event)).hasCauseInstanceOf(JsonParseException.class); 73 | } 74 | 75 | private void send(LiiklusEvent event) { 76 | APP.getBean(LiiklusService.class).publish(Mono.just( 77 | PublishRequest.newBuilder() 78 | .setTopic("events") 79 | .setLiiklusEvent(event) 80 | .build() 81 | )).block(); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /plugins/schema/src/test/resources/schemas/basic.yml: -------------------------------------------------------------------------------- 1 | events: 2 | deprecatedEvent: 3 | type: object 4 | deprecated: true 5 | 6 | withDeprecatedField: 7 | type: object 8 | additionalProperties: false 9 | properties: 10 | eventType: { type: string } 11 | goodField: { type: string } 12 | deprecatedField: { type: string, deprecated: true } 13 | 14 | simpleEvent: 15 | type: object 16 | additionalProperties: false 17 | required: 18 | - requiredField 19 | properties: 20 | eventType: { type: string } 21 | requiredField: { type: string } 22 | intField: { type: integer } 23 | 24 | "event/type/with/slashes": 25 | type: object 26 | additionalProperties: false 27 | properties: 28 | eventType: { type: string } 29 | foo: { type: string } 30 | 31 | "com.example.cloudevent": 32 | type: object 33 | additionalProperties: false 34 | required: 35 | - foo 36 | properties: 37 | foo: { type: string } -------------------------------------------------------------------------------- /protocol/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'idea' 3 | id 'java-library' 4 | id 'com.google.protobuf' 5 | } 6 | 7 | sourceCompatibility = targetCompatibility = 8 8 | 9 | protobuf { 10 | protoc { 11 | artifact = 'com.google.protobuf:protoc' 12 | } 13 | 14 | generatedFilesBaseDir = "$projectDir/generated" 15 | } 16 | 17 | clean { 18 | delete protobuf.generatedFilesBaseDir 19 | } 20 | 21 | idea { 22 | module { 23 | generatedSourceDirs += file("${protobuf.generatedFilesBaseDir}/main/java") 24 | } 25 | } 26 | 27 | dependencies { 28 | api 'com.google.protobuf:protobuf-java' 29 | } -------------------------------------------------------------------------------- /protocol/src/main/proto/LiiklusService.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package com.github.bsideup.liiklus; 4 | 5 | import "google/protobuf/timestamp.proto"; 6 | import "google/protobuf/empty.proto"; 7 | 8 | option java_package = "com.github.bsideup.liiklus.protocol"; 9 | option optimize_for = SPEED; 10 | option java_multiple_files = true; 11 | 12 | service LiiklusService { 13 | rpc Publish (PublishRequest) returns (PublishReply) { 14 | 15 | } 16 | 17 | rpc Subscribe (SubscribeRequest) returns (stream SubscribeReply) { 18 | } 19 | 20 | rpc Receive (ReceiveRequest) returns (stream ReceiveReply) { 21 | 22 | } 23 | 24 | rpc Ack (AckRequest) returns (google.protobuf.Empty) { 25 | 26 | } 27 | 28 | rpc GetOffsets (GetOffsetsRequest) returns (GetOffsetsReply) { 29 | 30 | } 31 | 32 | rpc GetEndOffsets (GetEndOffsetsRequest) returns (GetEndOffsetsReply) { 33 | 34 | } 35 | } 36 | 37 | message LiiklusEvent { 38 | // Required 39 | string id = 1; 40 | 41 | // Required 42 | string type = 2; 43 | 44 | // Required 45 | string source = 3; 46 | 47 | string time = 5; 48 | 49 | string data_content_type = 100; 50 | 51 | bytes data = 101; 52 | 53 | map extensions = 200; 54 | } 55 | 56 | message PublishRequest { 57 | string topic = 1; 58 | 59 | bytes key = 2; 60 | 61 | bytes value = 3 [deprecated = true]; 62 | 63 | oneof event { 64 | LiiklusEvent liiklusEvent = 4; 65 | } 66 | } 67 | 68 | message PublishReply { 69 | 70 | uint32 partition = 1; 71 | 72 | uint64 offset = 2; 73 | 74 | string topic = 3; 75 | } 76 | 77 | message SubscribeRequest { 78 | 79 | string topic = 1; 80 | 81 | string group = 2; 82 | 83 | uint32 group_version = 4; 84 | 85 | AutoOffsetReset auto_offset_reset = 3; 86 | 87 | enum AutoOffsetReset { 88 | EARLIEST = 0; 89 | LATEST = 1; 90 | } 91 | } 92 | 93 | message Assignment { 94 | string session_id = 1; 95 | 96 | uint32 partition = 2; 97 | } 98 | 99 | message SubscribeReply { 100 | oneof reply { 101 | Assignment assignment = 1; 102 | } 103 | } 104 | 105 | message AckRequest { 106 | Assignment assignment = 1 [deprecated = true]; 107 | 108 | string topic = 3; 109 | string group = 4; 110 | uint32 group_version = 5; 111 | uint32 partition = 6; 112 | 113 | uint64 offset = 2; 114 | } 115 | 116 | message ReceiveRequest { 117 | Assignment assignment = 1; 118 | 119 | uint64 last_known_offset = 2; 120 | 121 | ContentFormat format = 3; 122 | 123 | enum ContentFormat { 124 | BINARY = 0; 125 | LIIKLUS_EVENT = 1; 126 | } 127 | } 128 | 129 | message ReceiveReply { 130 | oneof reply { 131 | Record record = 1; 132 | LiiklusEventRecord liiklus_event_record = 2; 133 | } 134 | 135 | message Record { 136 | uint64 offset = 1; 137 | 138 | bytes key = 2; 139 | 140 | bytes value = 3; 141 | 142 | google.protobuf.Timestamp timestamp = 4; 143 | 144 | bool replay = 5; 145 | } 146 | 147 | message LiiklusEventRecord { 148 | uint64 offset = 1; 149 | 150 | bytes key = 2; 151 | 152 | LiiklusEvent event = 3; 153 | 154 | // TODO drop? 155 | google.protobuf.Timestamp timestamp = 4; 156 | 157 | bool replay = 5; 158 | } 159 | } 160 | 161 | message GetOffsetsRequest { 162 | 163 | string topic = 1; 164 | 165 | string group = 2; 166 | 167 | uint32 group_version = 3; 168 | 169 | } 170 | 171 | message GetOffsetsReply { 172 | map offsets = 1; 173 | } 174 | 175 | message GetEndOffsetsRequest { 176 | string topic = 1; 177 | } 178 | 179 | message GetEndOffsetsReply { 180 | map offsets = 1; 181 | } -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | include 'api' 2 | include 'tck' 3 | include 'app' 4 | include 'protocol' 5 | include 'client' 6 | include 'testing' 7 | 8 | include 'examples:java' 9 | include 'examples:plugin' 10 | 11 | include 'plugins:dynamodb-positions-storage' 12 | include 'plugins:grpc-transport' 13 | include 'plugins:grpc-transport-auth' 14 | include 'plugins:inmemory-positions-storage' 15 | include 'plugins:inmemory-records-storage' 16 | include 'plugins:kafka-records-storage' 17 | include 'plugins:metrics' 18 | include 'plugins:pulsar-records-storage' 19 | include 'plugins:redis-positions-storage' 20 | include 'plugins:rsocket-transport' 21 | include 'plugins:schema' 22 | 23 | file('plugins').eachDir { dir -> 24 | def projectName = dir.name 25 | if (projectName != "build" && findProject(":plugins:${projectName}") == null) { 26 | throw new GradleException("Plugin /plugins/${projectName} was not included into the build. Check settings.gradle for details.") 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /tck/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | } 4 | 5 | compileJava.dependsOn(tasks.findByPath(":app:bootJar")) 6 | 7 | java { 8 | withSourcesJar() 9 | } 10 | 11 | dependencies { 12 | api project(":app") 13 | api 'org.springframework.boot:spring-boot-loader' 14 | 15 | api 'org.junit.jupiter:junit-jupiter-api' 16 | api 'org.assertj:assertj-core' 17 | api 'io.projectreactor:reactor-test' 18 | api 'org.awaitility:awaitility' 19 | } 20 | 21 | tasks.test.dependsOn( 22 | rootProject.project(":plugins:inmemory-positions-storage").getTasksByName("jar", false), 23 | rootProject.project(":plugins:inmemory-records-storage").getTasksByName("jar", false) 24 | ) 25 | 26 | tasks.register('appDependency') { 27 | dependsOn( 28 | tasks.findByPath(":app:sourcesJar"), 29 | tasks.findByPath(":app:bootJar"), 30 | tasks.findByPath(":app:jar") 31 | ) 32 | 33 | doLast { 34 | def app = tasks.findByPath(":app:bootJar").outputs.files.singleFile 35 | if (app.name != 'app-boot.jar') { 36 | throw new GradleException(':app:bootJar task should produce exactly "app-boot.jar" file to be included into TCK, got: ' + app.absolutePath) 37 | } 38 | } 39 | } 40 | 41 | processResources.dependsOn(tasks.appDependency) 42 | 43 | sourceSets { 44 | main { 45 | resources.srcDir tasks.findByPath(":app:bootJar").outputs.files.singleFile.parentFile 46 | resources.include 'app-boot.jar' 47 | } 48 | } 49 | 50 | sourcesJar { 51 | dependsOn processResources 52 | } 53 | -------------------------------------------------------------------------------- /tck/src/main/java/com/github/bsideup/liiklus/positions/PositionsStorageTestSupport.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.positions; 2 | 3 | import lombok.SneakyThrows; 4 | 5 | import java.util.Collections; 6 | import java.util.HashMap; 7 | import java.util.Map; 8 | import java.util.concurrent.CompletionStage; 9 | import java.util.concurrent.TimeUnit; 10 | 11 | public interface PositionsStorageTestSupport { 12 | 13 | PositionsStorage getStorage(); 14 | 15 | @SneakyThrows 16 | default T await(CompletionStage stage) { 17 | return stage.toCompletableFuture().get(10, TimeUnit.SECONDS); 18 | } 19 | 20 | default Map mapOf(T key, V value) { 21 | return Collections.singletonMap(key, value); 22 | } 23 | 24 | default Map mapOf(T key, V value, T key2, V value2) { 25 | HashMap map = new HashMap<>(); 26 | map.put(key, value); 27 | map.put(key2, value2); 28 | return map; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /tck/src/main/java/com/github/bsideup/liiklus/positions/PositionsStorageTests.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.positions; 2 | 3 | import com.github.bsideup.liiklus.positions.tests.GroupsTest; 4 | import com.github.bsideup.liiklus.positions.tests.PersistenceTest; 5 | 6 | public interface PositionsStorageTests extends GroupsTest, PersistenceTest { 7 | } 8 | -------------------------------------------------------------------------------- /tck/src/main/java/com/github/bsideup/liiklus/positions/tests/GroupsTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.positions.tests; 2 | 3 | import com.github.bsideup.liiklus.positions.GroupId; 4 | import com.github.bsideup.liiklus.positions.PositionsStorage; 5 | import com.github.bsideup.liiklus.positions.PositionsStorageTestSupport; 6 | import org.junit.jupiter.api.DisplayName; 7 | import org.junit.jupiter.api.Test; 8 | import reactor.core.publisher.Flux; 9 | 10 | import java.time.Duration; 11 | import java.util.List; 12 | import java.util.Map; 13 | import java.util.UUID; 14 | 15 | import static java.util.Arrays.asList; 16 | import static org.assertj.core.api.Assertions.assertThat; 17 | 18 | public interface GroupsTest extends PositionsStorageTestSupport { 19 | 20 | @Test 21 | @DisplayName("Should return all groups and topics") 22 | default void shouldReturnMultipleGroups() { 23 | var topic = UUID.randomUUID().toString(); 24 | var topic2 = UUID.randomUUID().toString(); 25 | var groupId = GroupId.ofString(UUID.randomUUID().toString()); 26 | var groupId2 = GroupId.ofString(UUID.randomUUID().toString()); 27 | 28 | await(getStorage().update(topic, groupId, 2, 2)); 29 | await(getStorage().update(topic, groupId2, 2, 3)); 30 | await(getStorage().update(topic2, groupId2, 2, 4)); 31 | await(getStorage().update(topic2, groupId2, 3, 5)); 32 | 33 | List positions = Flux.from(getStorage().findAll()) 34 | .collectList() 35 | .block(Duration.ofSeconds(10)); 36 | 37 | assertThat(positions) 38 | .filteredOn(pos -> asList(groupId, groupId2).contains(pos.getGroupId())) 39 | .flatExtracting(PositionsStorage.Positions::getTopic) 40 | .contains(topic, topic2); 41 | 42 | assertThat(positions) 43 | .filteredOn(pos -> asList(groupId, groupId2).contains(pos.getGroupId())) 44 | .hasSize(3) 45 | .flatExtracting(PositionsStorage.Positions::getValues) 46 | .contains( 47 | mapOf( 48 | 2, 4L, 49 | 3, 5L 50 | ), 51 | mapOf(2, 3L), 52 | mapOf(2, 2L) 53 | ); 54 | } 55 | 56 | @Test 57 | @DisplayName("Should return all versions by group name") 58 | default void shouldReturnAllVersionsByGroup() { 59 | var topic = UUID.randomUUID().toString(); 60 | var groupName = UUID.randomUUID().toString(); 61 | var groupId1 = GroupId.of(groupName, 1); 62 | var groupId2 = GroupId.of(groupName, 2); 63 | 64 | await(getStorage().update(topic, groupId1, 2, 2)); 65 | await(getStorage().update(topic, groupId2, 2, 3)); 66 | await(getStorage().update(topic, groupId2, 4, 5)); 67 | 68 | Map> positions = await(getStorage().findAllVersionsByGroup(topic, groupName)); 69 | 70 | assertThat(positions) 71 | .hasSize(2) 72 | .containsEntry(1, mapOf(2, 2L)) 73 | .containsEntry(2, mapOf( 74 | 2, 3L, 75 | 4, 5L 76 | )); 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /tck/src/main/java/com/github/bsideup/liiklus/positions/tests/PersistenceTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.positions.tests; 2 | 3 | import com.github.bsideup.liiklus.positions.GroupId; 4 | import com.github.bsideup.liiklus.positions.PositionsStorageTestSupport; 5 | import org.junit.jupiter.api.DisplayName; 6 | import org.junit.jupiter.api.Test; 7 | 8 | import java.util.Map; 9 | import java.util.UUID; 10 | 11 | import static org.assertj.core.api.Assertions.assertThat; 12 | 13 | public interface PersistenceTest extends PositionsStorageTestSupport { 14 | 15 | 16 | @Test 17 | @DisplayName("Should save new position") 18 | default void shouldSavePosition() { 19 | var topic = UUID.randomUUID().toString(); 20 | var groupId = GroupId.ofString(UUID.randomUUID().toString()); 21 | 22 | await(getStorage().update(topic, groupId, 2, 2)); 23 | Map positions = await(getStorage().findAll(topic, groupId)); 24 | 25 | assertThat(positions).containsEntry(2, 2L); 26 | } 27 | 28 | 29 | @Test 30 | @DisplayName("Should update existing position") 31 | default void shouldUpdatePosition() { 32 | var topic = UUID.randomUUID().toString(); 33 | var groupId = GroupId.ofString(UUID.randomUUID().toString()); 34 | 35 | await(getStorage().update(topic, groupId, 2, 2)); 36 | await(getStorage().update(topic, groupId, 2, 3)); 37 | Map positions = await(getStorage().findAll(topic, groupId)); 38 | 39 | assertThat(positions) 40 | .containsKeys(2) 41 | .containsEntry(2, 3L); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /tck/src/main/java/com/github/bsideup/liiklus/records/RecordStorageTestSupport.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.records; 2 | 3 | import lombok.SneakyThrows; 4 | import org.awaitility.core.ConditionFactory; 5 | import reactor.core.publisher.Flux; 6 | import reactor.core.publisher.Mono; 7 | 8 | import java.nio.ByteBuffer; 9 | import java.time.Duration; 10 | import java.time.ZonedDateTime; 11 | import java.util.*; 12 | import java.util.concurrent.CompletableFuture; 13 | import java.util.concurrent.CompletionStage; 14 | import java.util.concurrent.TimeUnit; 15 | import java.util.function.Supplier; 16 | 17 | import static org.awaitility.Awaitility.await; 18 | 19 | public interface RecordStorageTestSupport { 20 | 21 | static ConditionFactory await = await().atMost(30, TimeUnit.SECONDS); 22 | 23 | RecordsStorage getTarget(); 24 | 25 | String getTopic(); 26 | 27 | default RecordsStorage.Envelope createEnvelope(byte[] key) { 28 | return createEnvelope(key, UUID.randomUUID().toString().getBytes()); 29 | } 30 | 31 | @Deprecated 32 | default RecordsStorage.Envelope createEnvelope(byte[] key, byte[] value) { 33 | return new RecordsStorage.Envelope( 34 | getTopic(), 35 | 36 | ByteBuffer.wrap(key).asReadOnlyBuffer(), 37 | ByteBuffer.class::cast, 38 | 39 | new LiiklusCloudEvent( 40 | UUID.randomUUID().toString(), 41 | "com.example.event", 42 | "/tck/RecordStorageTestSupport", 43 | "application/json", 44 | ZonedDateTime.now().toString(), 45 | ByteBuffer.wrap(value).asReadOnlyBuffer(), 46 | Collections.emptyMap() 47 | ), 48 | LiiklusCloudEvent::asJson 49 | ); 50 | } 51 | 52 | default List publishMany(byte[] key, int num) { 53 | return publishMany(key, Flux.range(0, num).map(__ -> UUID.randomUUID().toString().getBytes())); 54 | } 55 | 56 | default List publishMany(byte[] key, Flux values) { 57 | return values 58 | .flatMapSequential(value -> Mono.fromCompletionStage(getTarget().publish(createEnvelope(key, value)))) 59 | .collectList() 60 | .block(Duration.ofSeconds(10)); 61 | } 62 | 63 | default RecordsStorage.OffsetInfo publish(byte[] key, byte[] value) { 64 | return publish(createEnvelope(key, value)); 65 | } 66 | 67 | @SneakyThrows 68 | default RecordsStorage.OffsetInfo publish(RecordsStorage.Envelope envelope) { 69 | return getTarget().publish(envelope).toCompletableFuture().get(10, TimeUnit.SECONDS); 70 | } 71 | 72 | default Flux subscribeToPartition(int partition) { 73 | return subscribeToPartition(partition, "earliest"); 74 | } 75 | 76 | default Flux subscribeToPartition(int partition, String offsetReset) { 77 | return subscribeToPartition(partition, Optional.of(offsetReset)); 78 | } 79 | 80 | default Flux subscribeToPartition(int partition, Optional offsetReset) { 81 | return subscribeToPartition(partition, offsetReset, () -> CompletableFuture.completedFuture(Collections.emptyMap())); 82 | } 83 | 84 | default Flux subscribeToPartition( 85 | int partition, 86 | Optional offsetReset, 87 | Supplier>> offsetsProvider 88 | ) { 89 | return Flux.from(getTarget().subscribe(getTopic(), UUID.randomUUID().toString(), offsetReset).getPublisher(offsetsProvider)) 90 | .flatMapIterable(it -> { 91 | var iterator = it.iterator(); 92 | return () -> iterator; 93 | }) 94 | .filter(it -> partition == it.getPartition()); 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /tck/src/main/java/com/github/bsideup/liiklus/records/RecordStorageTests.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.records; 2 | 3 | import com.github.bsideup.liiklus.records.tests.BackPressureTest; 4 | import com.github.bsideup.liiklus.records.tests.ConsumerGroupTest; 5 | import com.github.bsideup.liiklus.records.tests.EndOffsetsTest; 6 | import com.github.bsideup.liiklus.records.tests.PublishTest; 7 | import com.github.bsideup.liiklus.records.tests.SubscribeTest; 8 | 9 | public interface RecordStorageTests extends PublishTest, SubscribeTest, ConsumerGroupTest, BackPressureTest, EndOffsetsTest { 10 | } 11 | -------------------------------------------------------------------------------- /tck/src/main/java/com/github/bsideup/liiklus/records/tests/BackPressureTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.records.tests; 2 | 3 | import com.github.bsideup.liiklus.records.RecordStorageTestSupport; 4 | import com.github.bsideup.liiklus.records.RecordsStorage; 5 | import org.junit.jupiter.api.Test; 6 | import reactor.test.StepVerifier; 7 | 8 | import java.time.Duration; 9 | 10 | public interface BackPressureTest extends RecordStorageTestSupport { 11 | 12 | @Test 13 | default void testSubscribeBackpressure() throws Exception { 14 | var numRecords = 20; 15 | var key = "key".getBytes(); 16 | 17 | var offsetInfos = publishMany(key, numRecords); 18 | var partition = offsetInfos.get(0).getPartition(); 19 | 20 | var recordFlux = subscribeToPartition(partition, "earliest").flatMap(RecordsStorage.PartitionSource::getPublisher); 21 | 22 | var initialRequest = 1; 23 | StepVerifier.create(recordFlux, initialRequest) 24 | .expectSubscription() 25 | .expectNextCount(1) 26 | .then(() -> publishMany(key, 10)) 27 | .thenRequest(1) 28 | .expectNextCount(1) 29 | .thenCancel() 30 | .verify(Duration.ofSeconds(10)); 31 | } 32 | 33 | @Test 34 | default void testSubscribeWithoutRequest() throws Exception { 35 | var numRecords = 20; 36 | var key = "key".getBytes(); 37 | 38 | var offsetInfos = publishMany(key, numRecords); 39 | var partition = offsetInfos.get(0).getPartition(); 40 | 41 | var recordFlux = subscribeToPartition(partition, "earliest").flatMap(RecordsStorage.PartitionSource::getPublisher); 42 | 43 | var initialRequest = 0; 44 | StepVerifier.create(recordFlux, initialRequest) 45 | .expectSubscription() 46 | .then(() -> publishMany(key, 10)) 47 | .expectNoEvent(Duration.ofSeconds(1)) 48 | .thenCancel() 49 | .verify(Duration.ofSeconds(10)); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /tck/src/main/java/com/github/bsideup/liiklus/records/tests/EndOffsetsTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.records.tests; 2 | 3 | import com.github.bsideup.liiklus.records.FiniteRecordsStorage; 4 | import com.github.bsideup.liiklus.records.RecordStorageTestSupport; 5 | import org.junit.jupiter.api.Assumptions; 6 | import org.junit.jupiter.api.BeforeEach; 7 | import org.junit.jupiter.api.Test; 8 | import org.junit.jupiter.api.TestInfo; 9 | 10 | import java.lang.reflect.Method; 11 | import java.util.HashMap; 12 | import java.util.UUID; 13 | import java.util.concurrent.TimeUnit; 14 | 15 | import static org.assertj.core.api.Assertions.assertThat; 16 | 17 | public interface EndOffsetsTest extends RecordStorageTestSupport { 18 | 19 | default FiniteRecordsStorage getFiniteTarget() { 20 | return (FiniteRecordsStorage) getTarget(); 21 | } 22 | 23 | int getNumberOfPartitions(); 24 | 25 | String keyByPartition(int partition); 26 | 27 | @BeforeEach 28 | default void blah(TestInfo testInfo) { 29 | if (EndOffsetsTest.class == testInfo.getTestMethod().map(Method::getDeclaringClass).orElse(null)) { 30 | Assumptions.assumeTrue(getTarget() instanceof FiniteRecordsStorage, "target is finite"); 31 | } 32 | } 33 | 34 | @Test 35 | default void testEndOffsets() throws Exception { 36 | var topic = getTopic(); 37 | 38 | var lastReceivedOffsets = new HashMap(); 39 | for (int partition = 0; partition < getNumberOfPartitions(); partition++) { 40 | for (int i = 0; i < partition + 1; i++) { 41 | var offsetInfo = publish(keyByPartition(partition).getBytes(), new byte[1]); 42 | lastReceivedOffsets.put(offsetInfo.getPartition(), offsetInfo.getOffset()); 43 | } 44 | } 45 | 46 | var offsets = getFiniteTarget().getEndOffsets(topic).toCompletableFuture().get(10, TimeUnit.SECONDS); 47 | 48 | assertThat(offsets).isEqualTo(lastReceivedOffsets); 49 | } 50 | 51 | @Test 52 | default void testEndOffsets_unknownTopic() throws Exception { 53 | var topic = UUID.randomUUID().toString(); 54 | 55 | var offsets = getFiniteTarget().getEndOffsets(topic).toCompletableFuture().get(10, TimeUnit.SECONDS); 56 | 57 | assertThat(offsets) 58 | .allSatisfy((partition, offset) -> { 59 | assertThat(offset) 60 | .as("offset of p" + partition) 61 | .isEqualTo(-1L); 62 | }); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /tck/src/main/java/com/github/bsideup/liiklus/records/tests/PublishTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.records.tests; 2 | 3 | import com.github.bsideup.liiklus.records.RecordStorageTestSupport; 4 | import com.github.bsideup.liiklus.records.RecordsStorage; 5 | import io.cloudevents.CloudEvent; 6 | import io.cloudevents.json.Json; 7 | import org.junit.jupiter.api.Test; 8 | 9 | import java.time.Duration; 10 | import java.util.Comparator; 11 | 12 | import static org.assertj.core.api.Assertions.assertThat; 13 | 14 | public interface PublishTest extends RecordStorageTestSupport { 15 | 16 | @Test 17 | default void testPublish() throws Exception { 18 | var envelope = createEnvelope("key".getBytes()); 19 | 20 | var offsetInfo = publish(envelope); 21 | 22 | assertThat(offsetInfo) 23 | .satisfies(info -> { 24 | assertThat(info.getTopic()).as("topic").isEqualTo(getTopic()); 25 | assertThat(info.getOffset()).as("offset").isNotNegative(); 26 | }); 27 | 28 | var receivedRecord = subscribeToPartition(offsetInfo.getPartition()) 29 | .flatMap(RecordsStorage.PartitionSource::getPublisher) 30 | .blockFirst(Duration.ofSeconds(10)); 31 | 32 | assertThat(receivedRecord.getEnvelope()).as("envelope") 33 | .usingComparatorForType(Comparator.comparing(Json::encode), CloudEvent.class) 34 | .isEqualToIgnoringGivenFields(envelope, "keyEncoder", "valueEncoder") 35 | .satisfies(it -> { 36 | assertThat(it.getRawValue()).isInstanceOf(CloudEvent.class); 37 | }); 38 | 39 | assertThat(receivedRecord.getPartition()).as("partition").isEqualTo(offsetInfo.getPartition()); 40 | assertThat(receivedRecord.getOffset()).as("offset").isEqualTo(offsetInfo.getOffset()); 41 | } 42 | 43 | @Test 44 | default void testPublishMany() { 45 | var numRecords = 5; 46 | 47 | var offsetInfos = publishMany("key".getBytes(), numRecords); 48 | 49 | assertThat(offsetInfos).hasSize(numRecords); 50 | 51 | var partition = offsetInfos.get(0).getPartition(); 52 | 53 | assertThat(offsetInfos).extracting(RecordsStorage.OffsetInfo::getPartition).containsOnly(partition); 54 | } 55 | 56 | @Test 57 | default void testPublishOffsetIsGrowing() { 58 | RecordsStorage.OffsetInfo first = publish("key".getBytes(), "value1".getBytes()); 59 | RecordsStorage.OffsetInfo second = publish("key".getBytes(), "value2".getBytes()); 60 | 61 | assertThat(first.getPartition()).isEqualTo(second.getPartition()); 62 | assertThat(second.getOffset()).isGreaterThan(first.getOffset()); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /tck/src/main/java/com/github/bsideup/liiklus/support/DisabledUntil.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.support; 2 | 3 | import org.junit.jupiter.api.extension.ConditionEvaluationResult; 4 | import org.junit.jupiter.api.extension.ExecutionCondition; 5 | import org.junit.jupiter.api.extension.ExtendWith; 6 | import org.junit.jupiter.api.extension.ExtensionContext; 7 | 8 | import java.lang.annotation.ElementType; 9 | import java.lang.annotation.Retention; 10 | import java.lang.annotation.RetentionPolicy; 11 | import java.lang.annotation.Target; 12 | import java.time.LocalDate; 13 | import java.util.Optional; 14 | 15 | import static org.junit.platform.commons.support.AnnotationSupport.findAnnotation; 16 | 17 | 18 | @Target({ElementType.TYPE, ElementType.METHOD}) 19 | @Retention(RetentionPolicy.RUNTIME) 20 | @ExtendWith(DisabledUntil.DisabledCondition.class) 21 | public @interface DisabledUntil { 22 | /** 23 | * Local date, ISO formatted string such as 2000-01-30 24 | */ 25 | String value(); 26 | 27 | String comment(); 28 | 29 | class DisabledCondition implements ExecutionCondition { 30 | 31 | @Override 32 | public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { 33 | Optional until = findAnnotation(context.getElement(), DisabledUntil.class); 34 | if (until.map(it -> LocalDate.parse(it.value()).isAfter(LocalDate.now())).orElse(false)) { 35 | String reason = until.map(DisabledUntil::comment).orElse("Disabled for now"); 36 | return ConditionEvaluationResult.disabled(reason); 37 | } 38 | 39 | return ConditionEvaluationResult.enabled("Enabled"); 40 | } 41 | } 42 | } -------------------------------------------------------------------------------- /tck/src/test/java/com/github/bsideup/liiklus/ApplicationRunnerTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus; 2 | 3 | import org.junit.jupiter.api.Test; 4 | 5 | import static org.assertj.core.api.Assertions.assertThat; 6 | 7 | class ApplicationRunnerTest { 8 | 9 | @Test 10 | void shouldStart() { 11 | var context = new ApplicationRunner("MEMORY", "MEMORY").run(); 12 | 13 | assertThat(context).isNotNull(); 14 | } 15 | 16 | } -------------------------------------------------------------------------------- /testing/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java-library' 3 | } 4 | 5 | dependencies { 6 | api 'org.testcontainers:testcontainers' 7 | api 'org.testcontainers:kafka' 8 | 9 | testImplementation 'ch.qos.logback:logback-classic' 10 | testImplementation 'org.assertj:assertj-core' 11 | testImplementation 'org.awaitility:awaitility' 12 | } -------------------------------------------------------------------------------- /testing/src/main/java/com/github/bsideup/liiklus/container/LiiklusContainer.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.container; 2 | 3 | import org.testcontainers.containers.GenericContainer; 4 | import org.testcontainers.containers.KafkaContainer; 5 | import org.testcontainers.containers.Network; 6 | 7 | 8 | public class LiiklusContainer extends GenericContainer { 9 | 10 | public LiiklusContainer(String version) { 11 | super("bsideup/liiklus:" + version); 12 | 13 | withEnv("spring_profiles_active", "gateway"); 14 | withEnv("storage_positions_type", "MEMORY"); 15 | withEnv("storage_records_type", "MEMORY"); 16 | withExposedPorts(6565); 17 | } 18 | 19 | public LiiklusContainer withKafka(KafkaContainer kafkaContainer) { 20 | return withKafka(kafkaContainer.getNetwork(), kafkaContainer.getNetworkAliases().get(0) + ":9092"); 21 | } 22 | 23 | public LiiklusContainer withKafka(Network network, String bootstrapServers) { 24 | withNetwork(network); 25 | withEnv("kafka_bootstrapServers", bootstrapServers); 26 | withEnv("storage_records_type", "KAFKA"); 27 | return self(); 28 | } 29 | 30 | public String getTarget() { 31 | return getContainerIpAddress() + ":" + getMappedPort(6565); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /testing/src/test/java/com/github/bsideup/liiklus/container/LiiklusContainerTest.java: -------------------------------------------------------------------------------- 1 | package com.github.bsideup.liiklus.container; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.testcontainers.containers.KafkaContainer; 5 | import org.testcontainers.containers.Network; 6 | import org.testcontainers.utility.DockerImageName; 7 | 8 | public class LiiklusContainerTest { 9 | 10 | static final String LATEST_VERSION = "0.10.0-rc1"; 11 | static KafkaContainer kafka = new KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:5.4.3")) 12 | .withNetwork(Network.newNetwork()); 13 | 14 | static { 15 | kafka.start(); 16 | } 17 | 18 | @Test 19 | void shouldStartWithKafkaRecordStorage() { 20 | try (LiiklusContainer liiklusContainer = new LiiklusContainer(LATEST_VERSION)) { 21 | liiklusContainer.withKafka(kafka).start(); 22 | } 23 | } 24 | 25 | @Test 26 | void shouldStartDefaultMemoryRecordStorage() { 27 | try (LiiklusContainer liiklusContainer = new LiiklusContainer(LATEST_VERSION)) { 28 | liiklusContainer.start(); 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /testing/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | --------------------------------------------------------------------------------