├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── config.yaml ├── actions │ ├── push-docker-images │ │ └── action.yml │ ├── test-source │ │ └── action.yml │ └── update-docs │ │ └── action.yml ├── changelog-config.json ├── path_filter.yaml ├── semver └── workflows │ ├── gradle-wrapper-validation.yml │ ├── master-ci.yml │ ├── pull-request-ci.yml │ ├── release.yml │ └── update-helm-chart.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── build-logic ├── quick-plugins │ ├── build.gradle.kts │ └── src │ │ └── main │ │ └── kotlin │ │ └── buildlogic │ │ └── quickplugins │ │ ├── BaseExtension.kt │ │ ├── BasePlugin.kt │ │ ├── ProtobufGeneratorPlugin.kt │ │ ├── QuickCodeQualityPlugin.kt │ │ ├── QuickJibPlugin.kt │ │ ├── ReporterPlugin.kt │ │ └── VersionPlugin.kt └── settings.gradle.kts ├── build.gradle.kts ├── common ├── build.gradle.kts └── src │ ├── main │ ├── avro │ │ ├── QuickTopicType.avsc │ │ └── TopicData.avsc │ ├── java │ │ └── com │ │ │ └── bakdata │ │ │ └── quick │ │ │ └── common │ │ │ ├── api │ │ │ ├── client │ │ │ │ ├── ClientUtils.java │ │ │ │ ├── HttpClient.java │ │ │ │ ├── application │ │ │ │ │ └── ApplicationClient.java │ │ │ │ ├── gateway │ │ │ │ │ ├── DefaultGatewayClient.java │ │ │ │ │ └── GatewayClient.java │ │ │ │ ├── ingest │ │ │ │ │ └── IngestClient.java │ │ │ │ ├── mirror │ │ │ │ │ ├── DefaultMirrorClient.java │ │ │ │ │ ├── DefaultMirrorRequestManager.java │ │ │ │ │ ├── HeaderConstants.java │ │ │ │ │ ├── MirrorClient.java │ │ │ │ │ ├── MirrorClientFactory.java │ │ │ │ │ ├── MirrorHost.java │ │ │ │ │ ├── MirrorRegistryClient.java │ │ │ │ │ ├── MirrorRequestManager.java │ │ │ │ │ ├── MirrorRequestManagerWithFallback.java │ │ │ │ │ ├── MirrorValueParser.java │ │ │ │ │ ├── ParserFunction.java │ │ │ │ │ ├── PartitionedMirrorClient.java │ │ │ │ │ ├── PartitionedMirrorClientFactory.java │ │ │ │ │ ├── ResponseWrapper.java │ │ │ │ │ ├── StreamsStateHost.java │ │ │ │ │ └── TopicRegistryClient.java │ │ │ │ └── routing │ │ │ │ │ ├── DefaultPartitionFinder.java │ │ │ │ │ ├── PartitionFinder.java │ │ │ │ │ ├── PartitionRouter.java │ │ │ │ │ └── Router.java │ │ │ └── model │ │ │ │ ├── ErrorMessage.java │ │ │ │ ├── HttpStatusError.java │ │ │ │ ├── KeyValueEnum.java │ │ │ │ ├── KeyValuePair.java │ │ │ │ ├── TopicData.java │ │ │ │ ├── TopicSchemaTypes.java │ │ │ │ ├── TopicWriteType.java │ │ │ │ ├── gateway │ │ │ │ └── SchemaData.java │ │ │ │ ├── manager │ │ │ │ ├── ApplicationDescription.java │ │ │ │ ├── GatewayDescription.java │ │ │ │ ├── GatewaySchema.java │ │ │ │ └── creation │ │ │ │ │ ├── ApplicationCreationData.java │ │ │ │ │ ├── CreationData.java │ │ │ │ │ ├── GatewayCreationData.java │ │ │ │ │ ├── MirrorArguments.java │ │ │ │ │ ├── MirrorCreationData.java │ │ │ │ │ └── TopicCreationData.java │ │ │ │ └── mirror │ │ │ │ └── MirrorValue.java │ │ │ ├── condition │ │ │ ├── AvroSchemaFormatCondition.java │ │ │ └── ProtobufSchemaFormatCondition.java │ │ │ ├── config │ │ │ ├── AvroConfig.java │ │ │ ├── KafkaConfig.java │ │ │ ├── MirrorConfig.java │ │ │ ├── ProtobufConfig.java │ │ │ ├── QuickTopicConfig.java │ │ │ ├── SchemaConfig.java │ │ │ └── TopicRegistryConfig.java │ │ │ ├── exception │ │ │ ├── BadArgumentException.java │ │ │ ├── BadServiceAnnotationException.java │ │ │ ├── HttpClientException.java │ │ │ ├── InternalErrorException.java │ │ │ ├── MirrorException.java │ │ │ ├── MirrorTopologyException.java │ │ │ ├── NotFoundException.java │ │ │ ├── QuickException.java │ │ │ ├── ServiceException.java │ │ │ ├── ServiceUnavailableException.java │ │ │ ├── handler │ │ │ │ ├── AuthorizedExceptionHandler.java │ │ │ │ ├── AvroExceptionHandler.java │ │ │ │ ├── CompositeExceptionHandler.java │ │ │ │ ├── GeneralExceptionHandler.java │ │ │ │ ├── HttpClientResponseExceptionHandler.java │ │ │ │ ├── JsonErrorExceptionHandler.java │ │ │ │ ├── KafkaExceptionHandler.java │ │ │ │ ├── SchemaRegistryExceptionHandler.java │ │ │ │ ├── StatusExceptionHandler.java │ │ │ │ └── StatusHandler.java │ │ │ └── schema │ │ │ │ └── SchemaNotFoundException.java │ │ │ ├── graphql │ │ │ └── GraphQLUtils.java │ │ │ ├── json │ │ │ ├── AvroJacksonConfiguration.java │ │ │ ├── AvroRecordSerializer.java │ │ │ ├── ObjectMapperConfiguration.java │ │ │ ├── ObjectMapperListener.java │ │ │ ├── ProtobufJacksonConfiguration.java │ │ │ └── ProtobufMessageSerializer.java │ │ │ ├── metrics │ │ │ └── MetricsFactory.java │ │ │ ├── resolver │ │ │ ├── DoubleResolver.java │ │ │ ├── GenericAvroResolver.java │ │ │ ├── IntegerResolver.java │ │ │ ├── KnownTypeResolver.java │ │ │ ├── LongResolver.java │ │ │ ├── ProtobufResolver.java │ │ │ ├── StringResolver.java │ │ │ └── TypeResolver.java │ │ │ ├── schema │ │ │ ├── SchemaFetcher.java │ │ │ ├── SchemaFormat.java │ │ │ ├── SchemaProviderFactory.java │ │ │ └── SchemaRegistryFetcher.java │ │ │ ├── security │ │ │ ├── ApiKeyAuthentication.java │ │ │ ├── ApiKeyAuthenticationFetcher.java │ │ │ ├── SecurityConfig.java │ │ │ └── TokenPropagator.java │ │ │ ├── type │ │ │ ├── ConversionProvider.java │ │ │ ├── DefaultConversionProvider.java │ │ │ ├── QuickTopicData.java │ │ │ ├── QuickTopicType.java │ │ │ ├── TopicTypeService.java │ │ │ └── registry │ │ │ │ ├── QuickTopicTypeService.java │ │ │ │ └── TypeResolverWithSchema.java │ │ │ └── util │ │ │ ├── CliArgHandler.java │ │ │ └── Lazy.java │ └── resources │ │ └── log4j2.yaml │ ├── test │ ├── avro │ │ └── test.avsc │ ├── java │ │ └── com │ │ │ └── bakdata │ │ │ └── quick │ │ │ └── common │ │ │ ├── AvroJacksonConfigurationTest.java │ │ │ ├── api │ │ │ └── client │ │ │ │ ├── TestUtils.java │ │ │ │ ├── mirror │ │ │ │ ├── ClientUtilsTest.java │ │ │ │ ├── MirrorHostTest.java │ │ │ │ ├── PartitionedMirrorClientTest.java │ │ │ │ ├── StreamsStateHostTest.java │ │ │ │ └── TopicRegistryMirrorClientTest.java │ │ │ │ └── routing │ │ │ │ └── PartitionRouterTest.java │ │ │ ├── config │ │ │ ├── AvroConfigTest.java │ │ │ ├── KafkaConfigTest.java │ │ │ ├── ProtobufConfigTest.java │ │ │ ├── SchemaConfigTest.java │ │ │ └── TopicRegistryConfigTest.java │ │ │ ├── metrics │ │ │ ├── MetricsFactoryDisabledTest.java │ │ │ └── MetricsFactoryTest.java │ │ │ ├── resolver │ │ │ ├── GenericAvroResolverTest.java │ │ │ ├── KnownTypeResolverTest.java │ │ │ └── ProtobufResolverTest.java │ │ │ ├── schema │ │ │ ├── SchemaProviderFactoryTest.java │ │ │ └── SchemaRegistryFetcherTest.java │ │ │ └── type │ │ │ └── registry │ │ │ └── QuickTopicTypeServiceTest.java │ └── resources │ │ ├── application-test.yaml │ │ ├── product-schema.avsc │ │ └── product.json │ └── testFixtures │ ├── avro │ ├── ChartRecord.avsc │ ├── ClickStats.avsc │ ├── Person.avsc │ ├── Purchase.avsc │ ├── PurchaseList.avsc │ └── RangeQuery.avsc │ ├── java │ └── com │ │ └── bakdata │ │ └── quick │ │ └── common │ │ ├── ConfigUtils.java │ │ ├── TestConfigUtils.java │ │ ├── TestEnvironmentPropertySource.java │ │ ├── TestTopicRegistryClient.java │ │ ├── TestTopicTypeService.java │ │ ├── TestTypeUtils.java │ │ └── tags │ │ └── IntegrationTest.java │ ├── proto │ └── test.proto │ └── resources │ └── log4j2-test.yaml ├── config ├── checkstyle │ ├── checkstyle.xml │ └── suppressions.xml └── copyright │ ├── license_header.txt │ └── quick-license.xml ├── deployment └── helm │ └── quick │ ├── .helmignore │ ├── Chart.yaml │ ├── templates │ ├── _helpers.tpl │ ├── api-key-secret.yaml │ ├── default-service-account.yaml │ ├── ingest-deployment.yaml │ ├── ingest-ingress.yaml │ ├── ingest-service.yaml │ ├── log4j-configmap.yaml │ ├── manager-deployment.yaml │ ├── manager-ingress.yaml │ ├── manager-rbac.yaml │ ├── manager-role.yaml │ ├── manager-service-account.yaml │ ├── manager-service.yaml │ ├── quick-config.yaml │ ├── service-discoverer.yaml │ └── stripprefix-middleware.yaml │ └── values.yaml ├── docs ├── docs │ ├── assets │ │ └── images │ │ │ ├── carsharing.png │ │ │ ├── favicon.webp │ │ │ ├── quick-architecture.jpg │ │ │ ├── source │ │ │ ├── quick-architecture.drawio │ │ │ └── tinyurl-topology.drawio.xml │ │ │ ├── stream-expose-graphic.svg │ │ │ └── tinyurl-topology.png │ ├── changelog.md │ ├── developer │ │ ├── architecture.md │ │ ├── cli.md │ │ ├── contributing.md │ │ ├── development.md │ │ ├── multi-subscription-details.md │ │ ├── notice.md │ │ ├── operations.md │ │ └── range-query-details.md │ ├── index.md │ ├── roadmap.md │ └── user │ │ ├── examples │ │ ├── TinyURL.md │ │ ├── index.md │ │ ├── real-time-customer-profiles.md │ │ └── real-time-monitoring.md │ │ ├── getting-started │ │ ├── index.md │ │ ├── setup-cli.md │ │ ├── setup-quick.md │ │ ├── teardown-resources.md │ │ └── working-with-quick │ │ │ ├── gateway.md │ │ │ ├── index.md │ │ │ ├── ingest-data.md │ │ │ ├── multi-subscriptions.md │ │ │ ├── query-data.md │ │ │ ├── range-query.md │ │ │ ├── subscriptions.md │ │ │ └── topics.md │ │ ├── index.md │ │ └── reference │ │ ├── breaking-changes.md │ │ ├── cli-commands.md │ │ ├── configuration.md │ │ ├── dependency-versions.md │ │ ├── graphql-support.md │ │ └── helm-chart.md ├── main.py ├── mkdocs.yml ├── overrides │ └── home.html └── requirements.txt ├── e2e └── functional │ ├── Dockerfile │ ├── README.md │ ├── crud │ ├── query-new-item.gql │ ├── query.gql │ ├── schema.gql │ └── tests.bats │ ├── entrypoint.sh │ ├── multi-stream │ ├── products.json │ ├── purchases.json │ ├── query.gql │ ├── result.json │ ├── schema.gql │ └── tests.bats │ ├── range-key │ ├── purchases.json │ ├── query-range.gql │ ├── result-range.json │ ├── schema.gql │ └── tests.bats │ ├── range │ ├── products.json │ ├── query-range.gql │ ├── result-range.json │ ├── schema.gql │ └── tests.bats │ ├── schema │ ├── invalid-product.json │ ├── products.json │ ├── query-all.gql │ ├── query-list.gql │ ├── query-single.gql │ ├── result-all.json │ ├── result-list.json │ ├── result-single.json │ ├── schema-query-all.gql │ ├── schema-query-single.gql │ └── tests.bats │ └── subscription │ ├── products.json │ ├── purchases.json │ ├── query.gql │ ├── schema.gql │ ├── subscription.gql │ ├── tests.bats │ └── users.json ├── gateway ├── build.gradle.kts └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── bakdata │ │ │ └── quick │ │ │ └── gateway │ │ │ ├── DataFetcherSpecification.java │ │ │ ├── GatewayApplication.java │ │ │ ├── GatewayConfig.java │ │ │ ├── GatewayController.java │ │ │ ├── GatewayInitializer.java │ │ │ ├── GraphQLSchemaGenerator.java │ │ │ ├── QuickGraphQLContext.java │ │ │ ├── QuickGraphQLInvocation.java │ │ │ ├── TypePrinter.java │ │ │ ├── custom │ │ │ ├── QuickPropertyDataFetcher.java │ │ │ ├── QuickWiringFactory.java │ │ │ └── type │ │ │ │ ├── QuickGraphQLType.java │ │ │ │ ├── RestDirectiveMethod.java │ │ │ │ └── RestDirectiveMethodType.java │ │ │ ├── directives │ │ │ ├── QuickDirective.java │ │ │ ├── QuickDirectiveException.java │ │ │ ├── QuickDirectiveWiring.java │ │ │ ├── rest │ │ │ │ ├── RestDirective.java │ │ │ │ ├── RestDirectiveWiring.java │ │ │ │ └── RestParameter.java │ │ │ └── topic │ │ │ │ ├── TopicDirective.java │ │ │ │ ├── TopicDirectiveContext.java │ │ │ │ ├── TopicDirectiveWiring.java │ │ │ │ └── rule │ │ │ │ ├── TopicDirectiveRule.java │ │ │ │ ├── TopicDirectiveRules.java │ │ │ │ ├── fetcher │ │ │ │ ├── DataFetcherRule.java │ │ │ │ ├── DataFetcherRules.java │ │ │ │ ├── ListArgumentFetcherRule.java │ │ │ │ ├── ModificationListRule.java │ │ │ │ ├── ModificationRule.java │ │ │ │ ├── MutationRule.java │ │ │ │ ├── QueryFetcherRule.java │ │ │ │ ├── QueryListFetcherRule.java │ │ │ │ ├── RangeFetcherRule.java │ │ │ │ └── SubscriptionRule.java │ │ │ │ └── validation │ │ │ │ ├── ExclusiveArguments.java │ │ │ │ ├── KeyInformation.java │ │ │ │ ├── MutationRequiresTwoArguments.java │ │ │ │ ├── RangeArguments.java │ │ │ │ ├── SubscriptionList.java │ │ │ │ ├── ValidationRule.java │ │ │ │ ├── ValidationRules.java │ │ │ │ └── ValidationUtility.java │ │ │ ├── fetcher │ │ │ ├── ClientSupplier.java │ │ │ ├── DataFetcherClient.java │ │ │ ├── DefaultClientSupplier.java │ │ │ ├── DeferFetcher.java │ │ │ ├── FetcherFactory.java │ │ │ ├── KeyFieldFetcher.java │ │ │ ├── ListArgumentFetcher.java │ │ │ ├── ListFieldFetcher.java │ │ │ ├── MirrorDataFetcherClient.java │ │ │ ├── MutationFetcher.java │ │ │ ├── QueryKeyArgumentFetcher.java │ │ │ ├── QueryListFetcher.java │ │ │ ├── RangeQueryFetcher.java │ │ │ ├── RestDataFetcher.java │ │ │ └── subscription │ │ │ │ ├── KafkaSubscriptionProvider.java │ │ │ │ ├── MultiSubscriptionFetcher.java │ │ │ │ ├── SubscriptionFetcher.java │ │ │ │ └── SubscriptionProvider.java │ │ │ ├── ingest │ │ │ ├── IngestService.java │ │ │ └── KafkaIngestService.java │ │ │ ├── subscriptions │ │ │ ├── GraphQLRequestBody.java │ │ │ ├── GraphQLWsAuthFilter.java │ │ │ ├── GraphQLWsController.java │ │ │ ├── GraphQLWsFilter.java │ │ │ ├── GraphQLWsFilterChain.java │ │ │ ├── GraphQLWsMessageHandler.java │ │ │ ├── GraphQLWsOperations.java │ │ │ ├── GraphQLWsRequest.java │ │ │ ├── GraphQLWsSender.java │ │ │ └── GraphQLWsState.java │ │ │ └── transformer │ │ │ └── MultiSubscriptionTransformer.java │ └── resources │ │ └── application.yaml │ └── test │ ├── java │ └── com │ │ └── bakdata │ │ └── quick │ │ └── gateway │ │ ├── ControllerReturnSchemaTest.java │ │ ├── ControllerUpdateSchemaTest.java │ │ ├── GatewayConfigTest.java │ │ ├── GatewayInitializerTest.java │ │ ├── GraphQLQueryExecutionTest.java │ │ ├── GraphQLSchemaGeneratorTest.java │ │ ├── GraphQLSecurityTest.java │ │ ├── GraphQLTestUtil.java │ │ ├── GraphQLWsClient.java │ │ ├── directives │ │ └── rest │ │ │ ├── RestDirectiveTest.java │ │ │ └── RestDirectiveWiringTest.java │ │ ├── fetcher │ │ ├── KeyFieldFetcherTest.java │ │ ├── ListFieldFetcherTest.java │ │ ├── MutationFetcherTest.java │ │ ├── QueryKeyArgumentFetcherTest.java │ │ ├── QueryListArgumentFetcherTest.java │ │ ├── QueryListFetcherTest.java │ │ ├── RangeQueryFetcherTest.java │ │ ├── RestDataFetcherTest.java │ │ ├── TestModels.java │ │ └── subscription │ │ │ ├── MultiSubscriptionFetcherTest.java │ │ │ └── SubscriptionFetcherTest.java │ │ └── security │ │ └── ApiKeyTest.java │ └── resources │ ├── application-test.yaml │ ├── initializer │ └── schema.graphql │ └── schema │ ├── contract.graphql │ ├── conversion │ ├── shouldConvertComplexSubscription.graphql │ ├── shouldConvertDefinitionWithSingleFieldAndObjectName.graphql │ ├── shouldConvertIfMultipleValues.graphql │ ├── shouldConvertListQueryWithSingleFieldAndModification.graphql │ ├── shouldConvertMutation.graphql │ ├── shouldConvertQueryAllWithComplexType.graphql │ ├── shouldConvertQueryAllWithPrimitiveType.graphql │ ├── shouldConvertQueryWithListArgument.graphql │ ├── shouldConvertQueryWithMultipleFields.graphql │ ├── shouldConvertQueryWithPrimitiveType.graphql │ ├── shouldConvertQueryWithRange.graphql │ ├── shouldConvertQueryWithRangeOnField.graphql │ ├── shouldConvertQueryWithSingleField.graphql │ ├── shouldConvertQueryWithSingleFieldAndModification.graphql │ ├── shouldConvertSubscription.graphql │ ├── shouldNotConvertIfKeyArgAndInputNameDifferentInNonQueryType.graphql │ ├── shouldNotConvertIfKeyArgAndInputNameDifferentInQueryType.graphql │ ├── shouldNotConvertIfMissingKeyInfoInBasicType.graphql │ ├── shouldNotConvertIfMissingKeyInfoInQueryType.graphql │ ├── shouldNotConvertIfMutationDoesNotHaveTwoArgs.graphql │ ├── shouldNotConvertIfRangeToArgumentIsMissing.graphql │ ├── shouldNotConvertKeyFieldWithWrongFieldType.graphql │ ├── shouldNotConvertKeyFieldWithWrongKeyFieldName.graphql │ ├── shouldNotCovertIfKeyArgumentInRangeQueryIsMissing.graphql │ ├── shouldNotCovertIfRangeFromArgumentIsMissing.graphql │ ├── shouldNotCovertIfRangeIsDefinedOnField.graphql │ └── shouldNotCovertIfReturnTypeOfRangeQueryIsNotList.graphql │ ├── directives │ └── rest │ │ ├── directive │ │ ├── shouldParseBodyParameter.graphql │ │ ├── shouldParseHttpMethod.graphql │ │ ├── shouldParseMultipleDirectives.graphql │ │ ├── shouldParseNonOptionalArguments.graphql │ │ └── shouldParseOptionalArguments.graphql │ │ └── wiring │ │ ├── shouldDefaultToGetMethod.graphql │ │ ├── shouldNotAllowSameArgumentTwice.graphql │ │ ├── shouldNotParseNonExistingPathArgument.graphql │ │ ├── shouldNotParseNonExistingQueryArgument.graphql │ │ ├── shouldParseBodyParameter.graphql │ │ ├── shouldParseEnum.graphql │ │ ├── shouldParseHttpMethod.graphql │ │ ├── shouldParseMultipleArguments.graphql │ │ ├── shouldParseOverriddenArguments.graphql │ │ ├── shouldParsePathArguments.graphql │ │ ├── shouldParseQueryArguments.graphql │ │ ├── shouldParseSpecifiedPathArguments.graphql │ │ └── shouldRespectOrderOfOverriddenArguments.graphql │ ├── execution │ ├── shouldExecuteDefinitionWithSingleFieldAndObjectName.graphql │ ├── shouldExecuteDefinitionWithSingleFieldAndObjectNameQuery.graphql │ ├── shouldExecuteListQueryWithSingleFieldAndModification.graphql │ ├── shouldExecuteListQueryWithSingleFieldAndModificationQuery.graphql │ ├── shouldExecuteQueryAllWithPrimitiveType.graphql │ ├── shouldExecuteQueryAllWithPrimitiveTypeQuery.graphql │ ├── shouldExecuteQueryWithListArgumentTypeId.graphql │ ├── shouldExecuteQueryWithListArgumentTypeIdQuery.graphql │ ├── shouldExecuteQueryWithListArgumentTypeInt.graphql │ ├── shouldExecuteQueryWithListArgumentTypeIntQuery.graphql │ ├── shouldExecuteQueryWithPrimitiveType.graphql │ ├── shouldExecuteQueryWithPrimitiveTypeQuery.graphql │ ├── shouldExecuteQueryWithSingleField.graphql │ ├── shouldExecuteQueryWithSingleFieldAndModification.graphql │ ├── shouldExecuteQueryWithSingleFieldAndModificationQuery.graphql │ ├── shouldExecuteQueryWithSingleFieldQuery.graphql │ ├── shouldExecuteRange.graphql │ └── shouldExecuteRangeQuery.graphql │ ├── person.graphql │ ├── product.graphql │ ├── purchase.graphql │ └── schema.graphql ├── gradle.properties ├── gradle ├── libs.versions.toml └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── ingest ├── build.gradle.kts └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── bakdata │ │ │ └── quick │ │ │ └── ingest │ │ │ ├── IngestApplication.java │ │ │ ├── controller │ │ │ └── IngestController.java │ │ │ └── service │ │ │ ├── IngestFilter.java │ │ │ ├── IngestParser.java │ │ │ ├── IngestService.java │ │ │ └── KafkaIngestService.java │ └── resources │ │ └── application.yaml │ └── test │ ├── java │ └── com │ │ └── bakdata │ │ └── quick │ │ └── ingest │ │ ├── controller │ │ └── IngestControllerTest.java │ │ ├── security │ │ └── ApiKeyTest.java │ │ └── service │ │ └── KafkaIngestServiceTest.java │ └── resources │ └── application-test.yaml ├── justfile ├── lombok.config ├── manager ├── build.gradle.kts └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── bakdata │ │ │ └── quick │ │ │ └── manager │ │ │ ├── ManagerApplication.java │ │ │ ├── ManagerFactory.java │ │ │ ├── application │ │ │ ├── ApplicationController.java │ │ │ ├── ApplicationService.java │ │ │ ├── KubernetesApplicationService.java │ │ │ └── resources │ │ │ │ ├── ApplicationDeployment.java │ │ │ │ ├── ApplicationResourceLoader.java │ │ │ │ ├── ApplicationResources.java │ │ │ │ └── ApplicationService.java │ │ │ ├── config │ │ │ ├── ApplicationSpecificationConfig.java │ │ │ ├── DeploymentConfig.java │ │ │ ├── HardwareResource.java │ │ │ ├── ImagePullPolicy.java │ │ │ └── ManagerConfig.java │ │ │ ├── gateway │ │ │ ├── GatewayController.java │ │ │ ├── GatewayService.java │ │ │ ├── KubernetesGatewayService.java │ │ │ └── resource │ │ │ │ ├── GatewayConfigMap.java │ │ │ │ ├── GatewayDeployment.java │ │ │ │ ├── GatewayIngress.java │ │ │ │ ├── GatewayMiddleware.java │ │ │ │ ├── GatewayResourceLoader.java │ │ │ │ ├── GatewayResources.java │ │ │ │ └── GatewayService.java │ │ │ ├── graphql │ │ │ ├── GraphQLConverter.java │ │ │ ├── GraphQLToAvroConverter.java │ │ │ └── GraphQLToProtobufConverter.java │ │ │ ├── k8s │ │ │ ├── ImageConfig.java │ │ │ ├── KubernetesManagerClient.java │ │ │ ├── KubernetesResources.java │ │ │ ├── cluster │ │ │ │ ├── ImageUpdater.java │ │ │ │ ├── JobCleaner.java │ │ │ │ ├── TopicRegistryInitializer.java │ │ │ │ └── package-info.java │ │ │ ├── middleware │ │ │ │ ├── Middleware.java │ │ │ │ ├── MiddlewareList.java │ │ │ │ ├── MiddlewareSpec.java │ │ │ │ └── StripPrefix.java │ │ │ ├── package-info.java │ │ │ └── resource │ │ │ │ ├── KubernetesErrorHandler.java │ │ │ │ ├── QuickResource.java │ │ │ │ ├── QuickResourceErrorHandler.java │ │ │ │ ├── QuickResources.java │ │ │ │ └── ResourceLoader.java │ │ │ ├── mirror │ │ │ ├── KubernetesMirrorService.java │ │ │ ├── MirrorController.java │ │ │ ├── MirrorService.java │ │ │ └── resources │ │ │ │ ├── MirrorDeployment.java │ │ │ │ ├── MirrorResourceLoader.java │ │ │ │ ├── MirrorResources.java │ │ │ │ └── MirrorService.java │ │ │ └── topic │ │ │ ├── KafkaTopicService.java │ │ │ ├── TopicController.java │ │ │ └── TopicService.java │ └── resources │ │ ├── application.yaml │ │ └── k8s │ │ ├── crd │ │ └── middleware-crd.yaml │ │ └── templates │ │ ├── gateway │ │ ├── config-map.th.yaml │ │ ├── deployment.th.yaml │ │ ├── ingress.th.yaml │ │ ├── middleware.th.yaml │ │ └── service.th.yaml │ │ ├── mirror │ │ ├── deployment.th.yaml │ │ └── service.th.yaml │ │ └── streamsApp │ │ ├── deletion-job.th.yaml │ │ ├── deployment.th.yaml │ │ └── service.th.yaml │ └── test │ ├── java │ └── com │ │ └── bakdata │ │ └── quick │ │ └── manager │ │ ├── TestUtil.java │ │ ├── application │ │ ├── ApplicationControllerTest.java │ │ ├── KubernetesApplicationServiceTest.java │ │ └── resources │ │ │ └── ApplicationResourceLoaderTest.java │ │ ├── config │ │ ├── DeploymentConfigInjectionTest.java │ │ └── ManagerConfigTest.java │ │ ├── gateway │ │ ├── GatewayControllerTest.java │ │ ├── KubernetesGatewayServiceTest.java │ │ └── resource │ │ │ └── GatewayResourceLoaderTest.java │ │ ├── graphql │ │ ├── GraphQLToAvroConverterTest.java │ │ └── GraphQLToProtobufConverterTest.java │ │ ├── k8s │ │ ├── KubernetesTest.java │ │ ├── cluster │ │ │ ├── ImageUpdaterTest.java │ │ │ ├── JobCleanerTest.java │ │ │ └── TopicRegistryInitializerTest.java │ │ └── config │ │ │ └── ApplicationSpecificationConfigTest.java │ │ ├── mirror │ │ ├── KubernetesMirrorServiceTest.java │ │ ├── MirrorControllerTest.java │ │ └── resources │ │ │ └── MirrorResourceLoaderTest.java │ │ ├── security │ │ └── ApiKeyTest.java │ │ └── topic │ │ ├── KafkaTopicServiceTest.java │ │ └── TopicControllerTest.java │ └── resources │ ├── application-test.yaml │ └── schema │ ├── avro │ └── test.avsc │ └── graphql │ ├── shouldConvertGraphQLEnumFields.graphql │ ├── shouldConvertGraphQLObjectTypes.graphql │ ├── shouldConvertGraphQLScalarFields.graphql │ ├── shouldConvertListType.graphql │ ├── shouldConvertOptionalAndRequired.graphql │ └── shouldCreateConfigmapWithSchema.graphql ├── mirror ├── build.gradle.kts └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── bakdata │ │ │ └── quick │ │ │ └── mirror │ │ │ ├── IndexInputStreamBuilder.java │ │ │ ├── MirrorApplication.java │ │ │ ├── MirrorController.java │ │ │ ├── StoreType.java │ │ │ ├── StreamsStateController.java │ │ │ ├── base │ │ │ ├── HostConfig.java │ │ │ └── QuickTopologyData.java │ │ │ ├── context │ │ │ ├── DefaultContextProvider.java │ │ │ ├── IndexInputStream.java │ │ │ ├── MirrorContext.java │ │ │ ├── MirrorContextProvider.java │ │ │ ├── RangeIndexProperties.java │ │ │ └── RetentionTimeProperties.java │ │ │ ├── point │ │ │ └── MirrorProcessor.java │ │ │ ├── range │ │ │ ├── MirrorRangeProcessor.java │ │ │ ├── extractor │ │ │ │ ├── AvroExtractor.java │ │ │ │ ├── ProtoExtractor.java │ │ │ │ ├── SchemaExtractor.java │ │ │ │ ├── type │ │ │ │ │ ├── AvroTypeExtractor.java │ │ │ │ │ ├── FieldTypeExtractor.java │ │ │ │ │ └── ProtoTypeExtractor.java │ │ │ │ └── value │ │ │ │ │ ├── FieldValueExtractor.java │ │ │ │ │ ├── GenericRecordValueExtractor.java │ │ │ │ │ └── MessageValueExtractor.java │ │ │ ├── indexer │ │ │ │ ├── RangeIndexer.java │ │ │ │ ├── ReadRangeIndexer.java │ │ │ │ └── WriteRangeIndexer.java │ │ │ └── padder │ │ │ │ ├── EndRange.java │ │ │ │ ├── IntPadder.java │ │ │ │ ├── LongPadder.java │ │ │ │ ├── ZeroPadder.java │ │ │ │ └── ZeroPadderFactory.java │ │ │ ├── retention │ │ │ ├── RetentionMirrorProcessor.java │ │ │ └── RetentionPunctuator.java │ │ │ ├── service │ │ │ ├── KafkaQueryService.java │ │ │ └── QueryService.java │ │ │ └── topology │ │ │ ├── MirrorTopology.java │ │ │ ├── TopologyFactory.java │ │ │ └── strategy │ │ │ ├── PointTopology.java │ │ │ ├── RangeTopology.java │ │ │ ├── RetentionTopology.java │ │ │ └── TopologyStrategy.java │ └── resources │ │ └── application.yaml │ └── test │ ├── java │ └── com │ │ └── bakdata │ │ └── quick │ │ └── mirror │ │ ├── MirrorControllerTest.java │ │ ├── MirrorTopologyTest.java │ │ ├── point │ │ ├── PointIndexMirrorIntegrationTest.java │ │ └── PointIndexStreamsStateIntegrationTest.java │ │ ├── range │ │ ├── MirrorRangeTopologyTest.java │ │ ├── MirrorWithRangeKeyIntegrationTest.java │ │ ├── RangeIndexMirrorIntegrationTest.java │ │ ├── RangeStreamsStateIntegrationTest.java │ │ ├── WriteRangeIndexerTest.java │ │ ├── extractor │ │ │ └── FieldValueExtractorTest.java │ │ └── padder │ │ │ └── ZeroPadderTest.java │ │ └── topology │ │ └── strategy │ │ └── TopologyStrategyTest.java │ └── resources │ └── application-test.yaml ├── openapi └── spec │ ├── Quick-Manager-v1.yaml │ ├── parameters.yaml │ ├── responses.yaml │ └── schemas.yaml └── settings.gradle.kts /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | ### Issue description 12 | 13 | 14 | 15 | ### Expected Behavior 16 | 17 | 18 | ### Current Behavior 19 | 20 | 21 | ### Steps to Reproduce 22 | 23 | 24 | 1. 25 | 2. 26 | 3. 27 | 4. 28 | 29 | ### Detailed Description 30 | 31 | 32 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yaml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | -------------------------------------------------------------------------------- /.github/actions/push-docker-images/action.yml: -------------------------------------------------------------------------------- 1 | name: "Push Quick Docker Images" 2 | description: "Build and push Quick's docker images to Docker Hub with jib" 3 | inputs: 4 | tag: 5 | description: "Image tag" 6 | required: true 7 | username: 8 | description: "Docker Hub Username" 9 | required: true 10 | token: 11 | description: "Docker Hub Access Token" 12 | required: true 13 | runs: 14 | using: "composite" 15 | steps: 16 | - name: Set up JDK build 11 17 | uses: actions/setup-java@v1 18 | with: 19 | java-version: 11 20 | 21 | - name: Login to Docker Hub 22 | uses: docker/login-action@v1 23 | with: 24 | username: ${{ inputs.username }} 25 | password: ${{ inputs.token }} 26 | 27 | - name: Cache Gradle 28 | uses: actions/cache@v2 29 | with: 30 | path: | 31 | ~/.gradle/caches 32 | ~/.gradle/wrapper 33 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} 34 | restore-keys: | 35 | ${{ runner.os }}-gradle- 36 | 37 | - name: Create and push PR image 38 | shell: bash 39 | run: ./gradlew -Pversion=${{ inputs.tag }} jib --no-daemon 40 | -------------------------------------------------------------------------------- /.github/actions/test-source/action.yml: -------------------------------------------------------------------------------- 1 | name: "Test Quick Source" 2 | description: "Run unit tests and if not disabled the integration test of Quick with Gradle." 3 | inputs: 4 | disable-integration-test: 5 | description: "Flag for running integration tests" 6 | required: true 7 | default: "false" 8 | runs: 9 | using: "composite" 10 | steps: 11 | - name: Set up JDK build 11 12 | uses: actions/setup-java@v1 13 | with: 14 | java-version: 11 15 | 16 | - name: Cache Gradle 17 | uses: actions/cache@v2 18 | with: 19 | path: | 20 | ~/.gradle/caches 21 | ~/.gradle/wrapper 22 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} 23 | restore-keys: | 24 | ${{ runner.os }}-gradle- 25 | 26 | - name: checkstyle main 27 | shell: bash 28 | run: ./gradlew checkstyleMain 29 | 30 | - name: checkstyle test 31 | shell: bash 32 | run: ./gradlew checkstyleTest 33 | 34 | - name: Run unit tests 35 | shell: bash 36 | run: ./gradlew unitTest 37 | 38 | - name: Run integration tests 39 | shell: bash 40 | run: | 41 | if [ "$DISABLE_INTEGRATION_TEST" = "false" ]; then 42 | ./gradlew integrationTest 43 | else 44 | echo "Skip integration test" 45 | fi 46 | env: 47 | DISABLE_INTEGRATION_TEST: ${{ inputs.disable-integration-test }} 48 | -------------------------------------------------------------------------------- /.github/actions/update-docs/action.yml: -------------------------------------------------------------------------------- 1 | name: "Update documentation in gh-pages" 2 | description: | 3 | Compile markdown documents to html and deploy to docs branch. If a semver tag is given, this action strips the patch. 4 | It then pushes the . as the latest alias to the documentation branch with mike. 5 | In case no tag is given, it pushes to 'dev'. 6 | inputs: 7 | token: 8 | description: "GitHub Token (must be a PAT for repository dispatch)" 9 | required: true 10 | tag: 11 | description: "Version tag" 12 | required: false 13 | runs: 14 | using: "composite" 15 | steps: 16 | - uses: actions/setup-python@v1 17 | with: 18 | python-version: 3.9 19 | - name: Update documentation branch with mike 20 | shell: bash 21 | env: 22 | TOKEN: ${{ inputs.token }} 23 | NEW_TAG: ${{ inputs.tag }} 24 | run: | 25 | pip install -r ./docs/requirements.txt 26 | git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" 27 | git config --local user.name "github-actions[bot]" 28 | git config --local user.password ${TOKEN} 29 | 30 | git pull # make sure docs branch is up-to-date 31 | if [ -z "$NEW_TAG" ]; then 32 | mike deploy dev --push --rebase --config-file ./docs/mkdocs.yml 33 | else 34 | new_tag=${NEW_TAG%.*} 35 | mike deploy "$new_tag" latest --update-aliases --push --rebase --config-file ./docs/mkdocs.yml 36 | fi 37 | -------------------------------------------------------------------------------- /.github/changelog-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "prefix": "**Updated dependencies:**", 4 | "labels": ["dependency", "type/update"] 5 | }, 6 | "deployment": { 7 | "prefix": "**Deployment/Helm chart:**", 8 | "labels": ["deployment", "area/deployment"] 9 | }, 10 | "documentation": { 11 | "prefix": "**Documentation updates:**", 12 | "labels": ["documentation", "area/documentation"] 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /.github/path_filter.yaml: -------------------------------------------------------------------------------- 1 | source: 2 | - "build-logic/**" 3 | - "common/**" 4 | - "gateway/**" 5 | - "ingest/**" 6 | - "manager/**" 7 | - "mirror/**" 8 | - "build.gradle.kts" 9 | - "gradle/**" 10 | - "gradle.properties" 11 | - "gradlew" 12 | - "gradlew.bat" 13 | - "settings.gradle.kts" 14 | - "lombok.config" 15 | helm: 16 | - "deployment/helm/**" 17 | docs: 18 | - "docs/**" 19 | -------------------------------------------------------------------------------- /.github/workflows/gradle-wrapper-validation.yml: -------------------------------------------------------------------------------- 1 | name: "Validate Gradle Wrapper" 2 | on: 3 | pull_request: 4 | paths: 5 | - 'gradle/wrapper/**' 6 | push: 7 | branches: [ master ] 8 | paths: 9 | - 'gradle/wrapper/**' 10 | 11 | jobs: 12 | validation: 13 | name: "Gradle wrapper validation" 14 | runs-on: ubuntu-20.04 15 | steps: 16 | - name: Checkout code 17 | uses: actions/checkout@v2 18 | 19 | - name: Validate Gradle wrapper 20 | uses: gradle/wrapper-validation-action@v1 21 | -------------------------------------------------------------------------------- /build-logic/quick-plugins/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | `kotlin-dsl` 3 | } 4 | 5 | group = "buildlogic" 6 | 7 | repositories { 8 | mavenCentral() 9 | gradlePluginPortal() 10 | } 11 | 12 | 13 | dependencies { 14 | implementation("gradle.plugin.com.google.cloud.tools:jib-gradle-plugin:3.2.1") 15 | implementation("io.spring.gradle:dependency-management-plugin:1.0.12.RELEASE") 16 | implementation("io.freefair.gradle:lombok-plugin:6.1.0") 17 | implementation("org.sonarsource.scanner.gradle:sonarqube-gradle-plugin:3.4.0.2513") 18 | implementation("com.adarshr:gradle-test-logger-plugin:3.2.0") 19 | implementation("net.ltgt.gradle:gradle-errorprone-plugin:2.0.2") 20 | implementation("com.google.protobuf:protobuf-gradle-plugin:0.8.19") 21 | } 22 | 23 | gradlePlugin { 24 | plugins { 25 | create("reporterPlugin") { 26 | id = "quick.reporter" 27 | implementationClass = "buildlogic.quickplugins.ReporterPlugin" 28 | } 29 | 30 | create("basePlugin") { 31 | id = "quick.base" 32 | implementationClass = "buildlogic.quickplugins.BasePlugin" 33 | } 34 | 35 | create("protobufPlugin") { 36 | id = "quick.protobuf.generator" 37 | implementationClass = "buildlogic.quickplugins.ProtobufGeneratorPlugin" 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /build-logic/quick-plugins/src/main/kotlin/buildlogic/quickplugins/BaseExtension.kt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package buildlogic.quickplugins 18 | 19 | open class BaseExtension { 20 | // Accessor for build DSL 21 | val APPLICATION = ProjectType.APPLICATION 22 | val LIBRARY = ProjectType.LIBRARY 23 | 24 | var type: ProjectType = ProjectType.APPLICATION 25 | 26 | enum class ProjectType { 27 | APPLICATION, 28 | LIBRARY; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /build-logic/settings.gradle.kts: -------------------------------------------------------------------------------- 1 | rootProject.name = "build-logic" 2 | 3 | include("quick-plugins") 4 | -------------------------------------------------------------------------------- /build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | id("quick.reporter") 3 | } 4 | -------------------------------------------------------------------------------- /common/src/main/avro/QuickTopicType.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "com.bakdata.quick.common.api.model", 3 | "type": "enum", 4 | "name": "AvroQuickTopicType", 5 | "symbols": [ 6 | "STRING", 7 | "LONG", 8 | "INTEGER", 9 | "DOUBLE", 10 | "SCHEMA" 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /common/src/main/avro/TopicData.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "com.bakdata.quick.common.api.model", 3 | "type": "record", 4 | "name": "AvroTopicData", 5 | "fields": [ 6 | { 7 | "name": "name", 8 | "type": "string" 9 | }, 10 | { 11 | "name": "writeType", 12 | "type": { 13 | "type": "enum", 14 | "name": "AvroWriteType", 15 | "symbols": [ 16 | "MUTABLE", 17 | "IMMUTABLE" 18 | ] 19 | } 20 | }, 21 | { 22 | "name": "keyType", 23 | "type": "com.bakdata.quick.common.api.model.AvroQuickTopicType" 24 | }, 25 | { 26 | "name": "valueType", 27 | "type": "com.bakdata.quick.common.api.model.AvroQuickTopicType" 28 | }, 29 | { 30 | "name": "schema", 31 | "type": [ 32 | "null", 33 | "string" 34 | ], 35 | "default": null 36 | } 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/client/mirror/HeaderConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.client.mirror; 18 | 19 | /** 20 | * A collection of constants for managing headers. 21 | */ 22 | public class HeaderConstants { 23 | 24 | // The X-Cache-Update header is set when there is a need to update the 25 | // partition-host mapping in the {@link com.bakdata.quick.common.api.client.routing.PartitionRouter}. 26 | // The need arises when a replica of the mirror is added or removed. 27 | public static final String UPDATE_PARTITION_HOST_MAPPING_HEADER = "X-Cache-Update"; 28 | // The constant below indicates the existence of a header. 29 | // See: https://stackoverflow.com/a/65241869 for more details. 30 | public static final String HEADER_EXISTS = "?1"; 31 | 32 | private HeaderConstants() { 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/client/mirror/MirrorRequestManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.client.mirror; 18 | 19 | import edu.umd.cs.findbugs.annotations.Nullable; 20 | import okhttp3.HttpUrl; 21 | 22 | /** 23 | * Responsible for managing requests send to the Mirror. 24 | */ 25 | public interface MirrorRequestManager { 26 | 27 | /** 28 | * Submits a request and processes the response. 29 | * 30 | * @param url A URL for which a request is made 31 | * @return a response body if successful; null if resource has not been found 32 | */ 33 | ResponseWrapper makeRequest(final HttpUrl url); 34 | 35 | @Nullable 36 | T processResponse(final ResponseWrapper responseWrapper, final ParserFunction parser); 37 | } 38 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/client/mirror/ParserFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.client.mirror; 18 | 19 | import com.bakdata.quick.common.api.model.mirror.MirrorValue; 20 | import java.io.IOException; 21 | import java.io.InputStream; 22 | 23 | /** 24 | * Custom parser for values returned by a mirror application. 25 | * 26 | * @param type of the returned value 27 | */ 28 | public interface ParserFunction { 29 | /** 30 | * Parses an input stream of data into a deserialized value. 31 | * 32 | * @param inputStream the raw response body 33 | * @return the deserialized mirror value 34 | * @throws IOException if an error reading the input stream occurs 35 | */ 36 | MirrorValue parse(InputStream inputStream) throws IOException; 37 | } 38 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/client/mirror/TopicRegistryClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.client.mirror; 18 | 19 | import com.bakdata.quick.common.api.model.TopicData; 20 | import io.reactivex.Completable; 21 | import io.reactivex.Single; 22 | import java.util.List; 23 | 24 | /** 25 | * Client for interacting with Quick's topic registry. 26 | */ 27 | public interface TopicRegistryClient { 28 | 29 | Completable register(String name, TopicData data); 30 | 31 | Completable delete(String name); 32 | 33 | Single getTopicData(String name); 34 | 35 | Single topicDataExists(final String name); 36 | 37 | Single> getAllTopics(); 38 | } 39 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/client/routing/DefaultPartitionFinder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.client.routing; 18 | 19 | import org.apache.kafka.common.utils.Utils; 20 | 21 | /** 22 | * PartitionFinder with the basic (default) logic for retrieving partitions. 23 | */ 24 | public class DefaultPartitionFinder implements PartitionFinder { 25 | 26 | @Override 27 | public int getForSerializedKey(final byte[] serializedKey, final int numPartitions) { 28 | return Utils.toPositive(Utils.murmur2(serializedKey)) % numPartitions; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/client/routing/PartitionFinder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.client.routing; 18 | 19 | /** 20 | * Represents logic for finding the partition for a specific key. 21 | */ 22 | public interface PartitionFinder { 23 | 24 | /** 25 | * Calculates the partition for a specific serialized key. 26 | * 27 | * @param serializedKey the byte representation of a key for which the partition is sought 28 | * @param numPartitions the total number of partitions in a topic 29 | * @return an int that represents a partition 30 | */ 31 | int getForSerializedKey(final byte[] serializedKey, final int numPartitions); 32 | } 33 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/model/ErrorMessage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.model; 18 | 19 | import java.io.Serializable; 20 | import lombok.Builder; 21 | import lombok.Value; 22 | 23 | /** 24 | * Standard error message format used for all REST APIs in Quick. 25 | */ 26 | @Value 27 | @Builder 28 | public class ErrorMessage implements Serializable { 29 | String type; 30 | String title; 31 | int code; 32 | String detail; 33 | String uriPath; 34 | } 35 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/model/KeyValueEnum.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.model; 18 | 19 | /** 20 | * Enum differentiating between Kafka's key/value. 21 | */ 22 | public enum KeyValueEnum { 23 | KEY, 24 | VALUE; 25 | 26 | public String asSubject(final String topic) { 27 | return String.format("%s-%s", topic, this.toString().toLowerCase()); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/model/KeyValuePair.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.model; 18 | 19 | import edu.umd.cs.findbugs.annotations.Nullable; 20 | import lombok.Value; 21 | 22 | /** 23 | * POJO for a key value pair. 24 | * 25 | * @param type of the key 26 | * @param type of the value 27 | */ 28 | @Value 29 | public class KeyValuePair { 30 | K key; 31 | @Nullable //for tombstones 32 | V value; 33 | } 34 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/model/TopicData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.model; 18 | 19 | import com.bakdata.quick.common.type.QuickTopicType; 20 | import edu.umd.cs.findbugs.annotations.Nullable; 21 | import lombok.Value; 22 | 23 | /** 24 | * Data for a topic. 25 | */ 26 | @Value 27 | public class TopicData { 28 | String name; 29 | TopicWriteType writeType; 30 | QuickTopicType keyType; 31 | QuickTopicType valueType; 32 | @Nullable String schema; 33 | 34 | } 35 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/model/TopicSchemaTypes.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.model; 18 | 19 | import com.bakdata.quick.common.type.QuickTopicType; 20 | import lombok.Value; 21 | 22 | /** 23 | * POJO for types in a Kafka topic. 24 | */ 25 | @Value 26 | public class TopicSchemaTypes { 27 | QuickTopicType keyType; 28 | QuickTopicType valueType; 29 | } 30 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/model/TopicWriteType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.model; 18 | 19 | /** 20 | * Possible topic wrtie configurations. 21 | */ 22 | public enum TopicWriteType { 23 | MUTABLE, 24 | IMMUTABLE 25 | } 26 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/model/gateway/SchemaData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.model.gateway; 18 | 19 | import lombok.Value; 20 | 21 | /** 22 | * POJO storing GraphQL schema. 23 | */ 24 | @Value 25 | public class SchemaData { 26 | String schema; 27 | } 28 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/model/manager/ApplicationDescription.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.model.manager; 18 | 19 | import lombok.Value; 20 | 21 | /** 22 | * POJO containing application data. 23 | */ 24 | @Value 25 | public class ApplicationDescription { 26 | String name; 27 | } 28 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/model/manager/GatewayDescription.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.model.manager; 18 | 19 | import lombok.Value; 20 | 21 | /** 22 | * POJO for gateway information. 23 | */ 24 | @Value 25 | public class GatewayDescription { 26 | String name; 27 | int replicas; 28 | String tag; 29 | } 30 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/model/manager/GatewaySchema.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.model.manager; 18 | 19 | import lombok.Value; 20 | 21 | /** 22 | * POJO for gateway schema. 23 | */ 24 | @Value 25 | public class GatewaySchema { 26 | String gateway; 27 | String type; 28 | } 29 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/model/manager/creation/ApplicationCreationData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.model.manager.creation; 18 | 19 | import edu.umd.cs.findbugs.annotations.Nullable; 20 | import java.util.Map; 21 | import lombok.Value; 22 | 23 | /** 24 | * User supplied data for creating a new application. 25 | */ 26 | @Value 27 | public class ApplicationCreationData implements CreationData { 28 | String name; 29 | String registry; 30 | String imageName; 31 | String tag; 32 | @Nullable 33 | Integer replicas; 34 | @Nullable 35 | Integer port; 36 | @Nullable 37 | String imagePullSecret; 38 | @Nullable 39 | Map arguments; 40 | } 41 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/model/manager/creation/CreationData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.model.manager.creation; 18 | 19 | /** 20 | * Marker interface for data classes used in creation of resources. 21 | */ 22 | public interface CreationData {} 23 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/model/manager/creation/GatewayCreationData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.model.manager.creation; 18 | 19 | import edu.umd.cs.findbugs.annotations.Nullable; 20 | import lombok.Value; 21 | 22 | /** 23 | * User supplied data for creating a new gateway. 24 | */ 25 | @Value 26 | public class GatewayCreationData implements CreationData { 27 | String name; 28 | @Nullable 29 | Integer replicas; 30 | @Nullable 31 | String tag; 32 | @Nullable 33 | String schema; 34 | } 35 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/model/manager/creation/MirrorArguments.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.model.manager.creation; 18 | 19 | import java.time.Duration; 20 | import lombok.Value; 21 | 22 | /** 23 | * Optional arguments for the mirror. 24 | */ 25 | @Value 26 | public class MirrorArguments { 27 | Duration retentionTime; 28 | String rangeField; 29 | String rangeKey; 30 | } 31 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/model/manager/creation/MirrorCreationData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.model.manager.creation; 18 | 19 | import edu.umd.cs.findbugs.annotations.Nullable; 20 | import lombok.Value; 21 | 22 | /** 23 | * User supplied data for creating a new mirror. 24 | */ 25 | @Value 26 | public class MirrorCreationData implements CreationData { 27 | String name; 28 | String topicName; 29 | @Nullable 30 | Integer replicas; 31 | @Nullable 32 | String tag; 33 | @Nullable 34 | MirrorArguments mirrorArguments; 35 | } 36 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/model/manager/creation/TopicCreationData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.model.manager.creation; 18 | 19 | import com.bakdata.quick.common.api.model.TopicWriteType; 20 | import com.bakdata.quick.common.api.model.manager.GatewaySchema; 21 | import lombok.Value; 22 | 23 | /** 24 | * POJO for data when creating a new topic. 25 | */ 26 | @Value 27 | public class TopicCreationData implements CreationData { 28 | TopicWriteType writeType; 29 | GatewaySchema valueSchema; 30 | GatewaySchema keySchema; 31 | MirrorArguments mirrorArguments; 32 | } 33 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/api/model/mirror/MirrorValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.api.model.mirror; 18 | 19 | import com.fasterxml.jackson.annotation.JsonInclude; 20 | import com.fasterxml.jackson.annotation.JsonInclude.Include; 21 | import lombok.Value; 22 | 23 | /** 24 | * Response sent by the mirror. 25 | * 26 | * @param type of the response 27 | */ 28 | @Value 29 | public class MirrorValue { 30 | @JsonInclude(Include.ALWAYS) // Explicit annotation required, otherwise empty lists are not included 31 | V value; 32 | } 33 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/exception/BadArgumentException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.exception; 18 | 19 | import edu.umd.cs.findbugs.annotations.Nullable; 20 | import io.micronaut.http.HttpStatus; 21 | 22 | /** 23 | * Exception that occurs when the user provided an argument which lead to an error. 24 | */ 25 | public class BadArgumentException extends QuickException { 26 | public BadArgumentException(@Nullable final String message) { 27 | super(message); 28 | } 29 | 30 | @Override 31 | protected HttpStatus getStatus() { 32 | return HttpStatus.BAD_REQUEST; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/exception/BadServiceAnnotationException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.exception; 18 | 19 | import io.micronaut.http.HttpStatus; 20 | 21 | /** 22 | * Throw if d9p.io/has-service Annotation has an unexpected value 23 | */ 24 | public class BadServiceAnnotationException extends QuickException { 25 | private final HttpStatus status; 26 | 27 | public BadServiceAnnotationException(final String message, final HttpStatus status) { 28 | super(message); 29 | this.status = status; 30 | } 31 | 32 | @Override 33 | protected HttpStatus getStatus() { 34 | return this.status; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/exception/HttpClientException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.exception; 18 | 19 | import io.micronaut.http.HttpStatus; 20 | 21 | /** 22 | * Exception thrown when a client responded with a 400+ status code unexpectedly. 23 | */ 24 | public class HttpClientException extends QuickException { 25 | private final HttpStatus status; 26 | 27 | public HttpClientException(final HttpStatus status) { 28 | super(status.getReason()); 29 | this.status = status; 30 | } 31 | 32 | public HttpClientException(final String message, final HttpStatus status) { 33 | super(message); 34 | this.status = status; 35 | } 36 | 37 | @Override 38 | protected HttpStatus getStatus() { 39 | return this.status; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/exception/InternalErrorException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.exception; 18 | 19 | import io.micronaut.http.HttpStatus; 20 | 21 | /** 22 | * A internal error is used when dependencies did not work as expected. 23 | */ 24 | public class InternalErrorException extends QuickException { 25 | public InternalErrorException(final String message) { 26 | super(message); 27 | } 28 | 29 | @Override 30 | protected HttpStatus getStatus() { 31 | return HttpStatus.INTERNAL_SERVER_ERROR; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/exception/MirrorTopologyException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.exception; 18 | 19 | /** 20 | * An internal exception for the mirror. 21 | * 22 | *

23 | * It is thrown whenever something internally goes wrong in the mirror. 24 | */ 25 | public class MirrorTopologyException extends RuntimeException { 26 | 27 | /** 28 | * New instance with an error message. 29 | * 30 | * @param message Exception message 31 | */ 32 | public MirrorTopologyException(final String message) { 33 | super(message); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/exception/NotFoundException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.exception; 18 | 19 | import io.micronaut.http.HttpStatus; 20 | 21 | /** 22 | * Exception if a resource was not found. 23 | */ 24 | public class NotFoundException extends QuickException { 25 | public NotFoundException(final String message) { 26 | super(message); 27 | } 28 | 29 | @Override 30 | protected HttpStatus getStatus() { 31 | return HttpStatus.NOT_FOUND; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/exception/ServiceUnavailableException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.exception; 18 | 19 | import io.micronaut.http.HttpStatus; 20 | 21 | /** 22 | * Thrown when a required service is not accessible. 23 | */ 24 | public class ServiceUnavailableException extends QuickException { 25 | public ServiceUnavailableException(final String message) { 26 | super(message); 27 | } 28 | 29 | @Override 30 | protected HttpStatus getStatus() { 31 | return HttpStatus.SERVICE_UNAVAILABLE; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/exception/schema/SchemaNotFoundException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.exception.schema; 18 | 19 | import com.bakdata.quick.common.exception.QuickException; 20 | import io.micronaut.http.HttpStatus; 21 | 22 | /** 23 | * Exception when subject doesn't exist. 24 | */ 25 | public class SchemaNotFoundException extends QuickException { 26 | 27 | public SchemaNotFoundException(final String subject) { 28 | super(String.format("Subject \"%s\" not found in schema registry", subject)); 29 | } 30 | 31 | @Override 32 | protected HttpStatus getStatus() { 33 | return HttpStatus.NOT_FOUND; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/json/ObjectMapperConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.json; 18 | 19 | import com.fasterxml.jackson.databind.ObjectMapper; 20 | 21 | /** 22 | * Configuration for Jackson's {@link ObjectMapper}. 23 | */ 24 | public interface ObjectMapperConfiguration { 25 | /** 26 | * Configures the object mapper. 27 | * 28 | * @param objectMapper object to configure 29 | */ 30 | void configureObjectMapper(ObjectMapper objectMapper); 31 | } 32 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/resolver/DoubleResolver.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.resolver; 18 | 19 | /** 20 | * Resolver for native double. 21 | */ 22 | public class DoubleResolver implements TypeResolver { 23 | 24 | @Override 25 | public Double fromString(final String value) { 26 | return Double.valueOf(value); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/resolver/IntegerResolver.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.resolver; 18 | 19 | /** 20 | * Resolver for integers. 21 | */ 22 | public class IntegerResolver implements TypeResolver { 23 | 24 | @Override 25 | public Integer fromString(final String value) { 26 | return Integer.valueOf(value); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/resolver/LongResolver.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.resolver; 18 | 19 | /** 20 | * Resolver for long primitives. 21 | */ 22 | public class LongResolver implements TypeResolver { 23 | 24 | @Override 25 | public Long fromString(final String value) { 26 | return Long.valueOf(value); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/resolver/StringResolver.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.resolver; 18 | 19 | /** 20 | * Resolver for strings. 21 | */ 22 | public class StringResolver implements TypeResolver { 23 | 24 | @Override 25 | public String fromString(final String value) { 26 | return value; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/resolver/TypeResolver.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.resolver; 18 | 19 | /** 20 | * A TypeResolver is used for serializing values dynamically in quick. 21 | */ 22 | public interface TypeResolver { 23 | T fromString(String value); 24 | } 25 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/schema/SchemaFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.schema; 18 | 19 | /** 20 | * Schema formats supported by Quick. 21 | */ 22 | public enum SchemaFormat { 23 | AVRO, PROTOBUF 24 | } 25 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/security/SecurityConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.security; 18 | 19 | import io.micronaut.context.annotation.Property; 20 | import jakarta.inject.Singleton; 21 | import lombok.Getter; 22 | 23 | /** 24 | * Configuration for Quick's security. 25 | */ 26 | @Singleton 27 | @Getter 28 | public class SecurityConfig { 29 | private final boolean securityEnabled; 30 | private final String apiKey; 31 | 32 | public SecurityConfig( 33 | @Property(name = "micronaut.security.enabled", defaultValue = "true") final boolean securityEnabled, 34 | @Property(name = "quick.apikey", defaultValue = "none") final String apiKey) { 35 | this.securityEnabled = securityEnabled; 36 | this.apiKey = apiKey; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/type/TopicTypeService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.type; 18 | 19 | import io.reactivex.Completable; 20 | import io.reactivex.Single; 21 | 22 | /** 23 | * Service for retrieving topic information and deleting topics. 24 | */ 25 | public interface TopicTypeService { 26 | Single> getTopicData(final String topic); 27 | 28 | Completable deleteFromTopicRegistry(String topic); 29 | } 30 | -------------------------------------------------------------------------------- /common/src/main/java/com/bakdata/quick/common/type/registry/TypeResolverWithSchema.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.type.registry; 18 | 19 | import com.bakdata.quick.common.resolver.TypeResolver; 20 | import edu.umd.cs.findbugs.annotations.Nullable; 21 | import io.confluent.kafka.schemaregistry.ParsedSchema; 22 | import lombok.Value; 23 | 24 | /** 25 | * Tuple containing the type resolver and the parsed schema. 26 | */ 27 | @Value 28 | public class TypeResolverWithSchema { 29 | TypeResolver typeResolver; 30 | @Nullable 31 | ParsedSchema parsedSchema; 32 | } 33 | -------------------------------------------------------------------------------- /common/src/main/resources/log4j2.yaml: -------------------------------------------------------------------------------- 1 | Configuration: 2 | status: info 3 | monitorInterval: 30 # refresh config every 30s 4 | 5 | Appenders: 6 | Console: 7 | name: Console 8 | target: SYSTEM_OUT 9 | follow: true 10 | PatternLayOut: 11 | disableAnsi: false 12 | Pattern: "%d{HH:mm:ss.SSS} - %highlight{%5p} %style{%logger{36}}{cyan} - %m%n%throwable" 13 | 14 | Loggers: 15 | Root: 16 | level: info 17 | AppenderRef: 18 | ref: Console 19 | logger: 20 | - name: com.bakdata.quick 21 | level: info 22 | -------------------------------------------------------------------------------- /common/src/test/resources/application-test.yaml: -------------------------------------------------------------------------------- 1 | quick.kafka.bootstrap-server: localhost:9092 -------------------------------------------------------------------------------- /common/src/test/resources/product.json: -------------------------------------------------------------------------------- 1 | { 2 | "productId": 123, 3 | "name": "T-Shirt", 4 | "description": "black", 5 | "price": { 6 | "value": 19.99, 7 | "currency": "DOLLAR" 8 | }, 9 | "metadata": { 10 | "created_at": 1591710185, 11 | "source": null 12 | } 13 | } -------------------------------------------------------------------------------- /common/src/testFixtures/avro/ChartRecord.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "com.bakdata.quick.testutil", 3 | "name": "ChartRecord", 4 | "type": "record", 5 | "fields": [ 6 | { 7 | "name": "fieldId", 8 | "type": "long" 9 | }, 10 | { 11 | "name": "countPlays", 12 | "type": "long" 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /common/src/testFixtures/avro/ClickStats.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "com.bakdata.quick.avro", 3 | "name": "ClickStatsAvro", 4 | "type": "record", 5 | "fields": [ 6 | { 7 | "name": "id", 8 | "type": "string" 9 | }, 10 | { 11 | "name": "amount", 12 | "type": "long" 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /common/src/testFixtures/avro/Person.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "com.bakdata.quick.testutil", 3 | "name": "Person", 4 | "type": "record", 5 | "fields": [ 6 | {"name": "firstname", "type": "string"}, 7 | {"name": "lastname", "type": "string"}, 8 | { 9 | "name": "address", 10 | "type": { 11 | "type" : "record", 12 | "name" : "AddressRecord", 13 | "fields" : [ 14 | {"name": "streetaddress", "type": "string"}, 15 | {"name": "city", "type": "string"} 16 | ] 17 | } 18 | } 19 | ] 20 | } -------------------------------------------------------------------------------- /common/src/testFixtures/avro/Purchase.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "com.bakdata.quick.avro", 3 | "name": "PurchaseStatsAvro", 4 | "type": "record", 5 | "fields": [ 6 | { 7 | "name": "id", 8 | "type": "string" 9 | }, 10 | { 11 | "name": "amount", 12 | "type": "long" 13 | }, 14 | { 15 | "name": "productId", 16 | "type": ["null", "int"], 17 | "default": null 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /common/src/testFixtures/avro/PurchaseList.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "com.bakdata.quick.avro", 3 | "name": "PurchaseListAvro", 4 | "type": "record", 5 | "fields": [ 6 | { 7 | "name": "id", 8 | "type": "string" 9 | }, 10 | { 11 | "name": "productIds", 12 | "type": { 13 | "type": "array", 14 | "items": "int" 15 | } 16 | } 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /common/src/testFixtures/avro/RangeQuery.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "com.bakdata.quick.testutil", 3 | "name": "AvroRangeQueryTest", 4 | "type": "record", 5 | "fields": [ 6 | { 7 | "name": "userId", 8 | "type": "int" 9 | }, 10 | { 11 | "name": "timestamp", 12 | "type": "long" 13 | }, 14 | { 15 | "name": "age", 16 | "type": [ 17 | "null", 18 | "int" 19 | ], 20 | "default": null 21 | } 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /common/src/testFixtures/java/com/bakdata/quick/common/TestConfigUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common; 18 | 19 | import com.bakdata.quick.common.config.QuickTopicConfig; 20 | import lombok.experimental.UtilityClass; 21 | 22 | /** 23 | * Test utils for quick configurations. 24 | */ 25 | @UtilityClass 26 | public class TestConfigUtils { 27 | public QuickTopicConfig newQuickTopicConfig() { 28 | return new QuickTopicConfig(2, (short) 1); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /common/src/testFixtures/java/com/bakdata/quick/common/TestEnvironmentPropertySource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common; 18 | 19 | import io.micronaut.context.env.MapPropertySource; 20 | import java.util.Map; 21 | 22 | /** 23 | * Property source for testing manually set environment variables. 24 | */ 25 | public class TestEnvironmentPropertySource extends MapPropertySource { 26 | public static final String NAME = "test"; 27 | 28 | public TestEnvironmentPropertySource(final Map map) { 29 | super(NAME, map); 30 | } 31 | 32 | @Override 33 | public PropertyConvention getConvention() { 34 | return PropertyConvention.ENVIRONMENT_VARIABLE; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /common/src/testFixtures/java/com/bakdata/quick/common/tags/IntegrationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.common.tags; 18 | 19 | import java.lang.annotation.ElementType; 20 | import java.lang.annotation.Retention; 21 | import java.lang.annotation.RetentionPolicy; 22 | import java.lang.annotation.Target; 23 | import org.junit.jupiter.api.Tag; 24 | 25 | /** 26 | * Annotation for integration tests. 27 | * 28 | *

29 | * Integration tests are run included in {@code gradle test}. 30 | * To run them separately, use {@code gradle integrationTest}. 31 | * 32 | *

33 | * For use, see: 34 | * $project/build-logic/convention/src/main/kotlin/buildlogic/convention/BasePlugin.kt 35 | */ 36 | @Target({ElementType.TYPE, ElementType.METHOD}) 37 | @Retention(RetentionPolicy.RUNTIME) 38 | @Tag("integration-test") 39 | public @interface IntegrationTest { 40 | } 41 | -------------------------------------------------------------------------------- /common/src/testFixtures/proto/test.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package bakdata.quick.testutil; 4 | 5 | option java_multiple_files = true; 6 | option java_package = "com.bakdata.quick.testutil"; 7 | 8 | message ProtoTestRecord { 9 | string id = 1; 10 | int32 value = 2; 11 | } 12 | 13 | message ComplexProtoTestRecord { 14 | string id = 1; 15 | ProtoTestRecord protoTestRecord = 2; 16 | } 17 | 18 | message ClickStatsProto { 19 | string id = 1; 20 | int64 amount = 2; 21 | } 22 | 23 | message PurchaseStatsProto { 24 | string id = 1; 25 | int64 amount = 2; 26 | int32 productId = 3; 27 | } 28 | 29 | message ProtoRangeQueryTest { 30 | int64 userId = 1; 31 | int32 timestamp = 2; 32 | } 33 | 34 | message PurchaseListProto { 35 | string id = 1; 36 | repeated int32 productIds = 2; 37 | } 38 | -------------------------------------------------------------------------------- /common/src/testFixtures/resources/log4j2-test.yaml: -------------------------------------------------------------------------------- 1 | Configuration: 2 | status: info 3 | 4 | Appenders: 5 | Console: 6 | name: Console 7 | target: SYSTEM_OUT 8 | follow: true 9 | PatternLayOut: 10 | disableAnsi: false 11 | Pattern: "%d{HH:mm:ss.SSS} - %highlight{%5p} %style{%logger{36}}{cyan} - %m%n%throwable" 12 | 13 | Loggers: 14 | Root: 15 | level: info 16 | additivity: true 17 | AppenderRef: 18 | ref: Console 19 | 20 | logger: 21 | - name: com.bakdata.quick 22 | level: debug 23 | # set to error as they contain mostly info that is not needed 24 | - name: org.apache.kafka 25 | level: error 26 | - name: kafka 27 | level: error 28 | - name: org.apache.curator.test 29 | level: error 30 | - name: org.apache.zookeeper 31 | level: warn 32 | - name: net.mguenther.kafka.junit 33 | level: warn 34 | - name: org.eclipse.jetty.util.log 35 | level: warn 36 | - name: io.micronaut.context.env 37 | level: warn 38 | -------------------------------------------------------------------------------- /config/checkstyle/suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /config/copyright/license_header.txt: -------------------------------------------------------------------------------- 1 | /\* 2 | \* Copyright (\d{4}-)?\d{4} bakdata GmbH 3 | \* 4 | \* Licensed under the Apache License, Version 2.0 \(the "License"\); 5 | \* you may not use this file except in compliance with the License. 6 | \* You may obtain a copy of the License at 7 | \* 8 | \* http://www.apache.org/licenses/LICENSE-2.0 9 | \* 10 | \* Unless required by applicable law or agreed to in writing, software 11 | \* distributed under the License is distributed on an "AS IS" BASIS, 12 | \* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | \* See the License for the specific language governing permissions and 14 | \* limitations under the License. 15 | \*/ 16 | -------------------------------------------------------------------------------- /config/copyright/quick-license.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | -------------------------------------------------------------------------------- /deployment/helm/quick/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /deployment/helm/quick/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: quick 3 | description: A Helm chart for Kubernetes for quick 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: 0.9.0-dev 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | appVersion: 0.9.0-dev 24 | -------------------------------------------------------------------------------- /deployment/helm/quick/templates/api-key-secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: api-key-secret 5 | namespace: {{ .Release.Namespace }} 6 | type: Opaque 7 | data: 8 | QUICK_APIKEY: {{.Values.apiKey | b64enc}} 9 | -------------------------------------------------------------------------------- /deployment/helm/quick/templates/default-service-account.yaml: -------------------------------------------------------------------------------- 1 | kind: RoleBinding 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | name: default-service-discoverer 5 | namespace: {{ .Release.Namespace }} 6 | subjects: 7 | - kind: ServiceAccount 8 | name: default 9 | namespace: {{ .Release.Namespace }} 10 | roleRef: 11 | kind: Role 12 | name: service-discoverer 13 | apiGroup: rbac.authorization.k8s.io 14 | -------------------------------------------------------------------------------- /deployment/helm/quick/templates/ingest-ingress.yaml: -------------------------------------------------------------------------------- 1 | kind: Ingress 2 | apiVersion: networking.k8s.io/v1 3 | metadata: 4 | name: {{ include "quick.ingest.fullname" . }} 5 | namespace: {{ .Release.Namespace }} 6 | annotations: 7 | kubernetes.io/ingress.class: traefik 8 | traefik.ingress.kubernetes.io/router.middlewares: {{ .Release.Namespace }}-service-stripprefix@kubernetescrd 9 | traefik.ingress.kubernetes.io/router.entrypoints: {{ .Values.ingress.entrypoint }} 10 | traefik.ingress.kubernetes.io/router.tls: {{ .Values.ingress.ssl | quote }} 11 | spec: 12 | rules: 13 | - http: 14 | paths: 15 | - path: /ingest 16 | pathType: Prefix 17 | backend: 18 | service: 19 | name: {{ include "quick.ingest.fullname" . }} 20 | port: 21 | number: 80 22 | {{- if .Values.ingress.host }} 23 | host: "{{ .Values.ingress.host }}" 24 | {{- end }} 25 | -------------------------------------------------------------------------------- /deployment/helm/quick/templates/ingest-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "quick.ingest.fullname" . }} 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | {{- include "quick.ingest.labels" . | nindent 4 }} 8 | spec: 9 | type: ClusterIP 10 | ports: 11 | - port: 80 12 | targetPort: 8080 13 | protocol: TCP 14 | name: http 15 | selector: 16 | {{- include "quick.ingest.selectorLabels" . | nindent 4 }} 17 | -------------------------------------------------------------------------------- /deployment/helm/quick/templates/log4j-configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: quick-log4j 5 | data: 6 | log4j2.yaml: |- 7 | {{ if .Values.log4jConfig }} 8 | {{ .Values.log4jConfig | indent 4 }} 9 | {{ else }} 10 | Configuration: 11 | status: info 12 | 13 | Appenders: 14 | Console: 15 | name: Console 16 | target: SYSTEM_OUT 17 | follow: true 18 | PatternLayOut: 19 | disableAnsi: false 20 | Pattern: "%d{HH:mm:ss.SSS} - %highlight{%5p} %style{%logger{36}}{cyan} - %m%n%throwable" 21 | 22 | Loggers: 23 | Root: 24 | level: info 25 | AppenderRef: 26 | ref: Console 27 | {{ end }} 28 | -------------------------------------------------------------------------------- /deployment/helm/quick/templates/manager-ingress.yaml: -------------------------------------------------------------------------------- 1 | kind: Ingress 2 | apiVersion: networking.k8s.io/v1 3 | metadata: 4 | name: {{ include "quick.manager.fullname" . }} 5 | namespace: {{ .Release.Namespace }} 6 | annotations: 7 | kubernetes.io/ingress.class: traefik 8 | traefik.ingress.kubernetes.io/router.middlewares: {{ .Release.Namespace }}-service-stripprefix@kubernetescrd 9 | traefik.ingress.kubernetes.io/router.entrypoints: {{ .Values.ingress.entrypoint }} 10 | traefik.ingress.kubernetes.io/router.tls: {{ .Values.ingress.ssl | quote }} 11 | spec: 12 | rules: 13 | - http: 14 | paths: 15 | - path: /manager 16 | pathType: Prefix 17 | backend: 18 | service: 19 | name: {{ include "quick.manager.fullname" . }} 20 | port: 21 | number: 80 22 | {{- if .Values.ingress.host }} 23 | host: "{{ .Values.ingress.host }}" 24 | {{- end }} 25 | -------------------------------------------------------------------------------- /deployment/helm/quick/templates/manager-rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: {{ .Values.manager.name }}-admin-binding 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | {{- include "quick.labels" . | nindent 4 }} 8 | roleRef: 9 | apiGroup: rbac.authorization.k8s.io 10 | kind: Role 11 | name: manager-admin 12 | subjects: 13 | - kind: ServiceAccount 14 | name: {{ include "quick.manager.service.account.name" . }} 15 | namespace: {{ .Release.Namespace }} 16 | --- 17 | apiVersion: rbac.authorization.k8s.io/v1 18 | kind: RoleBinding 19 | metadata: 20 | name: {{ .Values.manager.name }}-service-binding 21 | namespace: {{ .Release.Namespace }} 22 | labels: 23 | {{- include "quick.labels" . | nindent 4 }} 24 | roleRef: 25 | apiGroup: rbac.authorization.k8s.io 26 | kind: Role 27 | name: service-discoverer 28 | subjects: 29 | - kind: ServiceAccount 30 | name: {{ include "quick.manager.service.account.name" . }} 31 | namespace: {{ .Release.Namespace }} 32 | -------------------------------------------------------------------------------- /deployment/helm/quick/templates/manager-role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: Role 3 | metadata: 4 | name: manager-admin 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | {{- include "quick.labels" . | nindent 4 }} 8 | rules: 9 | - apiGroups: 10 | - "" 11 | - apps 12 | - extensions 13 | - batch 14 | - networking.k8s.io 15 | - traefik.containo.us 16 | resources: 17 | - deployments 18 | - services 19 | - ingresses 20 | - customresourcedefinitions 21 | - endpoints 22 | - jobs 23 | - ingressroutes 24 | - middlewares 25 | - pods 26 | - configmaps 27 | verbs: 28 | - update 29 | - list 30 | - create 31 | - watch 32 | - delete 33 | - deletecollection 34 | - get 35 | - patch 36 | - put 37 | - post 38 | -------------------------------------------------------------------------------- /deployment/helm/quick/templates/manager-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: {{ include "quick.manager.service.account.name" . }} 5 | namespace: {{ .Release.Namespace }} 6 | -------------------------------------------------------------------------------- /deployment/helm/quick/templates/manager-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "quick.manager.fullname" . }} 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | {{- include "quick.manager.labels" . | nindent 4 }} 8 | spec: 9 | type: ClusterIP 10 | ports: 11 | - port: 80 12 | targetPort: 8080 13 | protocol: TCP 14 | name: http 15 | selector: 16 | {{- include "quick.manager.selectorLabels" . | nindent 4 }} 17 | -------------------------------------------------------------------------------- /deployment/helm/quick/templates/quick-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: quick-config 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | {{- include "quick.labels" . | nindent 4 }} 8 | data: 9 | QUICK_DOCKER_REGISTRY: {{ .Values.image.repository }} 10 | QUICK_SERVICE_INGEST: {{ .Values.ingest.name }} 11 | {{- if .Values.ingress.host }} 12 | QUICK_INGRESS_HOST: {{ .Values.ingress.host }} 13 | {{- end }} 14 | QUICK_INGRESS_SSL: {{ .Values.ingress.ssl | quote}} 15 | QUICK_INGRESS_ENTRYPOINT: {{ .Values.ingress.entrypoint }} 16 | KUBERNETES_CLIENT_NAMESPACE: {{ .Release.Namespace }} 17 | {{- with .Values.quickConfig }} 18 | {{- toYaml . | nindent 2 }} 19 | {{- end }} 20 | -------------------------------------------------------------------------------- /deployment/helm/quick/templates/service-discoverer.yaml: -------------------------------------------------------------------------------- 1 | kind: Role 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | name: service-discoverer 5 | namespace: {{ .Release.Namespace }} 6 | rules: 7 | - apiGroups: [""] 8 | resources: ["services", "endpoints", "configmaps", "secrets", "pods"] 9 | verbs: ["get", "watch", "list"] 10 | -------------------------------------------------------------------------------- /deployment/helm/quick/templates/stripprefix-middleware.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: traefik.containo.us/v1alpha1 2 | kind: Middleware 3 | metadata: 4 | name: service-stripprefix 5 | namespace: {{ .Release.Namespace }} 6 | 7 | spec: 8 | stripPrefix: 9 | prefixes: 10 | - /manager 11 | - /ingest 12 | -------------------------------------------------------------------------------- /deployment/helm/quick/values.yaml: -------------------------------------------------------------------------------- 1 | image: 2 | repository: bakdata 3 | tag: "" 4 | pullPolicy: "Always" 5 | 6 | ingress: 7 | ssl: true 8 | entrypoint: "websecure" 9 | host: "" 10 | 11 | manager: 12 | name: "quick-manager" 13 | replicaCount: 1 14 | podAnnotations: {} 15 | 16 | ingest: 17 | name: "quick-ingest" 18 | replicaCount: 1 19 | podAnnotations: {} 20 | 21 | log4jConfig: {} 22 | 23 | apiKey: "" 24 | 25 | quickConfig: 26 | QUICK_KAFKA_BOOTSTRAP_SERVER: quick-kafka.default.svc.cluster.local:9092 27 | QUICK_KAFKA_SCHEMA_REGISTRY_URL: http://quick-sr-schema-registry.default.svc.cluster.local:8081 28 | QUICK_KAFKA_INTERNAL_PARTITIONS: "3" 29 | QUICK_KAFKA_INTERNAL_REPLICATION_FACTOR: "1" 30 | QUICK_TOPIC_REGISTRY_SERVICE_NAME: internal-topic-registry 31 | QUICK_TOPIC_REGISTRY_TOPIC_NAME: __topic-registry 32 | QUICK_TOPIC_REGISTRY_PARTITIONS: "3" 33 | QUICK_TOPIC_REGISTRY_REPLICATION_FACTOR: "1" 34 | QUICK_SCHEMA_FORMAT: "Avro" 35 | QUICK_SCHEMA_AVRO_NAMESPACE: "" 36 | -------------------------------------------------------------------------------- /docs/docs/assets/images/carsharing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bakdata/quick/49bf6a9cf1e7153ea3694b518bbcc7cb50a18c7b/docs/docs/assets/images/carsharing.png -------------------------------------------------------------------------------- /docs/docs/assets/images/favicon.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bakdata/quick/49bf6a9cf1e7153ea3694b518bbcc7cb50a18c7b/docs/docs/assets/images/favicon.webp -------------------------------------------------------------------------------- /docs/docs/assets/images/quick-architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bakdata/quick/49bf6a9cf1e7153ea3694b518bbcc7cb50a18c7b/docs/docs/assets/images/quick-architecture.jpg -------------------------------------------------------------------------------- /docs/docs/assets/images/source/tinyurl-topology.drawio.xml: -------------------------------------------------------------------------------- 1 | 2 | 7VbbbptAEP0aHm3hXdM6j/EtrdRKldyqzeMKxrDNwqBlfCFf3wEWY+I6Sqqk8UMkBDtnZpllzpkRnpyl+xur8uQrRmA84Ud7T849IUZjIbzq8qOyQT5eyQaIrY5cUAes9D040HfoRkdQ9AIJ0ZDO+2CIWQYh9TBlLe76YWs0/ay5iuEEWIXKnKI/dURJg04Cv8M/gY6TNvPId55UtcEOKBIV4e4IkgtPziwiNat0PwNTFa+tS7NvecZ7OJiFjJ6yIb8Pf6Tk34qbLCgH2efx92sxGLuzUdl+MET8/c5ESwnGmCmz6NCpxU0WQfVWn60u5gtizuCIwd9AVDoy1YaQoYRS47x8YFv+cvtr47YyhkFrzvfHznnprDVmtFSpNhWwwo0Ngc8+Y83x45tFF+I8cp4QsUpEIK/5xnWpblVAMYwRYwMq18UwxLR2hEUdulw3CXh5SBGIaZOEF02apmpVqc6S4aCiPcw5BlpRKxsDPRInDpLhXgNMgSvD+ywYRXrbP4dyoo8PcZ0ueOGk8QyZuPduldm4TGRVeDdYA4XJiYQ6gVRs7xJNsMpVXYUdT4m+GNbamBkatPVeeTVdLueTiyK7TtNoeTQ+kL8FS7B/nP5TutoN7VAoH9i7bsaMxg5LjubLB/+VGA7eYhBcDscv2NDiiQ0t37KhxUlDh0wmMYQsbH7cQVm8N/azG1sGl9bY8u9Mv4/uf2Q4mPw/htns/g9r39Fftlz8AQ== -------------------------------------------------------------------------------- /docs/docs/assets/images/tinyurl-topology.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bakdata/quick/49bf6a9cf1e7153ea3694b518bbcc7cb50a18c7b/docs/docs/assets/images/tinyurl-topology.png -------------------------------------------------------------------------------- /docs/docs/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | template: home.html 3 | title: Home 4 | --- 5 | -------------------------------------------------------------------------------- /docs/docs/roadmap.md: -------------------------------------------------------------------------------- 1 | # Roadmap 2 | 3 | The roadmap outlines topics that are currently worked on and gives an overview of the project's future direction. 4 | We track and priorities planned features through the corresponding [milestones](https://github.com/bakdata/quick/milestones) 5 | and [project boards](https://github.com/bakdata/quick/projects). 6 | 7 | ## Upcoming releases 8 | 9 | We are currently rethinking Quick's approach and moving from applications to composable libraries. 10 | 11 | ## Completed releases 12 | 13 | The [changelog](../changelog) has a detailed list of releases. 14 | 15 | ### 0.8 16 | 17 | - Range queries support 18 | - Improved gateway performance: Pre-computation of a key's location 19 | - Kafka 3.0 support 20 | 21 | ### 0.7 22 | 23 | - Protobuf support 24 | 25 | ### 0.6 26 | 27 | - Open-Source release 🎉 28 | -------------------------------------------------------------------------------- /docs/docs/user/examples/index.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | The examples showcase more complex use cases of Quick. 4 | 5 | - [TinyURL](TinyURL.md) is an application for shortening long URLs. 6 | This example shows you how to use applications to transform a data stream. 7 | It also uses immutable topics to ensure the same key isn't created twice. 8 | - [Real-time monitoring and analytics](real-time-monitoring.md) builds a monitoring system for a car-sharing platform with Quick. 9 | - [Real-time customer profiles](real-time-customer-profiles.md) combines several features of Quick 10 | for user profiles on real-world music listening behaviour. 11 | The results of multiple independent apps are managed in a single schema. 12 | This includes a recommendation REST-service. 13 | The frontend also makes use of graphql subscriptions and mutations. 14 | -------------------------------------------------------------------------------- /docs/docs/user/getting-started/index.md: -------------------------------------------------------------------------------- 1 | # Getting started 2 | 3 | The getting started guide gives you an overview of Quick and how to work with it. 4 | 5 | - In [Setup Quick](setup-quick.md), you learn how to deploy Quick's infrastructure and Quick itself. 6 | The guide covers both local and cloud environments. 7 | - [Setup Quick CLI](setup-cli.md) explains the installation process of the main tool for interacting with Quick: the CLI. 8 | - [Working with Quick](working-with-quick/index.md) explains the core concepts of Quick by going through an e-commerce example. 9 | -------------------------------------------------------------------------------- /docs/docs/user/getting-started/setup-cli.md: -------------------------------------------------------------------------------- 1 | # Setup Quick CLI 2 | 3 | The main tool for administrating Quick is its [CLI](https://github.com/bakdata/quick-cli). 4 | Before you can start to work with Quick, you will have to set it up. 5 | 6 | ## Installation 7 | 8 | The first step is to install the Quick CLI. 9 | You can do this via pip. 10 | Quick CLI works with Python versions 3.7-3.9: 11 | ```shell 12 | pip install {{ quick_cli_command() }} 13 | ``` 14 | 15 | The command `quick -v` lets you verify that the installation was successful. 16 | 17 | ## Context configuration 18 | 19 | Next, you can configure the Quick cluster's host and API key. 20 | The CLI's [`context`](../reference/cli-commands.md#quick-context) command manages this configuration. 21 | To create a new context named `guide`, you can run: 22 | 23 | ```shell 24 | quick context create \ 25 | --host "$QUICK_URL" \ 26 | --key "$QUICK_API_KEY" \ 27 | --context guide 28 | ``` 29 | 30 | You can then activate the context with the following command: 31 | 32 | ```shell 33 | quick context activate guide 34 | ``` 35 | 36 | -------------------------------------------------------------------------------- /docs/docs/user/getting-started/teardown-resources.md: -------------------------------------------------------------------------------- 1 | # Teardown resources 2 | 3 | This section progresses from single resources to the underlying infrastructure. 4 | If you want to delete everything, you can skip deleting single resources. 5 | 6 | ## Quick 7 | 8 | - Delete the gateway: 9 | ```shell 10 | quick gateway delete example 11 | ``` 12 | - Delete the topics: 13 | ```shell 14 | quick topic delete purchase 15 | quick topic delete product 16 | ``` 17 | 18 | - Delete the Helm chart: 19 | ```shell 20 | helm delete quick -n quick 21 | ``` 22 | 23 | - Delete the namespace: 24 | ```shell 25 | kubectl delete namespace quick 26 | ``` 27 | 28 | ## Infrastructure 29 | 30 | - Delete the Helm chart: 31 | ```shell 32 | helm delete k8kafka -n infrastructure 33 | ``` 34 | 35 | - Delete the namespace: 36 | ```shell 37 | kubectl delete namespace infrastructure 38 | ``` 39 | 40 | ## Local Cluster 41 | 42 | - Delete the cluster: 43 | ```shell 44 | k3d cluster delete quick 45 | ``` 46 | -------------------------------------------------------------------------------- /docs/docs/user/reference/breaking-changes.md: -------------------------------------------------------------------------------- 1 | # Breaking changes 2 | 3 | ## 0.7 4 | 5 | ### Avro configuration 6 | 7 | - The configuration `QUICK_AVRO_NAMESPACE` is now called `QUICK_SCHEMA_AVRO_NAMESPACE`. 8 | - `avro.namespace` was removed from the Helm chart. Instead, use `QUICK_SCHEMA_AVRO_NAMESPACE` in `quickConfig`. 9 | -------------------------------------------------------------------------------- /docs/docs/user/reference/dependency-versions.md: -------------------------------------------------------------------------------- 1 | # Dependency versions 2 | 3 | | Component | Version | 4 | |--------------------|---------------------| 5 | | Quick | {{ quick_version }} | 6 | | Kubernetes | 1.22 | 7 | | Kafka | 2.7 - 3.2 | 8 | | Schema Registry | 6.1 - 7.2 | 9 | | Traefik | 2.4 - 2.8.4 | 10 | | Quick CLI | {{ quick_version }} | 11 | | Python (Quick CLI) | 3.7-3.9 | 12 | -------------------------------------------------------------------------------- /docs/main.py: -------------------------------------------------------------------------------- 1 | """ 2 | Python functions for defining macros in mkdocs 3 | See https://mkdocs-macros-plugin.readthedocs.io/en/latest/ 4 | """ 5 | 6 | 7 | def define_env(env): 8 | @env.macro 9 | def quick_cli_command(): 10 | """ 11 | Function for setting the pip install command for quick cli. 12 | For dev versions, this uses testpypi and installs the latest version. 13 | For other versions, it install the corresponding quick cli. 14 | """ 15 | quick_version = env.variables["quick_version"] 16 | 17 | if quick_version.endswith("-dev"): 18 | return """--index-url https://test.pypi.org/simple/ \\ 19 | --extra-index-url https://pypi.org/simple/ \\ 20 | quick-cli""" 21 | else: 22 | return f'quick-cli=="{quick_version}"' 23 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | certifi==2022.9.24 2 | charset-normalizer==2.1.1 3 | click==8.0.4 4 | future==0.18.2 5 | ghp-import==2.0.2 6 | idna==3.4 7 | importlib-metadata==4.11.3 8 | Jinja2==3.0.3 9 | joblib==1.2.0 10 | livereload==2.6.3 11 | lunr==0.5.8 12 | Markdown==3.3.6 13 | MarkupSafe==2.1.1 14 | mergedeep==1.3.4 15 | mike==1.1.2 16 | mkdocs==1.4.1 17 | mkdocs-macros-plugin==0.7.0 18 | mkdocs-material==8.5.6 19 | mkdocs-material-extensions==1.0.3 20 | nltk==3.7 21 | packaging==21.3 22 | Pygments==2.13.0 23 | pygments-graphql==1.0.0 24 | pymdown-extensions==9.6 25 | pyparsing==3.0.7 26 | python-dateutil==2.8.2 27 | PyYAML==6.0 28 | pyyaml_env_tag==0.1 29 | regex==2022.3.15 30 | requests==2.28.1 31 | six==1.16.0 32 | termcolor==1.1.0 33 | tornado==6.1 34 | tqdm==4.63.0 35 | urllib3==1.26.12 36 | verspec==0.1.0 37 | watchdog==2.1.6 38 | zipp==3.7.0 39 | -------------------------------------------------------------------------------- /e2e/functional/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8.5-slim 2 | 3 | WORKDIR /tests 4 | 5 | COPY ./entrypoint.sh / 6 | 7 | RUN apt-get update \ 8 | && apt-get install \ 9 | curl \ 10 | unzip \ 11 | jq -y && \ 12 | apt-get clean 13 | 14 | ARG QUICK_CLI_VERSION 15 | ARG INDEX=main 16 | ENV BATS_VERSION 1.7.0 17 | 18 | RUN curl -#L https://github.com/bats-core/bats-core/archive/v${BATS_VERSION}.zip -o bats.zip \ 19 | && unzip ./bats.zip \ 20 | && bash bats-core-${BATS_VERSION}/install.sh /usr/local \ 21 | && rm -rf ./bats-core-${BATS_VERSION} \ 22 | && rm -rf ./bats.zip \ 23 | && pip install aiohttp==3.8.1 \ 24 | && pip install gql==3.0.0 25 | 26 | RUN if [ "$INDEX" = "test" ]; \ 27 | then pip install --index-url https://test.pypi.org/simple/ \ 28 | --extra-index-url https://pypi.org/simple/ quick-cli==${QUICK_CLI_VERSION}; \ 29 | else pip install quick-cli==${QUICK_CLI_VERSION}; \ 30 | fi 31 | RUN chmod +x /entrypoint.sh 32 | 33 | ENTRYPOINT ["/entrypoint.sh"] 34 | -------------------------------------------------------------------------------- /e2e/functional/README.md: -------------------------------------------------------------------------------- 1 | # E2E test 2 | There are three scenarios tested:
3 | 1. CRUD 4 | 2. Schema 5 | 3. Multi-stream 6 | 7 | **NOTE:** Subscription tests are currently skipped by bats. (WIP) 8 | 9 | ## Prerequisite 10 | You can use the [justfile](../../justfile) to build the E2E test runner. 11 | 12 | Example for building the image with the stable version of quick-cli: 13 | ```bash 14 | just e2e-build-runner 15 | ``` 16 | You can find the stable quick-cli version in the [PyPI](https://pypi.org/project/quick-cli/). 17 | 18 | Example for building the image with the dev version of quick-cli: 19 | ```bash 20 | just e2e-build-runner-dev 21 | ``` 22 | You can find the dev quick-cli version in the [Test PyPI](https://test.pypi.org/project/quick-cli/). 23 | 24 | ## How to run e2e tests 25 | Run all the e2e tests use the following command in the current directory: 26 | ```bash 27 | just e2e-run-all 28 | ``` 29 | Alternatively, you can run each test. For example: 30 | ```bash 31 | just e2e-run-crud 32 | ``` 33 | or 34 | ```bash 35 | just e2e-run-range 36 | ``` 37 | 38 | For help just run: 39 | ```bash 40 | just 41 | ``` -------------------------------------------------------------------------------- /e2e/functional/crud/query-new-item.gql: -------------------------------------------------------------------------------- 1 | { 2 | findProduct(productId: 456) 3 | } 4 | -------------------------------------------------------------------------------- /e2e/functional/crud/query.gql: -------------------------------------------------------------------------------- 1 | { 2 | findProduct(productId: 123) 3 | } 4 | -------------------------------------------------------------------------------- /e2e/functional/crud/schema.gql: -------------------------------------------------------------------------------- 1 | type Query { 2 | findProduct(productId: Long): String @topic(name: "crud-product-topic-test", keyArgument: "productId") 3 | } -------------------------------------------------------------------------------- /e2e/functional/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # This entrypoint goes inside of the tests directory and finds all the e2e subfolder tests (crud, schema, etc.) 4 | # Then for each folder it adds the API_KEY and the HOST and executes the bats command 5 | 6 | cd /tests || exit 7 | 8 | COMMAND="cd '{}' && printf '\n ★ Running tests in: ' && pwd && X_API_KEY=${X_API_KEY} HOST=${HOST} bats *.bats && printf '\n'" 9 | find . -maxdepth 1 -type d \( ! -name . \) -exec bash -c "$COMMAND" \; 10 | -------------------------------------------------------------------------------- /e2e/functional/multi-stream/products.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "key": 123, 4 | "value": { 5 | "productId": 123, 6 | "name": "T-Shirt", 7 | "description": "black", 8 | "price": { 9 | "total": 19.99, 10 | "currency": "DOLLAR" 11 | }, 12 | "metadata": { 13 | "created_at": 1591710185, 14 | "source": "www.foo.com" 15 | } 16 | } 17 | }, 18 | { 19 | "key": 456, 20 | "value": { 21 | "productId": 456, 22 | "name": "Jeans", 23 | "description": "Non-stretch denim", 24 | "price": { 25 | "total": 79.99, 26 | "currency": "EURO" 27 | }, 28 | "metadata": { 29 | "created_at": 1591710185, 30 | "source": "www.bar.de" 31 | } 32 | } 33 | }, 34 | { 35 | "key": 789, 36 | "value": { 37 | "productId": 789, 38 | "name": "Shoes", 39 | "description": "Sneaker", 40 | "price": { 41 | "total": 99.99, 42 | "currency": "DOLLAR" 43 | }, 44 | "metadata": { 45 | "created_at": 1591700185, 46 | "source": "www.foo.com" 47 | } 48 | } 49 | } 50 | ] 51 | -------------------------------------------------------------------------------- /e2e/functional/multi-stream/purchases.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "key": "abc", 4 | "value": { 5 | "purchaseId": "abc", 6 | "productId": 123, 7 | "userId": 1, 8 | "amount": 1, 9 | "price": { 10 | "total": 19.99, 11 | "currency": "DOLLAR" 12 | } 13 | } 14 | }, 15 | { 16 | "key": "def", 17 | "value": { 18 | "purchaseId": "def", 19 | "productId": 123, 20 | "userId": 2, 21 | "amount": 2, 22 | "price": { 23 | "total": 30.00, 24 | "currency": "DOLLAR" 25 | } 26 | } 27 | }, 28 | { 29 | "key": "ghi", 30 | "value": { 31 | "purchaseId": "ghi", 32 | "productId": 456, 33 | "userId": 2, 34 | "amount": 1, 35 | "price": { 36 | "total": 79.99, 37 | "currency": "DOLLAR" 38 | } 39 | } 40 | }, 41 | { 42 | "key": "jkl", 43 | "value": { 44 | "purchaseId": "jkl", 45 | "productId": 789, 46 | "userId": 2, 47 | "amount": 1, 48 | "price": { 49 | "total": 99.99, 50 | "currency": "DOLLAR" 51 | } 52 | } 53 | } 54 | ] 55 | -------------------------------------------------------------------------------- /e2e/functional/multi-stream/query.gql: -------------------------------------------------------------------------------- 1 | { 2 | findPurchase(purchaseId:"abc") { 3 | purchaseId 4 | productId 5 | userId 6 | amount 7 | price { 8 | total 9 | currency 10 | } 11 | product { 12 | productId 13 | name 14 | description 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /e2e/functional/multi-stream/result.json: -------------------------------------------------------------------------------- 1 | { 2 | "purchaseId": "abc", 3 | "productId": 123, 4 | "userId": 1, 5 | "amount": 1, 6 | "price": { 7 | "total": 19.99, 8 | "currency": "DOLLAR" 9 | }, 10 | "product": { 11 | "productId": 123, 12 | "name": "T-Shirt", 13 | "description": "black" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /e2e/functional/multi-stream/schema.gql: -------------------------------------------------------------------------------- 1 | type Query { 2 | findPurchase(purchaseId: String): Purchase @topic(name: "multi-stream-purchase-topic-test", keyArgument: "purchaseId") 3 | } 4 | 5 | type Purchase { 6 | purchaseId: String!, 7 | productId: Int!, 8 | userId: Int!, 9 | product: Product @topic(name: "multi-stream-product-topic-test", keyField: "productId") 10 | amount: Int, 11 | price: Price, 12 | } 13 | 14 | type Product { 15 | productId: Int!, 16 | name: String, 17 | description: String, 18 | price: Price, 19 | metadata: Metadata 20 | } 21 | 22 | type Price { 23 | total: Float, 24 | currency: String 25 | } 26 | 27 | type Metadata { 28 | created_at: Int, 29 | source: String 30 | } 31 | -------------------------------------------------------------------------------- /e2e/functional/range-key/purchases.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "key": "abc", 4 | "value": { 5 | "productId": 123, 6 | "userId": 1, 7 | "amount": 1, 8 | "price": { 9 | "total": 19.99, 10 | "currency": "DOLLAR" 11 | }, 12 | "timestamp": 1 13 | } 14 | }, 15 | { 16 | "key": "def", 17 | "value": { 18 | "productId": 123, 19 | "userId": 2, 20 | "amount": 2, 21 | "price": { 22 | "total": 30.00, 23 | "currency": "DOLLAR" 24 | }, 25 | "timestamp": 2 26 | } 27 | }, 28 | { 29 | "key": "ghi", 30 | "value": { 31 | "productId": 456, 32 | "userId": 2, 33 | "amount": 1, 34 | "price": { 35 | "total": 79.99, 36 | "currency": "DOLLAR" 37 | }, 38 | "timestamp": 3 39 | } 40 | }, 41 | { 42 | "key": "jkl", 43 | "value": { 44 | "productId": 789, 45 | "userId": 2, 46 | "amount": 1, 47 | "price": { 48 | "total": 99.99, 49 | "currency": "DOLLAR" 50 | }, 51 | "timestamp": 4 52 | } 53 | } 54 | ] 55 | -------------------------------------------------------------------------------- /e2e/functional/range-key/query-range.gql: -------------------------------------------------------------------------------- 1 | query { 2 | findUserPurchasesInTime(userId: 2, timestampFrom: 1, timestampTo: 4) { 3 | productId, 4 | price 5 | { 6 | total 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /e2e/functional/range-key/result-range.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "productId": 123, 4 | "price": { 5 | "total": 30.0 6 | } 7 | }, 8 | { 9 | "productId": 456, 10 | "price": { 11 | "total": 79.99 12 | } 13 | } 14 | ] 15 | -------------------------------------------------------------------------------- /e2e/functional/range-key/schema.gql: -------------------------------------------------------------------------------- 1 | type Query { 2 | findUserPurchasesInTime( 3 | userId: Int 4 | timestampFrom: Int 5 | timestampTo: Int 6 | ): [Purchase] @topic(name: "user-purchase-range-key-test", 7 | keyArgument: "userId" 8 | rangeFrom: "timestampFrom", 9 | rangeTo: "timestampTo") 10 | } 11 | 12 | type Purchase { 13 | productId: Int! 14 | userId: Int! 15 | amount: Int 16 | price: Price 17 | timestamp: Int 18 | } 19 | 20 | type Product { 21 | productId: Int! 22 | name: String 23 | description: String 24 | price: Price 25 | } 26 | 27 | type Price { 28 | total: Float 29 | currency: String 30 | } 31 | -------------------------------------------------------------------------------- /e2e/functional/range/query-range.gql: -------------------------------------------------------------------------------- 1 | query { 2 | productPriceInTime(productId:123, timestampFrom:1, timestampTo:4) { 3 | productId, 4 | price 5 | { 6 | total 7 | } 8 | timestamp 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /e2e/functional/range/result-range.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "productId": 123, 4 | "price": { 5 | "total": 14.99 6 | }, 7 | "timestamp": 1 8 | }, 9 | { 10 | "productId": 123, 11 | "price": { 12 | "total": 19.99 13 | }, 14 | "timestamp": 2 15 | }, 16 | { 17 | "productId": 123, 18 | "price": { 19 | "total": 24.99 20 | }, 21 | "timestamp": 3 22 | } 23 | ] 24 | -------------------------------------------------------------------------------- /e2e/functional/range/schema.gql: -------------------------------------------------------------------------------- 1 | type Query { 2 | productPriceInTime( 3 | productId: Int 4 | timestampFrom: Int 5 | timestampTo: Int 6 | ): [Product] @topic(name: "product-price-range-test", 7 | keyArgument: "productId", 8 | rangeFrom: "timestampFrom", 9 | rangeTo: "timestampTo") 10 | } 11 | 12 | type Product { 13 | productId: Int! 14 | name: String 15 | description: String 16 | price: Price 17 | timestamp: Int 18 | } 19 | 20 | type Price { 21 | total: Float 22 | currency: String 23 | } 24 | -------------------------------------------------------------------------------- /e2e/functional/schema/invalid-product.json: -------------------------------------------------------------------------------- 1 | { 2 | "key": 456, 3 | "value": { 4 | "name": "T-Shirt", 5 | "description": "black", 6 | "price": 19.99, 7 | "metadata": { 8 | "created_at": "2020-04-19T18:46:01+0000", 9 | "source": "www.foo.com" 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /e2e/functional/schema/products.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "key": 123, 4 | "value": { 5 | "productId": 123, 6 | "name": "T-Shirt", 7 | "description": "black", 8 | "price": { 9 | "total": 19.99, 10 | "currency": "DOLLAR" 11 | }, 12 | "metadata": { 13 | "created_at": 1591710185, 14 | "source": "www.foo.com" 15 | } 16 | } 17 | }, 18 | { 19 | "key": 456, 20 | "value": { 21 | "productId": 456, 22 | "name": "Jeans", 23 | "description": "Non-stretch denim", 24 | "price": { 25 | "total": 79.99, 26 | "currency": "EURO" 27 | }, 28 | "metadata": { 29 | "created_at": 1591710185, 30 | "source": "www.bar.de" 31 | } 32 | } 33 | }, 34 | { 35 | "key": 789, 36 | "value": { 37 | "productId": 789, 38 | "name": "Shoes", 39 | "description": "Sneaker", 40 | "price": { 41 | "total": 99.99, 42 | "currency": "DOLLAR" 43 | }, 44 | "metadata": { 45 | "created_at": 1591700185, 46 | "source": "www.foo.com" 47 | } 48 | } 49 | } 50 | ] 51 | -------------------------------------------------------------------------------- /e2e/functional/schema/query-all.gql: -------------------------------------------------------------------------------- 1 | { 2 | allProducts { 3 | name 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /e2e/functional/schema/query-list.gql: -------------------------------------------------------------------------------- 1 | { 2 | findProducts(productId: [123, 456]) { 3 | productId 4 | name 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /e2e/functional/schema/query-single.gql: -------------------------------------------------------------------------------- 1 | { 2 | findProduct(productId: 123) { 3 | productId 4 | name 5 | description 6 | price { 7 | currency 8 | total 9 | } 10 | metadata { 11 | created_at 12 | source 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /e2e/functional/schema/result-all.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Shoes" 4 | }, 5 | { 6 | "name": "T-Shirt" 7 | }, 8 | { 9 | "name": "Jeans" 10 | } 11 | ] 12 | -------------------------------------------------------------------------------- /e2e/functional/schema/result-list.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "productId": 123, 4 | "name": "T-Shirt" 5 | }, 6 | { 7 | "productId": 456, 8 | "name": "Jeans" 9 | } 10 | ] 11 | -------------------------------------------------------------------------------- /e2e/functional/schema/result-single.json: -------------------------------------------------------------------------------- 1 | { 2 | "productId": 123, 3 | "name": "T-Shirt", 4 | "description": "black", 5 | "price": { 6 | "currency": "DOLLAR", 7 | "total": 19.99 8 | }, 9 | "metadata": { 10 | "created_at": 1591710185, 11 | "source": "www.foo.com" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /e2e/functional/schema/schema-query-all.gql: -------------------------------------------------------------------------------- 1 | type Query { 2 | findProduct(productId: Int): Product @topic(name: "schema-product-topic-test", keyArgument: "productId") 3 | findProducts(productId: [Int]): [Product] @topic(name: "schema-product-topic-test", keyArgument: "productId") 4 | allProducts: [Product] @topic(name: "schema-product-topic-test") 5 | } 6 | 7 | type Product { 8 | productId: Int!, 9 | name: String, 10 | description: String, 11 | price: Price, 12 | metadata: Metadata 13 | } 14 | 15 | type Price { 16 | total: Float, 17 | currency: String 18 | } 19 | 20 | type Metadata { 21 | created_at: Int, 22 | source: String 23 | } 24 | -------------------------------------------------------------------------------- /e2e/functional/schema/schema-query-single.gql: -------------------------------------------------------------------------------- 1 | type Query { 2 | findProduct(productId: Int): Product @topic(name: "schema-product-topic-test", keyArgument: "productId") 3 | } 4 | 5 | type Product { 6 | productId: Int!, 7 | name: String, 8 | description: String, 9 | price: Price, 10 | metadata: Metadata 11 | } 12 | 13 | type Price { 14 | total: Float, 15 | currency: String 16 | } 17 | 18 | type Metadata { 19 | created_at: Int, 20 | source: String 21 | } 22 | -------------------------------------------------------------------------------- /e2e/functional/subscription/purchases.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "key": "gssqiw", 4 | "value": { 5 | "purchaseId": "gssqiw", 6 | "productId": "8cexbu", 7 | "userId": "tc3ks5", 8 | "amount": 14, 9 | "price": { 10 | "total": 649.18, 11 | "currency": "CHE" 12 | } 13 | } 14 | }, 15 | { 16 | "key": "4v5ynx", 17 | "value": { 18 | "purchaseId": "4v5ynx", 19 | "productId": "bpi4r4", 20 | "userId": "6w4fxc", 21 | "amount": 11, 22 | "price": { 23 | "total": 754.3, 24 | "currency": "DOLLAR" 25 | } 26 | } 27 | }, 28 | { 29 | "key": "g46qiw", 30 | "value": { 31 | "purchaseId": "g46qiw", 32 | "productId": "bpi4r4", 33 | "userId": "vro1gi", 34 | "amount": 2, 35 | "price": { 36 | "total": 111.48, 37 | "currency": "EURO" 38 | } 39 | } 40 | } 41 | ] 42 | -------------------------------------------------------------------------------- /e2e/functional/subscription/query.gql: -------------------------------------------------------------------------------- 1 | { 2 | allProducts { 3 | name 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /e2e/functional/subscription/schema.gql: -------------------------------------------------------------------------------- 1 | type Query { 2 | allPurchases: [Purchase] @topic(name: "subscription-purchase-topic-test") 3 | allProducts: [Product] @topic(name: "subscription-product-topic-test") 4 | } 5 | 6 | type Subscription { 7 | lastPurchases: Purchase @topic(name: "subscription-purchase-topic-test") 8 | } 9 | 10 | type Purchase { 11 | purchaseId: String!, 12 | productId: Int!, 13 | product: Product @topic(name: "subscription-product-topic-test", keyField: "productId") 14 | userId: String!, 15 | user: User @topic(name: "subscription-user-topic-test", keyField: "userId"), 16 | amount: Int, 17 | price: Price, 18 | } 19 | 20 | type Product { 21 | productId: Int!, 22 | name: String, 23 | description: String, 24 | price: Price, 25 | metadata: Metadata 26 | } 27 | 28 | type User { 29 | userId: String!, 30 | name: String!, 31 | address: String! 32 | } 33 | 34 | type Price { 35 | total: Float, 36 | currency: String 37 | } 38 | 39 | type Metadata { 40 | created_at: Int, 41 | source: String 42 | } 43 | -------------------------------------------------------------------------------- /e2e/functional/subscription/subscription.gql: -------------------------------------------------------------------------------- 1 | subscription { 2 | lastPurchases { 3 | product { 4 | name 5 | description 6 | } 7 | amount 8 | user { 9 | name 10 | address 11 | } 12 | price { 13 | total 14 | currency 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /gateway/src/main/java/com/bakdata/quick/gateway/DataFetcherSpecification.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.gateway; 18 | 19 | import graphql.schema.DataFetcher; 20 | import graphql.schema.FieldCoordinates; 21 | import lombok.Value; 22 | 23 | /** 24 | * Data class for specifying and data fetcher and where it should be applied. 25 | * 26 | * @see graphql.schema.GraphQLCodeRegistry.Builder#dataFetcher(FieldCoordinates, DataFetcher) 27 | */ 28 | @Value(staticConstructor = "of") 29 | public class DataFetcherSpecification { 30 | FieldCoordinates coordinates; 31 | DataFetcher dataFetcher; 32 | } 33 | -------------------------------------------------------------------------------- /gateway/src/main/java/com/bakdata/quick/gateway/GatewayApplication.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.gateway; 18 | 19 | import io.micronaut.runtime.Micronaut; 20 | 21 | /** 22 | * Main gateway application. 23 | */ 24 | public class GatewayApplication { 25 | public static void main(final String[] args) { 26 | Micronaut.run(GatewayApplication.class, args); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /gateway/src/main/java/com/bakdata/quick/gateway/GatewayConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.gateway; 18 | 19 | import io.micronaut.context.annotation.ConfigurationInject; 20 | import io.micronaut.context.annotation.ConfigurationProperties; 21 | import lombok.Getter; 22 | 23 | /** 24 | * Configuration specifying location of schema path. 25 | */ 26 | @Getter 27 | @ConfigurationProperties("quick.schema") 28 | public class GatewayConfig { 29 | private final String path; 30 | 31 | @ConfigurationInject 32 | public GatewayConfig(final String path) { 33 | this.path = path; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /gateway/src/main/java/com/bakdata/quick/gateway/custom/type/QuickGraphQLType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.gateway.custom.type; 18 | 19 | import graphql.language.SDLDefinition; 20 | 21 | /** 22 | * Builder for custom GraphQL types supported by quick. 23 | */ 24 | public interface QuickGraphQLType> { 25 | T getDefinition(); 26 | } 27 | -------------------------------------------------------------------------------- /gateway/src/main/java/com/bakdata/quick/gateway/custom/type/RestDirectiveMethod.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.gateway.custom.type; 18 | 19 | /** 20 | * Supported HTTP methods by the rest directive. 21 | */ 22 | public enum RestDirectiveMethod { 23 | GET, POST 24 | } 25 | -------------------------------------------------------------------------------- /gateway/src/main/java/com/bakdata/quick/gateway/directives/QuickDirectiveException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.gateway.directives; 18 | 19 | /** 20 | * Exception for invalid directive usage. 21 | */ 22 | public class QuickDirectiveException extends IllegalArgumentException { 23 | public QuickDirectiveException(final String msg) { 24 | super(msg); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /gateway/src/main/java/com/bakdata/quick/gateway/directives/QuickDirectiveWiring.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.gateway.directives; 18 | 19 | import graphql.language.DirectiveDefinition; 20 | import graphql.schema.idl.SchemaDirectiveWiring; 21 | 22 | /** 23 | * SchemaDirectiveWiring used within {@link com.bakdata.quick.gateway.GraphQLSchemaGenerator}. 24 | */ 25 | public interface QuickDirectiveWiring extends SchemaDirectiveWiring { 26 | /** 27 | * Returns the directive definition implemented in this wiring. 28 | */ 29 | DirectiveDefinition getDefinition(); 30 | 31 | /** 32 | * Returns name of the directive. 33 | * 34 | *

35 | * Example: {@code @skip(if: Boolean!)} --> Name in this case would be 'skip' 36 | */ 37 | String getName(); 38 | } 39 | -------------------------------------------------------------------------------- /gateway/src/main/java/com/bakdata/quick/gateway/directives/rest/RestParameter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.gateway.directives.rest; 18 | 19 | import lombok.Value; 20 | 21 | /** 22 | * Parameter used in a restful url. 23 | */ 24 | @Value 25 | public class RestParameter { 26 | String name; 27 | boolean isPrimitive; 28 | } 29 | -------------------------------------------------------------------------------- /gateway/src/main/java/com/bakdata/quick/gateway/directives/topic/rule/TopicDirectiveRule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.gateway.directives.topic.rule; 18 | 19 | 20 | /** 21 | * Marker interface for topic directive rules. 22 | */ 23 | public interface TopicDirectiveRule { 24 | } 25 | -------------------------------------------------------------------------------- /gateway/src/main/java/com/bakdata/quick/gateway/directives/topic/rule/TopicDirectiveRules.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.gateway.directives.topic.rule; 18 | 19 | import com.bakdata.quick.gateway.directives.topic.TopicDirectiveContext; 20 | 21 | /** 22 | * Rules that can be applied to a {@link com.bakdata.quick.gateway.directives.topic.TopicDirective}. 23 | */ 24 | public interface TopicDirectiveRules { 25 | 26 | /** 27 | * Applies rules to given context. 28 | */ 29 | void apply(final TopicDirectiveContext context); 30 | } 31 | -------------------------------------------------------------------------------- /gateway/src/main/java/com/bakdata/quick/gateway/directives/topic/rule/validation/ValidationRule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.gateway.directives.topic.rule.validation; 18 | 19 | import com.bakdata.quick.gateway.directives.topic.TopicDirectiveContext; 20 | import com.bakdata.quick.gateway.directives.topic.rule.TopicDirectiveRule; 21 | import java.util.Optional; 22 | 23 | /** 24 | * Interface for rules validating the proper use of a {@link com.bakdata.quick.gateway.directives.topic.TopicDirective}. 25 | */ 26 | public interface ValidationRule extends TopicDirectiveRule { 27 | /** 28 | * Validates if the topic infringes a rule. 29 | * 30 | * @param context the topic directive context 31 | * @return empty if no infringement, error message otherwise 32 | */ 33 | Optional validate(TopicDirectiveContext context); 34 | } 35 | -------------------------------------------------------------------------------- /gateway/src/main/java/com/bakdata/quick/gateway/fetcher/ClientSupplier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.gateway.fetcher; 18 | 19 | import com.bakdata.quick.common.type.QuickTopicData; 20 | import com.bakdata.quick.common.util.Lazy; 21 | import org.apache.kafka.common.serialization.Serde; 22 | 23 | /** 24 | * Supplier for creating a new data fetcher client for a topic. 25 | */ 26 | public interface ClientSupplier { 27 | DataFetcherClient createClient(final String topic, final Lazy> quickTopicData); 28 | 29 | DataFetcherClient createClient(final String topic, final Serde keySerde, 30 | final Lazy> quickTopicData); 31 | } 32 | -------------------------------------------------------------------------------- /gateway/src/main/java/com/bakdata/quick/gateway/fetcher/subscription/SubscriptionProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.gateway.fetcher.subscription; 18 | 19 | import graphql.schema.DataFetchingEnvironment; 20 | import org.apache.kafka.clients.consumer.ConsumerRecord; 21 | import reactor.core.publisher.Flux; 22 | 23 | /** 24 | * Provides streams of elements used in GraphQL subscriptions. 25 | * 26 | * @param key type 27 | * @param value type 28 | */ 29 | public interface SubscriptionProvider { 30 | Flux> getElementStream(final DataFetchingEnvironment environment); 31 | } 32 | -------------------------------------------------------------------------------- /gateway/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | micronaut: 2 | application: 3 | name: gateway 4 | server: 5 | cors: 6 | enabled: true 7 | security: 8 | enabled: true 9 | # Secure built-in endpoints graphql 10 | # Note: doing it here, we don't have to create our own subclasses 11 | intercept-url-map: 12 | - pattern: /graphql 13 | access: 14 | - isAuthenticated() 15 | # graphql-ws MUST be unsecured! JavaScript APIs do not allow passing custom headers during the handshake HTTP 16 | # request. The api-key gets checked during the first message of the GraphQL protocol. 17 | - pattern: /graphql-ws 18 | access: 19 | - isAnonymous() 20 | 21 | metrics: 22 | enabled: true 23 | 24 | jackson: 25 | always-serialize-errors-as-list: false 26 | 27 | quick: 28 | schema: 29 | path: "/app/schema.graphql" 30 | 31 | endpoints: 32 | # Enable all default endpoint but on custom port so they are not publicly accessible 33 | all: 34 | port: 8081 35 | sensitive: false 36 | enabled: true 37 | 38 | # https://micronaut-projects.github.io/micronaut-kubernetes/2.2.0/guide/index.html 39 | kubernetes: 40 | client: 41 | discovery: 42 | enabled: false 43 | 44 | graphql: 45 | graphql-ws: 46 | enabled: true 47 | keep-alive-enabled: true 48 | keep-alive-interval: 15s 49 | -------------------------------------------------------------------------------- /gateway/src/test/java/com/bakdata/quick/gateway/GatewayConfigTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.gateway; 18 | 19 | import static org.assertj.core.api.Assertions.assertThat; 20 | 21 | import com.bakdata.quick.common.ConfigUtils; 22 | import java.util.Map; 23 | import org.junit.jupiter.api.Test; 24 | 25 | class GatewayConfigTest { 26 | @Test 27 | void createConfig() { 28 | final Map properties = Map.of("quick.schema.path", "/test/path/schema.graphql"); 29 | final GatewayConfig config = ConfigUtils.createWithProperties(properties, GatewayConfig.class); 30 | assertThat(config.getPath()).isEqualTo("/test/path/schema.graphql"); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /gateway/src/test/java/com/bakdata/quick/gateway/fetcher/TestModels.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.gateway.fetcher; 18 | 19 | import java.util.List; 20 | import lombok.Builder; 21 | import lombok.Value; 22 | 23 | public class TestModels { 24 | @Value 25 | @Builder 26 | public static class PurchaseList { 27 | String purchaseId; 28 | List productIds; 29 | } 30 | 31 | @Value 32 | @Builder 33 | public static class Purchase { 34 | String purchaseId; 35 | T productId; 36 | int amount; 37 | } 38 | 39 | @Value 40 | @Builder 41 | public static class Product { 42 | T productId; 43 | String name; 44 | int ratings; 45 | } 46 | } 47 | 48 | -------------------------------------------------------------------------------- /gateway/src/test/resources/application-test.yaml: -------------------------------------------------------------------------------- 1 | micronaut: 2 | security: 3 | enabled: false 4 | 5 | metrics: 6 | enabled: false 7 | 8 | endpoints: 9 | # Enable all default endpoint but on custom port so they are not publicly accessible 10 | all: 11 | enabled: false 12 | 13 | quick: 14 | kafka: 15 | bootstrap-server: dummy:9092 16 | schema-registry-url: http://test:8081 17 | definition: 18 | path: "definition/definition.yaml" 19 | apikey: 20 | test_key 21 | mirror: 22 | prefix: "" # prefix must be empty, as the host is simply 'localhost' and not for example 'quick-mirror-localhost' 23 | -------------------------------------------------------------------------------- /gateway/src/test/resources/initializer/schema.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | getURL(hash: String): UrlInfo 3 | } 4 | 5 | type UrlInfo { 6 | url: String @topic(name: "url-topic", keyArgument: "hash") 7 | } 8 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/contract.graphql: -------------------------------------------------------------------------------- 1 | type Contract { 2 | _id: String! 3 | policyHolderId: [PersonGrainValue] 4 | insuredPersonId: [PersonGrainValue] 5 | term: [GrainValue] 6 | value: [GrainValue] 7 | } 8 | 9 | type PersonGrainValue { 10 | _in_utc: String! 11 | _v: String! 12 | policerHolder: Person 13 | } 14 | 15 | type GrainValue { 16 | _in_utc: String! 17 | _v: String! 18 | _c: Float 19 | _in: Int 20 | _ttl: String 21 | _origin: String 22 | } -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldConvertComplexSubscription.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | getURL(id: ID): String 3 | } 4 | 5 | type Subscription { 6 | subscribeProfile: Profile 7 | } 8 | 9 | type Profile { 10 | buyerProfile: String @topic(name: "buyer-profile", keyArgument: "id") 11 | customerProfile: String @topic(name: "customer-profile", keyArgument: "id") 12 | } 13 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldConvertDefinitionWithSingleFieldAndObjectName.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | getURL(hash: String): UrlInfo 3 | } 4 | 5 | type UrlInfo { 6 | url: String @topic(name: "url-topic", keyArgument: "hash") 7 | } -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldConvertIfMultipleValues.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | findContract(id: ID): Contract @topic(name: "contract-topic", keyArgument: "id") 3 | } 4 | 5 | type Contract { 6 | _id: String! 7 | policyHolderId: [PersonGrainValue] 8 | insuredPersonId: [PersonGrainValue] 9 | term: [GrainValue] 10 | value: [GrainValue] 11 | } 12 | 13 | type PersonGrainValue { 14 | _in_utc: String! 15 | _v: String! 16 | policyHolder: Person @topic(name: "person-topic", keyField: "_v") 17 | } 18 | 19 | type GrainValue { 20 | _in_utc: String! 21 | _v: String! 22 | _c: Float 23 | _in: Int 24 | _ttl: String 25 | _origin: String 26 | } 27 | 28 | type Person { 29 | _id: String! 30 | firstname: [GrainValue] 31 | lastname: [GrainValue] 32 | } -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldConvertListQueryWithSingleFieldAndModification.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | findPurchases: [Purchase] @topic(name: "purchase-topic") 3 | } 4 | 5 | type Purchase { 6 | purchaseId: ID!, 7 | productId: ID!, 8 | userId: ID!, 9 | amount: Int, 10 | price: Price, 11 | infos: [String] 12 | product: Product @topic(name: "product-topic", keyField: "productId") 13 | } 14 | 15 | type Product { 16 | productId: ID!, 17 | name: String, 18 | description: String, 19 | price: Price, 20 | metadata: Metadata 21 | } 22 | 23 | type Price { 24 | total: Float, 25 | currency: String 26 | } 27 | 28 | type Metadata { 29 | created_at: Int, 30 | source: String 31 | } -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldConvertMutation.graphql: -------------------------------------------------------------------------------- 1 | input ProductInput { 2 | id: ID 3 | name: String 4 | } 5 | 6 | type Product { 7 | id: ID 8 | name: String 9 | } 10 | 11 | type Query { 12 | getProduct(id: ID): Product @topic(name: "product-topic", keyArgument: "id") 13 | } 14 | 15 | type Mutation { 16 | setProduct(id: ID, product: ProductInput): Product @topic(name: "product-topic") 17 | } 18 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldConvertQueryAllWithComplexType.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | fetchCountOfToken(token: String): TinyUrlCount 3 | fetchAll: [TinyUrl] @topic(name: "tiny-url") 4 | } 5 | 6 | type TinyUrl { 7 | token: String! 8 | url: String! 9 | } 10 | 11 | type TinyUrlCount { 12 | url: String @topic(name: "tiny-url", keyArgument: "token") 13 | count: Long @topic(name: "count-fetch", keyArgument: "token") 14 | } 15 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldConvertQueryAllWithPrimitiveType.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | getURL: [String] @topic(name: "url-topic") 3 | } -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldConvertQueryWithListArgument.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | findPurchase(purchaseId: [ID]): [Purchase] @topic(name: "purchase-topic", keyArgument: "purchaseId") 3 | } 4 | 5 | type Purchase { 6 | purchaseId: ID!, 7 | productId: ID!, 8 | userId: ID!, 9 | amount: Int, 10 | price: Price, 11 | infos: [String] 12 | } 13 | 14 | type Price { 15 | total: Float, 16 | currency: String 17 | } 18 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldConvertQueryWithMultipleFields.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | getProduct(productId: ID): ProductInfo 3 | } 4 | 5 | type ProductInfo { 6 | product: Product @topic(name: "product-topic", keyArgument: "productId") 7 | url: String @topic(name: "url-topic", keyArgument: "productId") 8 | } 9 | 10 | type Product { 11 | productId: ID!, 12 | name: String, 13 | description: String, 14 | price: Price, 15 | metadata: Metadata 16 | } 17 | 18 | type Price { 19 | total: Float, 20 | currency: String 21 | } 22 | 23 | type Metadata { 24 | created_at: Int, 25 | source: String 26 | } -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldConvertQueryWithPrimitiveType.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | getURL(id: ID): String @topic(name: "url-topic", keyArgument: "id") 3 | } -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldConvertQueryWithRange.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | userRequests( 3 | userId: Int 4 | timestampFrom: Int 5 | timestampTo: Int 6 | ): [UserRequests] 7 | @topic(name: "user-request-range", 8 | keyArgument: "userId", 9 | rangeFrom: "timestampFrom", 10 | rangeTo: "timestampTo") 11 | } 12 | 13 | type UserRequests { 14 | userId: Int 15 | serviceId: Int 16 | timestamp: Int 17 | requests: Int 18 | success: Int 19 | } 20 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldConvertQueryWithRangeOnField.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | product(key: Int, timestampFrom: Int, timestampTo: Int): ProductInfo 3 | } 4 | 5 | type ProductInfo { 6 | info: [Info] @topic(name: "info-topic", keyArgument: "key", rangeFrom: "timestampFrom", rangeTo: "timestampTo") 7 | } 8 | 9 | type Info { 10 | key: Int 11 | timestamp: Int 12 | } 13 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldConvertQueryWithSingleField.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | findPurchase(purchaseId: ID): Purchase @topic(name: "purchase-topic", keyArgument: "purchaseId") 3 | } 4 | 5 | type Purchase { 6 | purchaseId: ID!, 7 | productId: ID!, 8 | userId: ID!, 9 | amount: Int, 10 | price: Price, 11 | infos: [String] 12 | } 13 | 14 | type Price { 15 | total: Float, 16 | currency: String 17 | } 18 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldConvertQueryWithSingleFieldAndModification.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | findPurchase(purchaseId: ID): Purchase @topic(name: "purchase-topic", keyArgument: "purchaseId") 3 | } 4 | 5 | type Purchase { 6 | purchaseId: ID!, 7 | productId: ID!, 8 | userId: ID!, 9 | amount: Int, 10 | price: Price, 11 | infos: [String] 12 | product: Product @topic(name: "product-topic" keyField: "productId") 13 | } 14 | 15 | type Product { 16 | productId: ID!, 17 | name: String, 18 | description: String, 19 | price: Price, 20 | metadata: Metadata 21 | } 22 | 23 | type Price { 24 | total: Float, 25 | currency: String 26 | } 27 | 28 | type Metadata { 29 | created_at: Int, 30 | source: String 31 | } -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldConvertSubscription.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | getURL(id: ID): String 3 | } 4 | 5 | type Subscription { 6 | getURL(id: ID): String @topic(name: "url-topic") 7 | } -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldNotConvertIfKeyArgAndInputNameDifferentInNonQueryType.graphql: -------------------------------------------------------------------------------- 1 | # Invalid Schema 2 | type Query { 3 | getProduct(productId: ID): ProductInfo 4 | } 5 | 6 | type ProductInfo { 7 | product: Product @topic(name: "product-topic", keyArgument: "id") 8 | url: String @topic(name: "url-topic", keyArgument: "productId") 9 | } 10 | 11 | type Product { 12 | productId: ID!, 13 | name: String, 14 | description: String 15 | } 16 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldNotConvertIfKeyArgAndInputNameDifferentInQueryType.graphql: -------------------------------------------------------------------------------- 1 | # Invalid Schema 2 | type Query { 3 | getClick(id: ID): Long @topic(name: "click-topic", keyArgument: "aDifferentKeyArgument") 4 | } 5 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldNotConvertIfMissingKeyInfoInBasicType.graphql: -------------------------------------------------------------------------------- 1 | # Invalid Schema 2 | type Query { 3 | getProduct(productId: ID): ProductInfo 4 | } 5 | 6 | type ProductInfo { 7 | product: Product @topic(name: "product-topic") 8 | url: String @topic(name: "url-topic", keyArgument: "productId") 9 | } 10 | 11 | type Product { 12 | productId: ID!, 13 | name: String, 14 | description: String 15 | } 16 | 17 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldNotConvertIfMissingKeyInfoInQueryType.graphql: -------------------------------------------------------------------------------- 1 | # Invalid Schema 2 | type Product { 3 | id: ID! 4 | name: String! 5 | } 6 | 7 | type Query { 8 | getProduct(id: ID): Product @topic(name: "product-topic") 9 | } 10 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldNotConvertIfMutationDoesNotHaveTwoArgs.graphql: -------------------------------------------------------------------------------- 1 | # Invalid Schema 2 | type Mutation { 3 | setClick(clickCount: Long): Long @topic(name: "click-topic") 4 | } 5 | 6 | type Query { 7 | dummy: Boolean 8 | } 9 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldNotConvertIfRangeToArgumentIsMissing.graphql: -------------------------------------------------------------------------------- 1 | # Invalid Schema 2 | type Query { 3 | userRequests( 4 | userId: Int 5 | timestampFrom: Int 6 | timestampTo: Int 7 | ): [UserRequests] 8 | @topic(name: "user-request-range", 9 | keyArgument: "userId", 10 | rangeFrom: "timestampFrom") 11 | } 12 | 13 | type UserRequests { 14 | userId: Int 15 | serviceId: Int 16 | timestamp: Int 17 | requests: Int 18 | success: Int 19 | } 20 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldNotConvertKeyFieldWithWrongFieldType.graphql: -------------------------------------------------------------------------------- 1 | # Invalid Schema 2 | type Query { 3 | findPurchase(purchaseId: ID): Purchase @topic(name: "purchase-topic", keyArgument: "purchaseId") 4 | } 5 | 6 | type Purchase { 7 | purchaseId: ID!, 8 | productId: [Long]!, 9 | product: Product @topic(name: "product-topic" keyField: "productId") 10 | } 11 | 12 | type Product { 13 | productId: ID!, 14 | name: String, 15 | description: String 16 | } 17 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldNotConvertKeyFieldWithWrongKeyFieldName.graphql: -------------------------------------------------------------------------------- 1 | # Invalid Schema 2 | type Query { 3 | findPurchase(purchaseId: ID): Purchase @topic(name: "purchase-topic", keyArgument: "purchaseId") 4 | } 5 | 6 | type Purchase { 7 | purchaseId: ID!, 8 | productId: Long!, 9 | product: Product @topic(name: "product-topic" keyField: "notExistingField") 10 | } 11 | 12 | type Product { 13 | productId: ID!, 14 | name: String, 15 | description: String 16 | } 17 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldNotCovertIfKeyArgumentInRangeQueryIsMissing.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | userRequests( 3 | userId: Int 4 | timestampFrom: Int 5 | timestampTo: Int 6 | ): [UserRequests] 7 | @topic(name: "user-request-range", 8 | rangeFrom: "timestampFrom", 9 | rangeTo: "timestampTo") 10 | } 11 | 12 | type UserRequests { 13 | userId: Int 14 | serviceId: Int 15 | timestamp: Int 16 | requests: Int 17 | success: Int 18 | } 19 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldNotCovertIfRangeFromArgumentIsMissing.graphql: -------------------------------------------------------------------------------- 1 | # Invalid Schema 2 | type Query { 3 | userRequests( 4 | userId: Int 5 | timestampFrom: Int 6 | timestampTo: Int 7 | ): [UserRequests] 8 | @topic(name: "user-request-range", 9 | keyArgument: "userId", 10 | rangeTo: "timestampTo") 11 | } 12 | 13 | type UserRequests { 14 | userId: Int 15 | serviceId: Int 16 | timestamp: Int 17 | requests: Int 18 | success: Int 19 | } 20 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldNotCovertIfRangeIsDefinedOnField.graphql: -------------------------------------------------------------------------------- 1 | # Invalid Schema 2 | type Query { 3 | product(key: Int, timestampFrom: Int, timestampTo: Int): ProductInfo! 4 | } 5 | 6 | type ProductInfo { 7 | info: [Info!] @topic(name: "info-topic", keyArgument: "key", rangeFrom: "timestampFrom", rangeTo: "timestampTo") 8 | } 9 | 10 | type Info { 11 | key: Int 12 | timestamp: Int! 13 | } 14 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/conversion/shouldNotCovertIfReturnTypeOfRangeQueryIsNotList.graphql: -------------------------------------------------------------------------------- 1 | # Invalid Schema 2 | type Query { 3 | userRequests( 4 | userId: Int 5 | timestampFrom: Int 6 | timestampTo: Int 7 | ): UserRequests 8 | @topic(name: "user-request-range", 9 | keyArgument: "userId", 10 | rangeFrom: "timestampFrom", 11 | rangeTo: "timestampTo") 12 | } 13 | 14 | type UserRequests { 15 | userId: Int 16 | serviceId: Int 17 | timestamp: Int 18 | requests: Int 19 | success: Int 20 | } 21 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/directives/rest/directive/shouldParseBodyParameter.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | recommendation(inputs: Input): Recommendation @rest(url: "http://localhost:8081", 3 | bodyParameter: "inputs", httpMethod: POST) 4 | } 5 | 6 | input Input { 7 | inputs: [Int] 8 | } 9 | 10 | type Recommendation { 11 | ids: [ID] 12 | } 13 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/directives/rest/directive/shouldParseHttpMethod.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | recommendation(userId: ID!, limit: Int): Recommendation @rest(url: "http://localhost:8081", pathParameter: ["userId"], 3 | queryParameter: ["limit"], httpMethod: POST) 4 | } 5 | 6 | type Recommendation { 7 | ids: [ID] 8 | } 9 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/directives/rest/directive/shouldParseMultipleDirectives.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | recommendation(userId: ID!, limit: Int): Recommendation @rest(url: "http://localhost:8081") 3 | metrics(userId: ID!, test: Int): Metric @rest(url: "http://localhost:8082") 4 | } 5 | 6 | type Metric { 7 | count: Int 8 | } 9 | 10 | type Recommendation { 11 | ids: [ID] 12 | } 13 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/directives/rest/directive/shouldParseNonOptionalArguments.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | recommendation(userId: ID!, limit: Int): Recommendation @rest(url: "http://localhost:8081") 3 | } 4 | 5 | type Recommendation { 6 | ids: [ID] 7 | } 8 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/directives/rest/directive/shouldParseOptionalArguments.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | recommendation(userId: ID!, limit: Int): Recommendation @rest(url: "http://localhost:8081", pathParameter: ["userId"], 3 | queryParameter: ["limit"]) 4 | } 5 | 6 | type Recommendation { 7 | ids: [ID] 8 | } 9 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/directives/rest/wiring/shouldDefaultToGetMethod.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | recommendation(userId: ID!, recommendationType: String!): Recommendation @rest(url: "http://localhost:8081") 3 | } 4 | 5 | type Recommendation { 6 | ids: [ID] 7 | } 8 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/directives/rest/wiring/shouldNotAllowSameArgumentTwice.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | recommendation(userId: ID!, recommendationType: String!, limit: Int, walks: Int): Recommendation @rest(url: "http://localhost:8081" 3 | pathParameter: ["userId", "limit"], queryParameter: ["recommendationType", "walks", "limit"]) 4 | } 5 | 6 | type Recommendation { 7 | ids: [ID] 8 | } 9 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/directives/rest/wiring/shouldNotParseNonExistingPathArgument.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | recommendation(userId: ID!, recommendationType: String!, limit: Int, walks: Int): Recommendation @rest(url: "http://localhost:8081" 3 | pathParameter: ["userId", "nonExisting"], queryParameter: ["recommendationType", "walks"]) 4 | } 5 | 6 | type Recommendation { 7 | ids: [ID] 8 | } 9 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/directives/rest/wiring/shouldNotParseNonExistingQueryArgument.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | recommendation(userId: ID!, recommendationType: String!, limit: Int, walks: Int): Recommendation @rest(url: "http://localhost:8081" 3 | pathParameter: ["userId", "limit"], queryParameter: ["nonExisting", "walks"]) 4 | } 5 | 6 | type Recommendation { 7 | ids: [ID] 8 | } 9 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/directives/rest/wiring/shouldParseBodyParameter.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | recommendation(inputs: Input): Recommendation @rest(url: "http://localhost:8081", 3 | bodyParameter: "inputs", httpMethod: POST) 4 | } 5 | 6 | input Input { 7 | inputs: [Int] 8 | } 9 | 10 | type Recommendation { 11 | ids: [ID] 12 | } 13 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/directives/rest/wiring/shouldParseEnum.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | recommendation(userId: ID!, recommendationType: Type!): Recommendation @rest(url: "http://localhost:8081") 3 | } 4 | 5 | enum Type { 6 | ALBUM 7 | ARTIST 8 | TRACK 9 | } 10 | type Recommendation { 11 | ids: [ID] 12 | } 13 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/directives/rest/wiring/shouldParseHttpMethod.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | recommendation(userId: ID!, limit: Int): Recommendation @rest(url: "http://localhost:8081", pathParameter: ["userId"], 3 | queryParameter: ["limit"], httpMethod: POST) 4 | } 5 | 6 | type Recommendation { 7 | ids: [ID] 8 | } 9 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/directives/rest/wiring/shouldParseMultipleArguments.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | recommendation(userId: ID!, recommendationType: String!, limit: Int, walks: Int): Recommendation @rest(url: "http://localhost:8081") 3 | } 4 | 5 | type Recommendation { 6 | ids: [ID] 7 | } 8 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/directives/rest/wiring/shouldParseOverriddenArguments.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | recommendation(userId: ID!, recommendationType: String!, limit: Int, walks: Int): Recommendation @rest(url: "http://localhost:8081" 3 | pathParameter: ["userId", "limit"], queryParameter: ["recommendationType", "walks"]) 4 | } 5 | 6 | type Recommendation { 7 | ids: [ID] 8 | } 9 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/directives/rest/wiring/shouldParsePathArguments.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | recommendation(userId: ID!, recommendationType: String!): Recommendation @rest(url: "http://localhost:8081") 3 | } 4 | 5 | type Recommendation { 6 | ids: [ID] 7 | } 8 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/directives/rest/wiring/shouldParseQueryArguments.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | recommendation(userId: ID, limit: Int): Recommendation @rest(url: "http://localhost:8081") 3 | } 4 | 5 | type Recommendation { 6 | ids: [ID] 7 | } 8 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/directives/rest/wiring/shouldParseSpecifiedPathArguments.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | recommendation(userId: ID!, recommendationType: String!, limit: Int, walks: Int): Recommendation @rest(url: "http://localhost:8081", 3 | pathParameter: ["recommendationType"]) 4 | } 5 | 6 | type Recommendation { 7 | ids: [ID] 8 | } 9 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/directives/rest/wiring/shouldRespectOrderOfOverriddenArguments.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | recommendation(userId: ID!, recommendationType: String!, limit: Int, walks: Int): Recommendation @rest(url: "http://localhost:8081" 3 | pathParameter: ["limit", "userId"], queryParameter: ["recommendationType", "walks"]) 4 | } 5 | 6 | type Recommendation { 7 | ids: [ID] 8 | } 9 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/execution/shouldExecuteDefinitionWithSingleFieldAndObjectName.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | getURL(hash: String): UrlInfo 3 | } 4 | 5 | type UrlInfo { 6 | url: String @topic(name: "url-topic", keyArgument: "hash") 7 | } -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/execution/shouldExecuteDefinitionWithSingleFieldAndObjectNameQuery.graphql: -------------------------------------------------------------------------------- 1 | { 2 | getURL(hash: "test") { 3 | url 4 | } 5 | } -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/execution/shouldExecuteListQueryWithSingleFieldAndModification.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | findPurchases: [Purchase] @topic(name: "purchase-topic") 3 | } 4 | 5 | type Purchase { 6 | purchaseId: ID!, 7 | productId: Long!, 8 | userId: ID!, 9 | amount: Int, 10 | price: Price, 11 | infos: [String] 12 | product: Product @topic(name: "product-topic", keyField: "productId") 13 | } 14 | 15 | type Product { 16 | productId: Long!, 17 | name: String, 18 | description: String, 19 | price: Price, 20 | metadata: Metadata 21 | } 22 | 23 | type Price { 24 | total: Float, 25 | currency: String 26 | } 27 | 28 | type Metadata { 29 | created_at: Int, 30 | source: String 31 | } -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/execution/shouldExecuteListQueryWithSingleFieldAndModificationQuery.graphql: -------------------------------------------------------------------------------- 1 | { 2 | findPurchases { 3 | purchaseId 4 | productId 5 | product { 6 | productId 7 | name 8 | price { 9 | total 10 | } 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/execution/shouldExecuteQueryAllWithPrimitiveType.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | getURL: [String] @topic(name: "url-topic") 3 | } -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/execution/shouldExecuteQueryAllWithPrimitiveTypeQuery.graphql: -------------------------------------------------------------------------------- 1 | { 2 | getURL 3 | } -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/execution/shouldExecuteQueryWithListArgumentTypeId.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | getURL(id: [ID]): [String] @topic(name: "url-topic", keyArgument: "id") 3 | } 4 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/execution/shouldExecuteQueryWithListArgumentTypeIdQuery.graphql: -------------------------------------------------------------------------------- 1 | { 2 | getURL(id: ["1", "2", "3"]) 3 | } 4 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/execution/shouldExecuteQueryWithListArgumentTypeInt.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | getURL(id: [Int]): [String] @topic(name: "url-topic", keyArgument: "id") 3 | } 4 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/execution/shouldExecuteQueryWithListArgumentTypeIntQuery.graphql: -------------------------------------------------------------------------------- 1 | { 2 | getURL(id: [1, 2, 3]) 3 | } 4 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/execution/shouldExecuteQueryWithPrimitiveType.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | getURL(id: ID): String @topic(name: "url-topic", keyArgument: "id") 3 | } -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/execution/shouldExecuteQueryWithPrimitiveTypeQuery.graphql: -------------------------------------------------------------------------------- 1 | { 2 | getURL(id: "test") 3 | } 4 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/execution/shouldExecuteQueryWithSingleField.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | findPurchase(purchaseId: ID): Purchase @topic(name: "purchase-topic", keyArgument: "purchaseId") 3 | } 4 | 5 | type Purchase { 6 | purchaseId: ID!, 7 | productId: Long!, 8 | userId: ID!, 9 | amount: Int, 10 | price: Price, 11 | infos: [String] 12 | } 13 | 14 | type Price { 15 | total: Float, 16 | currency: String 17 | } 18 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/execution/shouldExecuteQueryWithSingleFieldAndModification.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | findPurchase(purchaseId: ID): Purchase @topic(name: "purchase-topic", keyArgument: "purchaseId") 3 | } 4 | 5 | type Purchase { 6 | purchaseId: ID!, 7 | productId: Long!, 8 | userId: ID!, 9 | amount: Int, 10 | price: Price, 11 | infos: [String] 12 | product: Product @topic(name: "product-topic" keyField: "productId") 13 | } 14 | 15 | type Product { 16 | productId: Long!, 17 | name: String, 18 | description: String, 19 | price: Price, 20 | metadata: Metadata 21 | } 22 | 23 | type Price { 24 | total: Float, 25 | currency: String 26 | } 27 | 28 | type Metadata { 29 | created_at: Int, 30 | source: String 31 | } -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/execution/shouldExecuteQueryWithSingleFieldAndModificationQuery.graphql: -------------------------------------------------------------------------------- 1 | { 2 | findPurchase(purchaseId: "purchase1") { 3 | purchaseId 4 | productId 5 | product { 6 | productId 7 | name 8 | price { 9 | total 10 | } 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/execution/shouldExecuteQueryWithSingleFieldQuery.graphql: -------------------------------------------------------------------------------- 1 | { 2 | findPurchase(purchaseId: "test") { 3 | purchaseId, 4 | productId, 5 | amount, 6 | } 7 | } -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/execution/shouldExecuteRange.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | userRequests( 3 | userId: Int 4 | timestampFrom: Int 5 | timestampTo: Int 6 | ): [UserRequests] @topic(name: "user-request-range", 7 | keyArgument: "userId", 8 | rangeFrom: "timestampFrom", 9 | rangeTo: "timestampTo") 10 | } 11 | 12 | type UserRequests { 13 | userId: Int 14 | timestamp: Int 15 | requests: Int 16 | } 17 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/execution/shouldExecuteRangeQuery.graphql: -------------------------------------------------------------------------------- 1 | { 2 | userRequests(userId: 1, timestampFrom: 1, timestampTo: 3) { 3 | requests 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/person.graphql: -------------------------------------------------------------------------------- 1 | type Person { 2 | _id: String! 3 | firstname: [GrainValue] 4 | lastname: [GrainValue] 5 | } 6 | 7 | type GrainValue { 8 | _in_utc: String! 9 | _v: String! 10 | } -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/product.graphql: -------------------------------------------------------------------------------- 1 | type Product { 2 | description: String 3 | metadata: Metadata 4 | name: String 5 | price: Price 6 | productId: ID! 7 | } 8 | 9 | type Price { 10 | currency: String 11 | total: Float 12 | } 13 | 14 | type Metadata { 15 | created_at: Int 16 | source: String 17 | } 18 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/purchase.graphql: -------------------------------------------------------------------------------- 1 | type Purchase { 2 | amount: Int 3 | infos: [String] 4 | price: Price! 5 | priceHistory: [Price!] 6 | productId: ID! 7 | purchaseId: ID! 8 | userId: ID! 9 | } 10 | 11 | type Price { 12 | currency: String 13 | total: Float 14 | } 15 | 16 | -------------------------------------------------------------------------------- /gateway/src/test/resources/schema/schema.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | findPurchase(purchaseId: ID): Purchase @topic(name: "purchase-topic", keyArgument: "purchaseId") 3 | } 4 | 5 | type Purchase { 6 | purchaseId: ID!, 7 | productId: ID!, 8 | userId: ID!, 9 | amount: Int, 10 | price: Price!, 11 | priceHistory: [Price!], 12 | infos: [String] 13 | product: Product @topic(name: "product-topic" keyField: "productId") 14 | } 15 | 16 | type Product { 17 | productId: ID!, 18 | name: String, 19 | description: String, 20 | price: Price, 21 | metadata: Metadata 22 | } 23 | 24 | type Price { 25 | total: Float, 26 | currency: String 27 | } 28 | 29 | type Metadata { 30 | created_at: Int, 31 | source: String 32 | } 33 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | org.gradle.caching=true 2 | org.gradle.parallel=flase # Embedded Kafka does not reliably work in parallel since Kafka 3.0 3 | #----- Sonar 4 | # systemProp.sonar.host.url=http://localhost:9000 5 | # systemProp.sonar.login= 6 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bakdata/quick/49bf6a9cf1e7153ea3694b518bbcc7cb50a18c7b/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/bakdata/quick/ingest/IngestApplication.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.ingest; 18 | 19 | import io.micronaut.runtime.Micronaut; 20 | 21 | /** 22 | * Main ingest application. 23 | */ 24 | public final class IngestApplication { 25 | private IngestApplication() { 26 | } 27 | 28 | public static void main(final String[] args) { 29 | Micronaut.run(IngestApplication.class); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /ingest/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | micronaut: 2 | application: 3 | name: ingest-service 4 | 5 | server: 6 | cors: 7 | enabled: true 8 | 9 | metrics: 10 | enabled: true 11 | 12 | 13 | # https://micronaut-projects.github.io/micronaut-kubernetes/2.2.0/guide/index.html 14 | kubernetes: 15 | client: 16 | discovery: 17 | enabled: false 18 | 19 | 20 | endpoints: 21 | # Enable all default endpoint but on custom port so they are not publicly accessible 22 | all: 23 | port: 8081 24 | sensitive: false 25 | enabled: true 26 | -------------------------------------------------------------------------------- /ingest/src/test/resources/application-test.yaml: -------------------------------------------------------------------------------- 1 | micronaut: 2 | security: 3 | enabled: false 4 | 5 | metrics: 6 | enabled: false 7 | 8 | endpoints: 9 | # Enable all default endpoint but on custom port so they are not publicly accessible 10 | all: 11 | enabled: false 12 | 13 | 14 | quick.apikey: 15 | test_key 16 | -------------------------------------------------------------------------------- /lombok.config: -------------------------------------------------------------------------------- 1 | # This file is generated by the 'io.freefair.lombok' Gradle plugin 2 | config.stopBubbling = true 3 | # Annotate lombok generated code so that we can exclude it from codequality 4 | lombok.addLombokGeneratedAnnotation = true 5 | # Allows jackson to (de)serialize POJOs annotated with lombok's `@Value` 6 | lombok.anyConstructor.addConstructorProperties = true 7 | -------------------------------------------------------------------------------- /manager/src/main/java/com/bakdata/quick/manager/ManagerApplication.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.manager; 18 | 19 | import io.micronaut.runtime.Micronaut; 20 | 21 | 22 | /** 23 | * Main application for manager. 24 | */ 25 | public class ManagerApplication { 26 | public static void main(final String[] args) { 27 | Micronaut.run(args); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /manager/src/main/java/com/bakdata/quick/manager/ManagerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.manager; 18 | 19 | import io.fabric8.kubernetes.client.DefaultKubernetesClient; 20 | import io.fabric8.kubernetes.client.KubernetesClient; 21 | import io.micronaut.context.annotation.Factory; 22 | import jakarta.inject.Singleton; 23 | 24 | /** 25 | * Factory for manager beans. 26 | */ 27 | @Factory 28 | public class ManagerFactory { 29 | 30 | @Singleton 31 | public KubernetesClient kubernetesClient() { 32 | return new DefaultKubernetesClient(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /manager/src/main/java/com/bakdata/quick/manager/config/ImagePullPolicy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.manager.config; 18 | 19 | import com.fasterxml.jackson.annotation.JsonValue; 20 | import lombok.Getter; 21 | 22 | /** 23 | * Defines the image pull policies of the manager resources. 24 | * 25 | * @see ApplicationSpecificationConfig 26 | */ 27 | public enum ImagePullPolicy { 28 | IF_NOT_PRESENT("IfNotPresent"), 29 | ALWAYS("Always"), 30 | NEVER("Never"); 31 | 32 | @Getter 33 | @JsonValue 34 | private final String policyName; 35 | 36 | ImagePullPolicy(final String policyName) { 37 | this.policyName = policyName; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /manager/src/main/java/com/bakdata/quick/manager/k8s/cluster/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /** 18 | * Collection of classes that ensure the correctness of the cluster's state. 19 | */ 20 | package com.bakdata.quick.manager.k8s.cluster; 21 | -------------------------------------------------------------------------------- /manager/src/main/java/com/bakdata/quick/manager/k8s/middleware/MiddlewareList.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.manager.k8s.middleware; 18 | 19 | import io.fabric8.kubernetes.client.CustomResourceList; 20 | 21 | /** 22 | * CRD list for middlewares. 23 | */ 24 | public class MiddlewareList extends CustomResourceList { 25 | } 26 | -------------------------------------------------------------------------------- /manager/src/main/java/com/bakdata/quick/manager/k8s/middleware/MiddlewareSpec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.manager.k8s.middleware; 18 | 19 | import edu.umd.cs.findbugs.annotations.Nullable; 20 | import io.sundr.builder.annotations.Buildable; 21 | import lombok.Data; 22 | 23 | /** 24 | * POJO for middleware spec. 25 | */ 26 | @Data 27 | @Buildable( 28 | editableEnabled = false, 29 | builderPackage = Middleware.BUILDER_PATH 30 | ) 31 | public class MiddlewareSpec { 32 | @Nullable 33 | private StripPrefix stripPrefix; 34 | } 35 | -------------------------------------------------------------------------------- /manager/src/main/java/com/bakdata/quick/manager/k8s/middleware/StripPrefix.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.manager.k8s.middleware; 18 | 19 | import edu.umd.cs.findbugs.annotations.Nullable; 20 | import io.sundr.builder.annotations.Buildable; 21 | import java.util.List; 22 | import lombok.Data; 23 | 24 | /** 25 | * POJO for strip prefix middleware. 26 | */ 27 | @Data 28 | @Buildable( 29 | editableEnabled = false, 30 | builderPackage = Middleware.BUILDER_PATH 31 | ) 32 | public class StripPrefix { 33 | @Nullable 34 | private List prefixes; 35 | } 36 | -------------------------------------------------------------------------------- /manager/src/main/java/com/bakdata/quick/manager/k8s/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /** 18 | * Quick manager kubernetes integration. 19 | */ 20 | 21 | @Configuration 22 | @Requires(env = Environment.KUBERNETES) 23 | package com.bakdata.quick.manager.k8s; 24 | 25 | import io.micronaut.context.annotation.Configuration; 26 | import io.micronaut.context.annotation.Requires; 27 | import io.micronaut.context.env.Environment; 28 | -------------------------------------------------------------------------------- /manager/src/main/java/com/bakdata/quick/manager/k8s/resource/QuickResource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.manager.k8s.resource; 18 | 19 | import io.fabric8.kubernetes.api.model.HasMetadata; 20 | 21 | /** 22 | * This interface describes a single quick kubernetes resource, like Deployment, Service, Ingress, etc. 23 | */ 24 | public interface QuickResource { 25 | HasMetadata getResource(); 26 | 27 | QuickResourceErrorHandler getErrorHandler(); 28 | } 29 | -------------------------------------------------------------------------------- /manager/src/main/java/com/bakdata/quick/manager/k8s/resource/QuickResourceErrorHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.manager.k8s.resource; 18 | 19 | import io.reactivex.Completable; 20 | 21 | /** 22 | * Error handler for operations on {@link QuickResource}. 23 | */ 24 | public interface QuickResourceErrorHandler { 25 | Completable handleCreationError(final Throwable ex); 26 | 27 | Completable handleDeletionError(final Throwable ex); 28 | } 29 | -------------------------------------------------------------------------------- /manager/src/main/java/com/bakdata/quick/manager/k8s/resource/ResourceLoader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.manager.k8s.resource; 18 | 19 | import com.bakdata.quick.common.api.model.manager.creation.CreationData; 20 | import com.bakdata.quick.manager.k8s.resource.QuickResources.ResourcePrefix; 21 | 22 | /** 23 | * This interface is implemented by the quick resources to create or delete the Kubernetes resources of each quick 24 | * resource. 25 | * 26 | */ 27 | public interface ResourceLoader { 28 | T forCreation(final S creationData, final ResourcePrefix resourcePrefix); 29 | 30 | T forDeletion(String name); 31 | } 32 | -------------------------------------------------------------------------------- /manager/src/main/java/com/bakdata/quick/manager/mirror/MirrorService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.manager.mirror; 18 | 19 | import com.bakdata.quick.common.api.model.manager.creation.MirrorCreationData; 20 | import io.reactivex.Completable; 21 | 22 | /** 23 | * Service managing Quick's mirror applications. 24 | */ 25 | public interface MirrorService { 26 | Completable createMirror(final MirrorCreationData mirrorCreationData); 27 | 28 | Completable createInternalMirror(final MirrorCreationData mirrorCreationData); 29 | 30 | Completable deleteMirror(final String topicName); 31 | } 32 | -------------------------------------------------------------------------------- /manager/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | micronaut: 2 | security: 3 | token: 4 | propagation: 5 | header: 6 | enabled: true 7 | headerName: "X-API-Key" 8 | prefix: "" 9 | enabled: true 10 | service-id-regex: http://${quick.service.ingest} 11 | 12 | application: 13 | name: manager 14 | 15 | metrics: 16 | enabled: true 17 | 18 | 19 | endpoints: 20 | # Enable all default endpoint but on custom port so they are not publicly accessible 21 | all: 22 | port: 8081 23 | sensitive: false 24 | enabled: true 25 | 26 | # https://micronaut-projects.github.io/micronaut-kubernetes/2.2.0/guide/index.html 27 | kubernetes: 28 | client: 29 | discovery: 30 | enabled: false 31 | 32 | quick: 33 | manager: 34 | update-managed-images: true 35 | create-topic-registry: true 36 | applications: 37 | spec: 38 | resources: 39 | memory: 40 | limit: 1G 41 | request: 256Mi 42 | cpu: 43 | limit: 1 44 | request: 0.1 45 | -------------------------------------------------------------------------------- /manager/src/main/resources/k8s/crd/middleware-crd.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apiextensions.k8s.io/v1beta1 2 | kind: CustomResourceDefinition 3 | metadata: 4 | name: middlewares.traefik.containo.us 5 | spec: 6 | group: traefik.containo.us 7 | names: 8 | kind: Middleware 9 | plural: middlewares 10 | scope: Namespaced 11 | version: v1alpha1 12 | versions: 13 | - name: v1alpha1 14 | served: true 15 | storage: true 16 | -------------------------------------------------------------------------------- /manager/src/main/resources/k8s/templates/gateway/config-map.th.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: [(${name})]-config 5 | labels: 6 | app.kubernetes.io/name: [(${name})]-config 7 | app.kubernetes.io/managed-by: quick 8 | app.kubernetes.io/component: gateway 9 | data: 10 | schema.graphql: | 11 | [(${schema})] 12 | -------------------------------------------------------------------------------- /manager/src/main/resources/k8s/templates/gateway/ingress.th.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.k8s.io/v1 2 | kind: Ingress 3 | metadata: 4 | labels: 5 | app.kubernetes.io/name: [(${deploymentName})] 6 | app.kubernetes.io/managed-by: quick 7 | app.kubernetes.io/component: gateway 8 | annotations: 9 | kubernetes.io/ingress.class: traefik 10 | traefik.ingress.kubernetes.io/router.middlewares: [(${namespace})]-[(${deploymentName})]@kubernetescrd 11 | traefik.ingress.kubernetes.io/router.entrypoints: [(${ingressEntrypoint})] 12 | traefik.ingress.kubernetes.io/router.tls: "[(${ingressSsl})]" 13 | name: [(${deploymentName})] 14 | spec: 15 | rules: 16 | - [# th:if= "${host}"]host: [(${host})][/] 17 | http: 18 | paths: 19 | - path: /gateway/[(${pathName})] 20 | pathType: Prefix 21 | backend: 22 | service: 23 | name: [(${deploymentName})] 24 | port: 25 | number: 80 26 | -------------------------------------------------------------------------------- /manager/src/main/resources/k8s/templates/gateway/middleware.th.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: traefik.containo.us/v1alpha1 2 | kind: Middleware 3 | metadata: 4 | name: [(${deploymentName})] 5 | labels: 6 | app.kubernetes.io/name: [(${deploymentName})] 7 | app.kubernetes.io/managed-by: quick 8 | app.kubernetes.io/component: gateway 9 | spec: 10 | stripPrefix: 11 | prefixes: 12 | - /gateway/[(${pathName})] 13 | -------------------------------------------------------------------------------- /manager/src/main/resources/k8s/templates/gateway/service.th.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | app.kubernetes.io/name: [(${name})] 6 | app.kubernetes.io/managed-by: quick 7 | app.kubernetes.io/component: gateway 8 | name: [(${name})] 9 | spec: 10 | ports: 11 | - port: 80 12 | protocol: TCP 13 | targetPort: 8080 14 | selector: 15 | app.kubernetes.io/name: [(${name})] 16 | app.kubernetes.io/component: gateway 17 | type: ClusterIP 18 | -------------------------------------------------------------------------------- /manager/src/main/resources/k8s/templates/mirror/service.th.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | app.kubernetes.io/name: [(${name})] 6 | app.kubernetes.io/managed-by: quick 7 | app.kubernetes.io/component: mirror 8 | name: [(${name})] 9 | spec: 10 | ports: 11 | - port: 80 12 | protocol: TCP 13 | targetPort: 8080 14 | selector: 15 | app.kubernetes.io/name: [(${name})] 16 | app.kubernetes.io/component: mirror 17 | type: ClusterIP 18 | -------------------------------------------------------------------------------- /manager/src/main/resources/k8s/templates/streamsApp/service.th.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | app: [(${name})] 6 | name: [(${name})] 7 | spec: 8 | ports: 9 | - port: 80 10 | protocol: TCP 11 | targetPort: [(${port})] 12 | selector: 13 | app.kubernetes.io/name: [(${name})] 14 | app.kubernetes.io/component: streamsApp 15 | type: ClusterIP 16 | -------------------------------------------------------------------------------- /manager/src/test/resources/application-test.yaml: -------------------------------------------------------------------------------- 1 | micronaut: 2 | security: 3 | enabled: false 4 | metrics: 5 | enabled: false 6 | 7 | endpoints: 8 | # Enable all default endpoint but on custom port so they are not publicly accessible 9 | all: 10 | enabled: false 11 | 12 | quick: 13 | manager: 14 | update-managed-images: false 15 | create-topic-registry: false 16 | kafka: 17 | bootstrap-server: dummy:9092 18 | schema-registry-url: http://test:8081 19 | apikey: test_key 20 | service: 21 | ingest: test-ingest 22 | gateway: test-gateway 23 | 24 | applications: 25 | spec: 26 | resources: 27 | memory: 28 | limit: 1G 29 | request: 256Mi 30 | cpu: 31 | limit: 1 32 | request: 0.1 33 | -------------------------------------------------------------------------------- /manager/src/test/resources/schema/avro/test.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "Test", 4 | "namespace": "com.bakdata.quick.avro", 5 | "fields": [ 6 | { 7 | "name": "test", 8 | "type": [ 9 | "null", 10 | "int" 11 | ] 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /manager/src/test/resources/schema/graphql/shouldConvertGraphQLEnumFields.graphql: -------------------------------------------------------------------------------- 1 | type Product { 2 | status: Status 3 | status2: Status 4 | } 5 | 6 | enum Status { 7 | SOLD 8 | AVAILABLE 9 | } 10 | -------------------------------------------------------------------------------- /manager/src/test/resources/schema/graphql/shouldConvertGraphQLObjectTypes.graphql: -------------------------------------------------------------------------------- 1 | type Mock { 2 | id: ID 3 | complexObject: ComplexObject 4 | simpleString: String 5 | } 6 | 7 | type ComplexObject { 8 | id: ID 9 | nestedObject: NestedObject 10 | } 11 | 12 | type NestedObject { 13 | id: ID 14 | } 15 | -------------------------------------------------------------------------------- /manager/src/test/resources/schema/graphql/shouldConvertGraphQLScalarFields.graphql: -------------------------------------------------------------------------------- 1 | type Scalars { 2 | int: Int! 3 | float: Float! 4 | string: String! 5 | bool: Boolean! 6 | id: ID! 7 | long: Long! 8 | short: Short! 9 | char: Char! 10 | } 11 | -------------------------------------------------------------------------------- /manager/src/test/resources/schema/graphql/shouldConvertListType.graphql: -------------------------------------------------------------------------------- 1 | type ListType { 2 | optionalSimpleList: [Int] 3 | optionalComplexList: [ComplexObject] 4 | 5 | requiredSimpleList: [Int]! 6 | requiredComplexList: [ComplexObject]! 7 | 8 | requiredSimpleList2: [Int!] 9 | requiredComplexList2: [ComplexObject!] 10 | 11 | requiredSimpleList3: [Int!]! 12 | requiredComplexList3: [ComplexObject!]! 13 | } 14 | 15 | type ComplexObject { 16 | id: ID 17 | } 18 | -------------------------------------------------------------------------------- /manager/src/test/resources/schema/graphql/shouldConvertOptionalAndRequired.graphql: -------------------------------------------------------------------------------- 1 | type OptionalRequired { 2 | required: Int! 3 | optional: Int 4 | } 5 | -------------------------------------------------------------------------------- /manager/src/test/resources/schema/graphql/shouldCreateConfigmapWithSchema.graphql: -------------------------------------------------------------------------------- 1 | type Query { 2 | findPurchase(purchaseId: String): Purchase @topic(name: "purchase", keyArgument: "purchaseId") 3 | allPurchases: [Purchase!] @topic(name: "purchase") 4 | } 5 | 6 | type Subscription { 7 | purchases: Purchase @topic(name: "purchase") 8 | } 9 | 10 | type Purchase { 11 | purchaseId: String! 12 | productId: Int! 13 | userId: Int! 14 | product: Product @topic(name: "product", keyField: "productId") 15 | amount: Int 16 | price: Price 17 | } 18 | 19 | type Product { 20 | productId: Int! 21 | name: String 22 | description: String 23 | price: Price 24 | } 25 | 26 | type Price { 27 | total: Float 28 | currency: String 29 | } 30 | -------------------------------------------------------------------------------- /mirror/src/main/java/com/bakdata/quick/mirror/StoreType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.mirror; 18 | 19 | /** 20 | * Store backend to use. 21 | */ 22 | public enum StoreType { 23 | INMEMORY, 24 | ROCKSDB 25 | } 26 | -------------------------------------------------------------------------------- /mirror/src/main/java/com/bakdata/quick/mirror/context/IndexInputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.mirror.context; 18 | 19 | import com.bakdata.quick.common.type.QuickTopicData.QuickData; 20 | import lombok.Value; 21 | import org.apache.kafka.streams.kstream.KStream; 22 | 23 | /** 24 | * Contains the key and value data along with the stream. 25 | * 26 | * @param Type of the key 27 | * @param Type of the value 28 | */ 29 | @Value 30 | public class IndexInputStream { 31 | QuickData keyData; 32 | QuickData valueData; 33 | KStream stream; 34 | } 35 | -------------------------------------------------------------------------------- /mirror/src/main/java/com/bakdata/quick/mirror/context/MirrorContextProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | package com.bakdata.quick.mirror.context; 19 | 20 | /** 21 | * Imitates the functionality of the javax's Provider 22 | * and extends it with an additional method to set a context. 23 | */ 24 | public interface MirrorContextProvider { 25 | 26 | /** 27 | * Provides a fully-constructed and injected instance of QueryServiceContext. 28 | * 29 | * @return An instance of the QueryServiceContext 30 | */ 31 | MirrorContext get(); 32 | 33 | /** 34 | * Sets a context. The idea behind this method is to 35 | * circumvent the need to a bean through the ApplicationContext 36 | * 37 | * @param context an instance of QueryServiceContext 38 | */ 39 | void setMirrorContext(MirrorContext context); 40 | } 41 | -------------------------------------------------------------------------------- /mirror/src/main/java/com/bakdata/quick/mirror/context/RangeIndexProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.mirror.context; 18 | 19 | 20 | import edu.umd.cs.findbugs.annotations.Nullable; 21 | import lombok.Value; 22 | 23 | /** 24 | * Contains the range index properties. 25 | */ 26 | @Value 27 | public class RangeIndexProperties { 28 | String storeName; 29 | @Nullable 30 | String rangeField; 31 | 32 | /** 33 | * Checks if the range index should be built or not. 34 | */ 35 | public boolean isEnabled() { 36 | return this.rangeField != null; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /mirror/src/main/java/com/bakdata/quick/mirror/context/RetentionTimeProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.mirror.context; 18 | 19 | import edu.umd.cs.findbugs.annotations.Nullable; 20 | import java.time.Duration; 21 | import lombok.Value; 22 | 23 | /** 24 | * Contains the retention time index properties. 25 | */ 26 | @Value 27 | public class RetentionTimeProperties { 28 | String storeName; 29 | @Nullable 30 | Duration retentionTime; 31 | 32 | /** 33 | * Checks if the retention time topology is enabled or not. 34 | */ 35 | public boolean isEnabled() { 36 | return this.retentionTime != null; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /mirror/src/main/java/com/bakdata/quick/mirror/range/extractor/SchemaExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.mirror.range.extractor; 18 | 19 | import com.bakdata.quick.mirror.range.extractor.type.FieldTypeExtractor; 20 | import com.bakdata.quick.mirror.range.extractor.value.FieldValueExtractor; 21 | 22 | /** 23 | * Provides the {@link FieldTypeExtractor} and the {@link FieldValueExtractor}. 24 | */ 25 | public interface SchemaExtractor { 26 | /** 27 | * Returns the {@link FieldTypeExtractor}. 28 | */ 29 | FieldTypeExtractor getFieldTypeExtractor(); 30 | 31 | /** 32 | * Returns the {@link FieldValueExtractor}. 33 | */ 34 | FieldValueExtractor getFieldValueExtractor(); 35 | } 36 | -------------------------------------------------------------------------------- /mirror/src/main/java/com/bakdata/quick/mirror/range/extractor/type/FieldTypeExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.mirror.range.extractor.type; 18 | 19 | import com.bakdata.quick.common.type.QuickTopicType; 20 | import io.confluent.kafka.schemaregistry.ParsedSchema; 21 | 22 | /** 23 | * Extracts the {@link QuickTopicType} of given field name in the {@link ParsedSchema}. 24 | */ 25 | @FunctionalInterface 26 | public interface FieldTypeExtractor { 27 | QuickTopicType extract(final ParsedSchema parsedSchema, final String fieldName); 28 | } 29 | -------------------------------------------------------------------------------- /mirror/src/main/java/com/bakdata/quick/mirror/range/extractor/value/FieldValueExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.mirror.range.extractor.value; 18 | 19 | /** 20 | * An extractor for retrieving values from schemas. 21 | * 22 | * @param Type of the schema 23 | */ 24 | public interface FieldValueExtractor { 25 | /** 26 | * Extracts the value of a field from a complex value (Avro record, or Protobuf message). 27 | * 28 | * @param complexValue Avro record or Protobuf message 29 | * @param fieldName Name of the field 30 | * @param fieldClass Class of the field 31 | * @param Type of the field to be extracted 32 | * @return The value of the field 33 | */ 34 | F extract(final V complexValue, final String fieldName, final Class fieldClass); 35 | } 36 | -------------------------------------------------------------------------------- /mirror/src/main/java/com/bakdata/quick/mirror/range/indexer/RangeIndexer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.mirror.range.indexer; 18 | 19 | import com.bakdata.quick.mirror.range.MirrorRangeProcessor; 20 | 21 | /** 22 | * Creates the range index in the {@link MirrorRangeProcessor}. 23 | */ 24 | @FunctionalInterface 25 | public interface RangeIndexer { 26 | String createIndex(final K key, final V value); 27 | 28 | default String createRangeIndexFormat(final T key, final String paddedValue) { 29 | return String.format("%s_%s", key, paddedValue); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /mirror/src/main/java/com/bakdata/quick/mirror/range/padder/EndRange.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.mirror.range.padder; 18 | 19 | /** 20 | * Defines the type of the end range. 21 | */ 22 | public enum EndRange { 23 | INCLUSIVE, 24 | EXCLUSIVE 25 | } 26 | -------------------------------------------------------------------------------- /mirror/src/main/java/com/bakdata/quick/mirror/range/padder/ZeroPadder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.mirror.range.padder; 18 | 19 | /** 20 | * Pads zeros to the left number. 21 | * 22 | * @param Type of the number 23 | */ 24 | public interface ZeroPadder { 25 | 26 | /** 27 | * Converts the value T to a numerical string with zeros padded to the left to keep the lexicographical order. 28 | */ 29 | String padZero(final T number); 30 | 31 | /** 32 | * Return the class type of T. 33 | */ 34 | Class getPadderClass(); 35 | 36 | /** 37 | * The implementation of this method should return the end of the range. The end of the range could be exclusive or 38 | * inclusive. 39 | */ 40 | T getEndOfRange(final String stringValue); 41 | } 42 | -------------------------------------------------------------------------------- /mirror/src/main/java/com/bakdata/quick/mirror/service/QueryService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 bakdata GmbH 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bakdata.quick.mirror.service; 18 | 19 | import com.bakdata.quick.common.api.model.mirror.MirrorValue; 20 | import io.micronaut.http.HttpResponse; 21 | import io.reactivex.Single; 22 | import java.util.List; 23 | 24 | /** 25 | * Service for querying kafka backend. 26 | * 27 | * @param value type 28 | */ 29 | public interface QueryService { 30 | Single>> get(final String key); 31 | 32 | Single>>> getValues(final List keys); 33 | 34 | Single>>> getAll(); 35 | 36 | Single>>> getRange(final String key, final String from, final String to); 37 | } 38 | -------------------------------------------------------------------------------- /mirror/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | micronaut: 2 | application: 3 | name: mirror 4 | 5 | security: 6 | enabled: false 7 | 8 | # Too expensive to enable for each mirror for now 9 | metrics: 10 | enabled: false 11 | 12 | # https://micronaut-projects.github.io/micronaut-kubernetes/2.2.0/guide/index.html 13 | kubernetes: 14 | client: 15 | discovery: 16 | enabled: false 17 | 18 | endpoints: 19 | # Enable all default endpoint but on custom port so they are not publicly accessible 20 | all: 21 | port: 8081 22 | sensitive: false 23 | enabled: true 24 | -------------------------------------------------------------------------------- /mirror/src/test/resources/application-test.yaml: -------------------------------------------------------------------------------- 1 | micronaut: 2 | security: 3 | enabled: false 4 | 5 | metrics: 6 | enabled: false 7 | 8 | endpoints: 9 | all: 10 | enabled: false 11 | 12 | quick: 13 | apikey: 14 | test_key 15 | kafka: 16 | bootstrap-server: dummy:123 17 | -------------------------------------------------------------------------------- /openapi/spec/responses.yaml: -------------------------------------------------------------------------------- 1 | components: 2 | responses: 3 | 4 | Success: 5 | description: OK 6 | 7 | NoContent: 8 | description: No Content 9 | 10 | DefaultError: 11 | description: Unexpected error 12 | content: 13 | application/json: 14 | schema: 15 | $ref: 'schemas.yaml#/components/schemas/ErrorMessage' 16 | -------------------------------------------------------------------------------- /settings.gradle.kts: -------------------------------------------------------------------------------- 1 | pluginManagement { 2 | repositories { 3 | gradlePluginPortal() 4 | mavenCentral() 5 | } 6 | } 7 | 8 | rootProject.name = "quick" 9 | 10 | includeBuild("build-logic") 11 | 12 | include("common") 13 | include("gateway") 14 | include("ingest") 15 | include("manager") 16 | include("mirror") 17 | --------------------------------------------------------------------------------