├── gradle.properties ├── apps ├── api │ ├── src │ │ ├── main │ │ │ ├── resources │ │ │ │ ├── ci.conf │ │ │ │ ├── logback.xml │ │ │ │ └── default.conf │ │ │ └── kotlin │ │ │ │ └── vanillakotlin │ │ │ │ └── api │ │ │ │ ├── Config.kt │ │ │ │ ├── App.kt │ │ │ │ └── favoritethings │ │ │ │ ├── FavoriteThingsService.kt │ │ │ │ └── FavoriteThingsRoutes.kt │ │ └── test │ │ │ └── kotlin │ │ │ └── vanillakotlin │ │ │ └── api │ │ │ ├── TestData.kt │ │ │ ├── MockThingServer.kt │ │ │ ├── ApiTestUtilities.kt │ │ │ ├── ConfigTest.kt │ │ │ ├── favoritethings │ │ │ ├── FavoriteThingsServiceTest.kt │ │ │ └── FavoriteThingsRoutesTest.kt │ │ │ └── AppTest.kt │ └── api.gradle.kts ├── outbox-processor │ ├── src │ │ ├── main │ │ │ ├── resources │ │ │ │ ├── ci.conf │ │ │ │ ├── local.conf │ │ │ │ ├── logback.xml │ │ │ │ └── default.conf │ │ │ └── kotlin │ │ │ │ └── vanillakotlin │ │ │ │ └── outboxprocessor │ │ │ │ ├── Config.kt │ │ │ │ ├── OutboxProcessor.kt │ │ │ │ └── App.kt │ │ └── test │ │ │ └── kotlin │ │ │ └── vanillakotlin │ │ │ └── outboxprocessor │ │ │ ├── OutboxTestFunctions.kt │ │ │ ├── OutboxProcessorTest.kt │ │ │ └── AppTest.kt │ └── outbox-processor.gradle.kts ├── apps.gradle.kts ├── bulk-inserter │ ├── src │ │ ├── main │ │ │ ├── resources │ │ │ │ ├── ci.conf │ │ │ │ ├── local.conf │ │ │ │ ├── logback.xml │ │ │ │ └── default.conf │ │ │ └── kotlin │ │ │ │ └── vanillakotlin │ │ │ │ └── bulkinserter │ │ │ │ ├── Config.kt │ │ │ │ ├── App.kt │ │ │ │ └── BulkInserterHandler.kt │ │ └── test │ │ │ └── resources │ │ │ └── local.conf │ └── bulk-inserter.gradle.kts └── kafka-transformer │ ├── src │ ├── main │ │ ├── resources │ │ │ ├── ci.conf │ │ │ ├── local.conf │ │ │ ├── logback.xml │ │ │ └── default.conf │ │ └── kotlin │ │ │ └── vanillakotlin │ │ │ └── kafkatransformer │ │ │ ├── Config.kt │ │ │ ├── App.kt │ │ │ └── FavoriteThingsEventHandler.kt │ └── test │ │ ├── kotlin │ │ └── vanillakotlin │ │ │ └── kafkatransformer │ │ │ ├── TestData.kt │ │ │ ├── FavoriteThingsEventHandlerTest.kt │ │ │ └── AppTest.kt │ │ └── resources │ │ └── local.conf │ └── kafka-transformer.gradle.kts ├── gradle ├── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties └── libs.versions.toml ├── libs ├── common │ ├── src │ │ ├── main │ │ │ └── kotlin │ │ │ │ └── vanillakotlin │ │ │ │ ├── extensions │ │ │ │ ├── JsonExtensions.kt │ │ │ │ └── TimeLimiterExtensions.kt │ │ │ │ ├── models │ │ │ │ ├── UserFavoriteThing.kt │ │ │ │ ├── Models.kt │ │ │ │ ├── FavoriteThing.kt │ │ │ │ ├── HealthCheck.kt │ │ │ │ └── Outbox.kt │ │ │ │ ├── app │ │ │ │ └── AppInterfaces.kt │ │ │ │ ├── config │ │ │ │ └── Configuration.kt │ │ │ │ └── serde │ │ │ │ └── ObjectMapperSingleton.kt │ │ ├── testFixtures │ │ │ └── kotlin │ │ │ │ └── vanillakotlin │ │ │ │ ├── extensions │ │ │ │ └── DateTime.kt │ │ │ │ └── random │ │ │ │ └── Random.kt │ │ └── test │ │ │ └── kotlin │ │ │ └── vanillakotlin │ │ │ ├── domain │ │ │ └── OutboxTest.kt │ │ │ ├── extensions │ │ │ └── TimeLimiterExtensionsTest.kt │ │ │ └── random │ │ │ └── RandomTest.kt │ └── common.gradle.kts ├── http4k │ ├── http4k.gradle.kts │ └── src │ │ └── main │ │ └── kotlin │ │ └── vanillakotlin │ │ └── http4k │ │ └── HttpServer.kt ├── rocksdb │ ├── rocksdb.gradle.kts │ └── src │ │ └── main │ │ └── kotlin │ │ └── vanillakotlin │ │ └── rocksdb │ │ ├── core │ │ ├── Disposable.kt │ │ ├── RocksDbColumnFamilyCompressionOptions.kt │ │ ├── RocksDbStore.kt │ │ └── Extensions.kt │ │ └── freshfilter │ │ └── FreshFilter.kt ├── metrics │ ├── metrics.gradle.kts │ └── src │ │ └── main │ │ └── kotlin │ │ └── vanillakotlin │ │ └── metrics │ │ └── OtelMetrics.kt ├── client │ ├── client.gradle.kts │ └── src │ │ ├── main │ │ └── kotlin │ │ │ └── vanillakotlin │ │ │ └── http │ │ │ ├── clients │ │ │ ├── thing │ │ │ │ ├── ThingGraphqlResponse.kt │ │ │ │ └── ThingGateway.kt │ │ │ └── ClientInitializer.kt │ │ │ └── interceptors │ │ │ ├── TelemetryInterceptor.kt │ │ │ ├── Models.kt │ │ │ └── RetryInterceptor.kt │ │ └── test │ │ ├── resources │ │ └── logback-test.xml │ │ └── kotlin │ │ └── vanillakotlin │ │ └── http │ │ ├── client │ │ └── thing │ │ │ └── ThingGraphqlResponseTest.kt │ │ └── interceptors │ │ └── ModelsTest.kt ├── db │ ├── db.gradle.kts │ └── src │ │ ├── test │ │ ├── resources │ │ │ └── logback-test.xml │ │ └── kotlin │ │ │ └── vanillakotlin │ │ │ └── db │ │ │ └── repository │ │ │ ├── OutboxRepositoryTest.kt │ │ │ └── FavoriteThingRepositoryTest.kt │ │ ├── testFixtures │ │ └── kotlin │ │ │ └── vanillakotlin │ │ │ └── db │ │ │ └── repository │ │ │ └── RepositoryTestUtility.kt │ │ └── main │ │ └── kotlin │ │ └── vanillakotlin │ │ └── db │ │ ├── JdbiConfig.kt │ │ └── repository │ │ ├── OutboxRepository.kt │ │ └── FavoriteThingRepository.kt └── kafka │ ├── kafka.gradle.kts │ └── src │ ├── main │ └── kotlin │ │ └── vanillakotlin │ │ └── kafka │ │ ├── transformer │ │ ├── WorkerSelector.kt │ │ └── TransformerEventHandler.kt │ │ ├── provenance │ │ └── Provenance.kt │ │ ├── Metrics.kt │ │ ├── producer │ │ └── PartitionCalculator.kt │ │ └── models │ │ └── Models.kt │ ├── testFixtures │ └── kotlin │ │ └── vanillakotlin │ │ └── kafka │ │ └── KafkaTestUtilities.kt │ └── test │ └── kotlin │ └── vanillakotlin │ └── kafka │ └── producer │ └── PartitionCalculatorTest.kt ├── db-migration ├── src │ └── main │ │ └── resources │ │ └── db │ │ └── migration │ │ ├── V02__add_outbox.sql │ │ ├── afterMigrate.sql │ │ └── V01__add_favorite_item.sql ├── initdb.d │ └── 01_docker-bootstrap.sql └── db-migration.gradle.kts ├── .gitignore ├── docker-compose.yml ├── .editorconfig ├── settings.gradle.kts └── gradlew.bat /gradle.properties: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /apps/api/src/main/resources/ci.conf: -------------------------------------------------------------------------------- 1 | db { 2 | host = "postgres" 3 | } 4 | -------------------------------------------------------------------------------- /apps/outbox-processor/src/main/resources/ci.conf: -------------------------------------------------------------------------------- 1 | kafka { 2 | producer { 3 | broker = "kafka:9092" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /apps/apps.gradle.kts: -------------------------------------------------------------------------------- 1 | tasks { 2 | withType { 3 | dependsOn(":db-migration:flywayMigrate") 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aceluby/vanilla-kotlin/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /libs/common/src/main/kotlin/vanillakotlin/extensions/JsonExtensions.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.extensions 2 | 3 | import vanillakotlin.serde.mapper 4 | 5 | fun Any.toJsonString(): String = mapper.writeValueAsString(this) 6 | -------------------------------------------------------------------------------- /apps/bulk-inserter/src/main/resources/ci.conf: -------------------------------------------------------------------------------- 1 | kafka { 2 | consumer { 3 | broker = "kafka:9092" 4 | } 5 | 6 | producer { 7 | broker = "kafka:9092" 8 | } 9 | } 10 | 11 | db { 12 | host = "postgres" 13 | } 14 | -------------------------------------------------------------------------------- /apps/bulk-inserter/src/main/resources/local.conf: -------------------------------------------------------------------------------- 1 | http { 2 | server { 3 | # When running locally, choose a port that doesn't conflict with the other apps. This lets us run all the apps simultaneously. 4 | port = 8083 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /apps/kafka-transformer/src/main/resources/ci.conf: -------------------------------------------------------------------------------- 1 | kafka { 2 | consumer { 3 | broker = "kafka:9092" 4 | } 5 | 6 | producer { 7 | broker = "kafka:9092" 8 | } 9 | } 10 | 11 | db { 12 | host = "postgres" 13 | } 14 | -------------------------------------------------------------------------------- /apps/kafka-transformer/src/main/resources/local.conf: -------------------------------------------------------------------------------- 1 | http { 2 | server { 3 | # When running locally, choose a port that doesn't conflict with the other apps. This lets us run all the apps simultaneously. 4 | port = 8082 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /apps/outbox-processor/src/main/resources/local.conf: -------------------------------------------------------------------------------- 1 | http { 2 | server { 3 | # When running locally, choose a port that doesn't conflict with the other apps. This lets us run all the apps simultaneously. 4 | port = 8081 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /libs/http4k/http4k.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | `java-test-fixtures` 3 | } 4 | 5 | dependencies { 6 | api(project(":libs:common")) 7 | api(libs.bundles.http4k) 8 | api(libs.kasechange) 9 | 10 | testImplementation(testFixtures(project(":libs:common"))) 11 | } 12 | -------------------------------------------------------------------------------- /libs/rocksdb/rocksdb.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | `java-test-fixtures` 3 | } 4 | 5 | dependencies { 6 | api(project(":libs:common")) 7 | api(project(":libs:metrics")) 8 | implementation(libs.rocksdb) 9 | 10 | testImplementation(testFixtures(project(":libs:common"))) 11 | } 12 | -------------------------------------------------------------------------------- /libs/metrics/metrics.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | `java-test-fixtures` 3 | } 4 | 5 | dependencies { 6 | api(project(":libs:common")) 7 | api(libs.otel.api) 8 | 9 | testImplementation(testFixtures(project(":libs:common"))) 10 | testImplementation(libs.bundles.otel.testing) 11 | } 12 | -------------------------------------------------------------------------------- /libs/client/client.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | `java-test-fixtures` 3 | } 4 | 5 | dependencies { 6 | api(project(":libs:common")) 7 | api(project(":libs:metrics")) 8 | api(libs.okhttp) 9 | implementation(libs.resilience4j.retry) 10 | 11 | testImplementation(testFixtures(project(":libs:common"))) 12 | testImplementation(libs.okhttp.mock.server) 13 | } 14 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Thu Nov 21 08:16:31 CST 2024 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | distributionUrl=https\://binrepo.target.com/artifactory/gradle-distributions/gradle-8.14.2-bin.zip 5 | 6 | #distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.2-bin.zip 7 | zipStoreBase=GRADLE_USER_HOME 8 | zipStorePath=wrapper/dists 9 | -------------------------------------------------------------------------------- /libs/db/db.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | `java-test-fixtures` 3 | } 4 | 5 | dependencies { 6 | api(libs.bundles.jdbi) 7 | api(project(":libs:common")) 8 | 9 | implementation(libs.postgresql) 10 | testImplementation(testFixtures(project(":libs:common"))) 11 | } 12 | 13 | tasks { 14 | withType { 15 | dependsOn(":db-migration:flywayMigrate") 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /db-migration/src/main/resources/db/migration/V02__add_outbox.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE outbox 2 | ( 3 | id INTEGER GENERATED ALWAYS AS IDENTITY, 4 | message_key TEXT NOT NULL, 5 | headers BYTEA, 6 | body BYTEA, 7 | created_ts TIMESTAMP NOT NULL DEFAULT (CURRENT_TIMESTAMP), 8 | PRIMARY KEY (id) 9 | ); 10 | 11 | CREATE INDEX idx_outbox_created ON outbox (created_ts); 12 | -------------------------------------------------------------------------------- /apps/api/src/test/kotlin/vanillakotlin/api/TestData.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.api 2 | 3 | import vanillakotlin.models.Thing 4 | import vanillakotlin.models.ThingIdentifier 5 | import vanillakotlin.random.randomThing 6 | 7 | fun buildTestThing(thingIdentifier: ThingIdentifier = randomThing()): Thing = Thing( 8 | id = thingIdentifier, 9 | productName = "Test Product", 10 | sellingPrice = 19.99, 11 | ) 12 | -------------------------------------------------------------------------------- /libs/common/src/main/kotlin/vanillakotlin/models/UserFavoriteThing.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.models 2 | 3 | /** 4 | * Represents a user's favorite thing for bulk processing. 5 | * This is used as the message format for Kafka messages in the bulk inserter. 6 | */ 7 | data class UserFavoriteThing( 8 | val userName: String, 9 | val thingIdentifier: ThingIdentifier, 10 | val isDeleted: Boolean = false, 11 | ) 12 | -------------------------------------------------------------------------------- /libs/client/src/main/kotlin/vanillakotlin/http/clients/thing/ThingGraphqlResponse.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.http.clients.thing 2 | 3 | import vanillakotlin.models.Thing 4 | 5 | /* 6 | This file contains the data classes used to deserialize the body for the item graphql response. 7 | */ 8 | 9 | data class ThingResponse(val data: ThingData?) { 10 | data class ThingData(val thing: Thing) 11 | 12 | fun toThing(): Thing? = data?.thing 13 | } 14 | -------------------------------------------------------------------------------- /apps/kafka-transformer/src/test/kotlin/vanillakotlin/kafkatransformer/TestData.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.kafkatransformer 2 | 3 | import vanillakotlin.models.Thing 4 | import vanillakotlin.models.ThingIdentifier 5 | import vanillakotlin.random.randomThing 6 | 7 | fun buildTestThing(thingIdentifier: ThingIdentifier = randomThing()): Thing = Thing( 8 | id = thingIdentifier, 9 | productName = "Test Product", 10 | sellingPrice = 19.99, 11 | ) 12 | -------------------------------------------------------------------------------- /db-migration/initdb.d/01_docker-bootstrap.sql: -------------------------------------------------------------------------------- 1 | -- This file is only used by local and CI docker setup scripts 2 | 3 | create database vanilla_kotlin; 4 | 5 | -- create the administrative user 6 | create user vanilla_kotlin with password 'vanilla_kotlin' superuser; 7 | 8 | -- the privileges for the following users are handled in the afterMigrate.sql script, which is run after every flyway migration 9 | create user vanilla_kotlin_app with password 'vanilla_kotlin_app'; 10 | -------------------------------------------------------------------------------- /db-migration/src/main/resources/db/migration/afterMigrate.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- the following statements are run after the migrate command has been executed. 3 | -- 4 | 5 | GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO vanilla_kotlin_app; 6 | GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO vanilla_kotlin_app; 7 | 8 | -- After granting to all tables, we then revoke permissions to the flyway schema table 9 | REVOKE ALL ON TABLE flyway_schema_history FROM vanilla_kotlin_app; 10 | -------------------------------------------------------------------------------- /libs/db/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %-12.12d{HH:mm:ss.SS} %highlight(%-5level) [%thread] %logger{15} %kvp{NONE} %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /apps/bulk-inserter/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %-12.12d{HH:mm:ss.SS} %highlight(%-5level) [%thread] %logger{15} %kvp{NONE} %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /libs/client/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %-12.12d{HH:mm:ss.SS} %highlight(%-5level) [%thread] %logger{15} %kvp{NONE} %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /apps/kafka-transformer/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %-12.12d{HH:mm:ss.SS} %highlight(%-5level) [%thread] %logger{15} %kvp{NONE} %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /apps/outbox-processor/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %-12.12d{HH:mm:ss.SS} %highlight(%-5level) [%thread] %logger{15} %kvp{NONE} %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /apps/api/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %-12.12d{HH:mm:ss.SS} %highlight(%-5level) [%thread] %logger{15} %kvp{NONE} %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /libs/common/src/main/kotlin/vanillakotlin/models/Models.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.models 2 | 3 | // using type aliases for some commonly used primitives helps with readability and avoiding some human errors when working 4 | // with parameters (e.g. preventing ordering issues). They're not as strict as working with e.g. inline classes, but are more flexible 5 | 6 | typealias ThingIdentifier = String 7 | 8 | data class Thing( 9 | val id: ThingIdentifier, 10 | val productName: String, 11 | val sellingPrice: Double, 12 | ) 13 | -------------------------------------------------------------------------------- /libs/db/src/testFixtures/kotlin/vanillakotlin/db/repository/RepositoryTestUtility.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.db.repository 2 | 3 | import org.jdbi.v3.core.Jdbi 4 | import vanillakotlin.db.DbConfig 5 | import vanillakotlin.db.createJdbi 6 | import vanillakotlin.serde.mapper 7 | 8 | fun buildTestDb(): Jdbi = createJdbi( 9 | config = DbConfig( 10 | username = "vanilla_kotlin_app", 11 | password = "vanilla_kotlin_app", 12 | databaseName = "vanilla_kotlin", 13 | ), 14 | objectMapper = mapper, 15 | ) 16 | -------------------------------------------------------------------------------- /db-migration/src/main/resources/db/migration/V01__add_favorite_item.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE favorite_thing ( 2 | id INTEGER GENERATED ALWAYS AS IDENTITY, 3 | thing VARCHAR(511) NOT NULL, 4 | created_ts TIMESTAMP NOT NULL DEFAULT (CURRENT_TIMESTAMP), 5 | updated_ts TIMESTAMP NOT NULL DEFAULT (CURRENT_TIMESTAMP), 6 | PRIMARY KEY (id) 7 | ); 8 | 9 | CREATE UNIQUE INDEX uidx_thing ON favorite_thing (thing); 10 | 11 | ALTER TABLE favorite_thing 12 | ADD CONSTRAINT unique_thing UNIQUE USING INDEX uidx_thing; 13 | -------------------------------------------------------------------------------- /libs/kafka/kafka.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | `java-test-fixtures` 3 | } 4 | 5 | dependencies { 6 | api(project(":libs:common")) 7 | api(project(":libs:metrics")) 8 | api(libs.kafka.client) 9 | implementation(libs.kotlin.coroutines) 10 | 11 | testFixturesImplementation(testFixtures(project(":libs:common"))) 12 | testFixturesImplementation(libs.kafka.client) 13 | 14 | testImplementation(testFixtures(project(":libs:common"))) 15 | testImplementation(libs.bundles.jupiter) 16 | testImplementation(libs.kafka.client) 17 | } 18 | -------------------------------------------------------------------------------- /apps/bulk-inserter/src/test/resources/local.conf: -------------------------------------------------------------------------------- 1 | http { 2 | server { 3 | port = 0 4 | host = "localhost" 5 | } 6 | } 7 | 8 | metrics { 9 | tags = { 10 | _blossom_id = "test" 11 | team = "test" 12 | } 13 | } 14 | 15 | kafka { 16 | consumer { 17 | appName = "test-bulk-inserter" 18 | broker = "localhost:9092" 19 | topics = ["test-topic"] 20 | group = "test-group" 21 | autoOffsetResetConfig = "earliest" 22 | } 23 | } 24 | 25 | db { 26 | username = "vanilla_kotlin_app" 27 | password = "vanilla_kotlin_app" 28 | databaseName = "vanilla_kotlin" 29 | } -------------------------------------------------------------------------------- /apps/bulk-inserter/src/main/resources/default.conf: -------------------------------------------------------------------------------- 1 | http { 2 | server { 3 | port = 8080 4 | } 5 | } 6 | 7 | kafka { 8 | consumer { 9 | appName = "vanillakotlinbulkinserter" 10 | broker = "localhost:9092" 11 | topic = "reference-favorite-items" 12 | group = "bulkinsertter" 13 | autoOffsetResetConfig = "earliest" 14 | } 15 | } 16 | 17 | db { 18 | username = "vanilla_kotlin_app" 19 | password = "vanilla_kotlin_app" 20 | databaseName = "vanilla_kotlin" 21 | } 22 | 23 | metrics { 24 | tags = { 25 | _blossom_id = "CI15247253" 26 | team = "reference" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /apps/bulk-inserter/src/main/kotlin/vanillakotlin/bulkinserter/Config.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.bulkinserter 2 | 3 | import vanillakotlin.db.DbConfig 4 | import vanillakotlin.kafka.consumer.KafkaConsumer 5 | import vanillakotlin.metrics.OtelMetrics 6 | 7 | data class Config( 8 | val http: HttpConfig, 9 | val metrics: OtelMetrics.Config, 10 | val kafka: KafkaConfig, 11 | val db: DbConfig, 12 | ) { 13 | data class HttpConfig(val server: HttpServerConfig) { 14 | data class HttpServerConfig(val port: Int) 15 | } 16 | 17 | data class KafkaConfig(val consumer: KafkaConsumer.Config) 18 | } 19 | -------------------------------------------------------------------------------- /libs/common/src/testFixtures/kotlin/vanillakotlin/extensions/DateTime.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.extensions 2 | 3 | import java.time.Duration 4 | import java.time.Instant 5 | import java.time.LocalDate 6 | 7 | val today: LocalDate 8 | get() = LocalDate.now() 9 | 10 | val Number.minutesAgo: Instant 11 | get() = Instant.now().minus(Duration.ofMinutes(this.toLong())) 12 | 13 | val Number.minutes: Duration 14 | get() = Duration.ofMinutes(this.toLong()) 15 | 16 | val Number.milliseconds: Duration 17 | get() = Duration.ofMillis(this.toLong()) 18 | 19 | val Number.seconds: Duration 20 | get() = Duration.ofSeconds(this.toLong()) 21 | -------------------------------------------------------------------------------- /apps/api/api.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | application 3 | `java-test-fixtures` 4 | } 5 | 6 | application { 7 | applicationName = "app" 8 | mainClass.set("vanillakotlin.api.AppKt") 9 | } 10 | 11 | dependencies { 12 | implementation(project(":libs:common")) 13 | implementation(project(":libs:client")) 14 | implementation(project(":libs:db")) 15 | implementation(project(":libs:metrics")) 16 | implementation(project(":libs:http4k")) 17 | 18 | testImplementation(testFixtures(project(":libs:common"))) 19 | testImplementation(libs.okhttp.mock.server) 20 | } 21 | 22 | tasks { 23 | withType { 24 | dependsOn(":db-migration:flywayMigrate") 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /apps/kafka-transformer/kafka-transformer.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | application 3 | `java-test-fixtures` 4 | } 5 | 6 | application { 7 | applicationName = "app" 8 | mainClass.set("vanillakotlin.kafkatransformer.AppKt") 9 | } 10 | 11 | dependencies { 12 | implementation(project(":libs:client")) 13 | implementation(project(":libs:http4k")) 14 | implementation(project(":libs:kafka")) 15 | implementation(project(":libs:common")) 16 | implementation(project(":libs:metrics")) 17 | testImplementation(libs.okhttp.mock.server) 18 | testImplementation(testFixtures(project(":libs:common"))) 19 | testImplementation(testFixtures(project(":libs:kafka"))) 20 | testImplementation(libs.kafka.client) 21 | } 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .gradle 2 | build/ 3 | !gradle/wrapper/gradle-wrapper.jar 4 | !**/src/main/**/build/ 5 | !**/src/test/**/build/ 6 | 7 | ### IntelliJ IDEA ### 8 | .idea/modules.xml 9 | .idea/jarRepositories.xml 10 | .idea/compiler.xml 11 | .idea/libraries/ 12 | *.iws 13 | *.iml 14 | *.ipr 15 | out/ 16 | !**/src/main/**/out/ 17 | !**/src/test/**/out/ 18 | 19 | ### Kotlin ### 20 | .kotlin 21 | 22 | ### Eclipse ### 23 | .apt_generated 24 | .classpath 25 | .factorypath 26 | .project 27 | .settings 28 | .springBeans 29 | .sts4-cache 30 | bin/ 31 | !**/src/main/**/bin/ 32 | !**/src/test/**/bin/ 33 | 34 | ### NetBeans ### 35 | /nbproject/private/ 36 | /nbbuild/ 37 | /dist/ 38 | /nbdist/ 39 | /.nb-gradle/ 40 | 41 | ### VS Code ### 42 | .vscode/ 43 | 44 | ### Mac OS ### 45 | .DS_Store -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | services: 4 | kafka: 5 | image: apache/kafka:3.8.1 6 | container_name: kafka 7 | hostname: kafka 8 | ports: 9 | - "9092:9092" 10 | networks: 11 | - app 12 | 13 | db: 14 | image: postgres:15.8-alpine 15 | container_name: vanilla-kotlin-db 16 | environment: 17 | - POSTGRES_USER=postgres 18 | - POSTGRES_PASSWORD=password 19 | ports: 20 | - '5432:5432' 21 | networks: 22 | - app 23 | volumes: 24 | - ./db-migration/initdb.d/:/docker-entrypoint-initdb.d 25 | healthcheck: 26 | test: ["CMD", "sh", "-c", "pg_isready -U $$POSTGRES_USER -h $$(hostname -i)"] 27 | interval: 5s 28 | timeout: 5s 29 | retries: 5 30 | 31 | networks: 32 | app: 33 | -------------------------------------------------------------------------------- /db-migration/db-migration.gradle.kts: -------------------------------------------------------------------------------- 1 | buildscript { 2 | dependencies { 3 | classpath(libs.postgresql) 4 | classpath(libs.flyway.database.postgresql) 5 | } 6 | } 7 | 8 | plugins { 9 | alias(libs.plugins.flyway) 10 | } 11 | 12 | flyway { 13 | val dbhost = 14 | when { 15 | System.getenv().containsKey("DATABASE_HOST") -> System.getenv("DATABASE_HOST") 16 | System.getenv().containsKey("CI") -> "postgres" 17 | else -> "localhost" 18 | } 19 | baselineOnMigrate = true 20 | baselineVersion = "0" 21 | url = "jdbc:postgresql://$dbhost:5432/vanilla_kotlin" 22 | user = System.getenv("DATABASE_USERNAME") ?: "vanilla_kotlin" 23 | password = System.getenv("DATABASE_PASSWORD") ?: "vanilla_kotlin" 24 | } 25 | -------------------------------------------------------------------------------- /apps/outbox-processor/src/main/resources/default.conf: -------------------------------------------------------------------------------- 1 | http { 2 | server { 3 | port = 8080 4 | } 5 | } 6 | 7 | kafka { 8 | producer { 9 | broker = "localhost:9092" 10 | topic = "reference-favorite-items" 11 | driverProperties = { 12 | # the kafka-client producer by default uses a JacksonSerializer; we want it to serialize the raw bytes 13 | "value.serializer" = "org.apache.kafka.common.serialization.ByteArraySerializer" 14 | } 15 | } 16 | } 17 | 18 | metrics { 19 | tags = { 20 | _blossom_id = "CI15247253" 21 | team = "reference" 22 | } 23 | } 24 | 25 | db { 26 | username = "vanilla_kotlin_app" 27 | password = "vanilla_kotlin_app" 28 | databaseName = "vanilla_kotlin" 29 | } 30 | 31 | outbox { 32 | pollEvery = "10s" 33 | popMessageLimit = 500 34 | } 35 | -------------------------------------------------------------------------------- /apps/outbox-processor/src/main/kotlin/vanillakotlin/outboxprocessor/Config.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.outboxprocessor 2 | 3 | import vanillakotlin.db.DbConfig 4 | import vanillakotlin.kafka.producer.KafkaProducer 5 | import vanillakotlin.metrics.OtelMetrics 6 | 7 | // This file includes the data classes needed to define the configuration for the app. 8 | // see /docs/configuration.md for more details 9 | 10 | data class Config( 11 | val http: HttpConfig, 12 | val metrics: OtelMetrics.Config, 13 | val db: DbConfig, 14 | val kafka: KafkaConfig, 15 | val outbox: OutboxProcessor.Config, 16 | ) { 17 | data class HttpConfig(val server: HttpServerConfig) { 18 | data class HttpServerConfig(val port: Int) 19 | } 20 | 21 | data class KafkaConfig(val producer: KafkaProducer.Config) 22 | } 23 | -------------------------------------------------------------------------------- /apps/api/src/main/resources/default.conf: -------------------------------------------------------------------------------- 1 | http { 2 | server { 3 | port = 0 4 | host = "localhost" 5 | } 6 | 7 | client { 8 | thing { 9 | apiKey = "test-api-key" 10 | baseUrl = "http://localhost:8080" 11 | } 12 | 13 | connectionConfig { 14 | connectTimeoutMillis = 1000, 15 | readTimeoutMillis = 3000, 16 | maxConnections = 10, 17 | keepAliveDurationMinutes = 5 18 | } 19 | 20 | retryConfig { 21 | maxAttempts = 3, 22 | initialRetryDelayMs = 1000, 23 | maxRetryDelayMs = 10000 24 | } 25 | } 26 | } 27 | 28 | metrics { 29 | tags = { 30 | _pyramid = "test" 31 | team = "test" 32 | } 33 | } 34 | 35 | db { 36 | username = "vanilla_kotlin_app" 37 | password = "vanilla_kotlin_app" 38 | databaseName = "vanilla_kotlin" 39 | } 40 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | [*.{kt,kts}] 2 | ij_kotlin_allow_trailing_comma = true 3 | ij_kotlin_allow_trailing_comma_on_call_site = true 4 | ij_kotlin_name_count_to_use_star_import = 99 5 | ij_kotlin_name_count_to_use_star_import_for_members = 99 6 | max_line_length = 180 7 | 8 | # ktlint rules for parameter formatting 9 | ktlint_function_signature_rule_force_multiline_when_parameter_count_greater_or_equal_than = 2 10 | ktlint_class_signature_rule_force_multiline_when_parameter_count_greater_or_equal_than = 2 11 | ktlint_standard_parameter-list-wrapping = enabled 12 | ktlint_standard_multiline-expression-wrapping = disabled 13 | ktlint_standard_annotation = enabled 14 | ktlint_standard_filename = disabled 15 | ktlint_standard_value-argument-comment = disabled 16 | ktlint_standard_wrapping = disabled 17 | 18 | [*gradle.kts] 19 | max_line_length = off 20 | -------------------------------------------------------------------------------- /libs/common/src/main/kotlin/vanillakotlin/models/FavoriteThing.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.models 2 | 3 | import vanillakotlin.serde.mapper 4 | import java.time.LocalDateTime 5 | 6 | data class FavoriteThing( 7 | val id: Long? = null, 8 | val thingIdentifier: ThingIdentifier, 9 | val createdTs: LocalDateTime = LocalDateTime.now(), 10 | val updatedTs: LocalDateTime = LocalDateTime.now(), 11 | val isDeleted: Boolean = false, 12 | ) { 13 | fun buildOutbox(): Outbox = Outbox( 14 | messageKey = thingIdentifier, 15 | body = mapper.writeValueAsBytes(this), 16 | ) 17 | } 18 | 19 | fun buildDeletedOutbox(thingIdentifier: ThingIdentifier): Outbox = Outbox( 20 | messageKey = thingIdentifier, 21 | body = mapper.writeValueAsBytes(FavoriteThing(thingIdentifier = thingIdentifier, isDeleted = true)), 22 | ) 23 | -------------------------------------------------------------------------------- /apps/bulk-inserter/bulk-inserter.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | application 3 | `java-test-fixtures` 4 | } 5 | 6 | application { 7 | applicationName = "app" 8 | mainClass.set("vanillakotlin.bulkinserter.AppKt") 9 | } 10 | 11 | dependencies { 12 | implementation(project(":libs:common")) 13 | implementation(project(":libs:db")) 14 | implementation(project(":libs:kafka")) 15 | implementation(project(":libs:metrics")) 16 | implementation(project(":libs:http4k")) 17 | testImplementation(testFixtures(project(":libs:common"))) 18 | testImplementation(testFixtures(project(":libs:kafka"))) 19 | testImplementation(testFixtures(project(":libs:db"))) 20 | testImplementation(libs.kafka.client) 21 | } 22 | 23 | tasks { 24 | withType { 25 | dependsOn(":db-migration:flywayMigrate") 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /apps/outbox-processor/outbox-processor.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | application 3 | `java-test-fixtures` 4 | } 5 | 6 | application { 7 | applicationName = "app" 8 | mainClass.set("vanillakotlin.outboxprocessor.AppKt") 9 | } 10 | 11 | dependencies { 12 | implementation(project(":libs:db")) 13 | implementation(project(":libs:kafka")) 14 | implementation(project(":libs:common")) 15 | implementation(project(":libs:http4k")) 16 | implementation(project(":libs:metrics")) 17 | 18 | testImplementation(testFixtures(project(":libs:db"))) 19 | testImplementation(testFixtures(project(":libs:common"))) 20 | testImplementation(testFixtures(project(":libs:kafka"))) 21 | testImplementation(libs.kafka.client) 22 | } 23 | 24 | tasks { 25 | withType { 26 | dependsOn(":db-migration:flywayMigrate") 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /libs/kafka/src/main/kotlin/vanillakotlin/kafka/transformer/WorkerSelector.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.kafka.transformer 2 | 3 | import kotlin.math.absoluteValue 4 | 5 | fun interface WorkerSelector { 6 | fun selectWorker(key: String?): Int 7 | } 8 | 9 | class DefaultWorkerSelector(private val numberOfWorkers: Int) : WorkerSelector { 10 | override fun selectWorker(key: String?): Int = key?.hashCode()?.absoluteValue?.rem(numberOfWorkers) ?: 0 11 | } 12 | 13 | // this is to be used when receiving null keys in the message 14 | class RoundRobinWorkerSelector(private val numberOfWorkers: Int) : WorkerSelector { 15 | // always send use worker index 0 since all workers are listening to the same channel 16 | // this allows for messages to be worked on as room becomes available for any worker 17 | override fun selectWorker(key: String?): Int = 0 18 | } 19 | -------------------------------------------------------------------------------- /libs/common/common.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | `java-test-fixtures` 3 | } 4 | 5 | // we're using api here so that the dependencies are available to all projects that depend on this project 6 | // these dependencies are the common dependencies that we use in all the modules. Be aware that this means 7 | // additions will be added to all applications. 8 | dependencies { 9 | api(libs.bundles.jackson) 10 | api(libs.bundles.hoplite) 11 | api(libs.bundles.resilience4j) 12 | api(libs.bundles.logging) 13 | api(libs.kotlin.coroutines) 14 | 15 | // this is another set of opinionated defaults you should use if you don't already have a strong preference 16 | // it runs tests using the JUnit runner, and adds the kotest assertions library for better assertions 17 | testApi(libs.bundles.testing) 18 | testFixturesApi(libs.bundles.testing) 19 | } 20 | -------------------------------------------------------------------------------- /apps/api/src/main/kotlin/vanillakotlin/api/Config.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.api 2 | 3 | import vanillakotlin.db.DbConfig 4 | import vanillakotlin.http.clients.ConnectionConfig 5 | import vanillakotlin.http.clients.thing.ThingGateway 6 | import vanillakotlin.http.interceptors.RetryInterceptor 7 | import vanillakotlin.metrics.OtelMetrics 8 | 9 | // This file includes the data classes needed to define the configuration for the app. 10 | 11 | data class Config( 12 | val db: DbConfig, 13 | val http: HttpConfig, 14 | val metrics: OtelMetrics.Config, 15 | ) { 16 | data class HttpConfig( 17 | val client: ClientConfig, 18 | val server: ServerConfig, 19 | ) { 20 | 21 | data class ClientConfig( 22 | val thing: ThingGateway.Config, 23 | val connectionConfig: ConnectionConfig, 24 | val retryConfig: RetryInterceptor.Config, 25 | ) 26 | 27 | data class ServerConfig( 28 | val port: Int, 29 | val host: String, 30 | ) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /libs/common/src/main/kotlin/vanillakotlin/app/AppInterfaces.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.app 2 | 3 | import org.slf4j.LoggerFactory 4 | import vanillakotlin.extensions.eventually 5 | 6 | private val log = LoggerFactory.getLogger("ReferenceApp") 7 | 8 | interface VanillaApp : AutoCloseable { 9 | fun start() 10 | } 11 | 12 | // helper function to start an application and add a shutdown hook to gracefully close it 13 | fun runApplication(block: () -> VanillaApp) = block().use { app -> 14 | log.atInfo().log { "adding shutdown hook to the application" } 15 | Runtime.getRuntime().addShutdownHook( 16 | Thread { 17 | runCatching { 18 | eventually { 19 | app.close() 20 | } 21 | }.onFailure { 22 | log.atError().log { "shutdown hook failed to complete within 10 seconds; halting the runtime." } 23 | Runtime.getRuntime().halt(1) 24 | } 25 | }, 26 | ) 27 | app.start() 28 | Thread.sleep(Long.MAX_VALUE) 29 | } 30 | -------------------------------------------------------------------------------- /apps/kafka-transformer/src/test/resources/local.conf: -------------------------------------------------------------------------------- 1 | http { 2 | server { 3 | port = 8080 4 | } 5 | 6 | client { 7 | thing { 8 | gateway { 9 | baseUrl = "http://localhost:8080" 10 | apiKey = "test-api-key" 11 | } 12 | 13 | connection { 14 | connectTimeoutMillis = 1000, 15 | readTimeoutMillis = 3000, 16 | maxConnections = 10, 17 | keepAliveDurationMinutes = 5 18 | } 19 | 20 | retry { 21 | maxAttempts = 3, 22 | initialRetryDelayMs = 1000, 23 | maxRetryDelayMs = 10000 24 | } 25 | } 26 | } 27 | } 28 | 29 | kafka { 30 | consumer { 31 | appName = "vanillakotlinkafkatransformer-test" 32 | broker = "localhost:9092" 33 | topics = ["reference-favorite-things"] 34 | group = "test-group" 35 | autoOffsetResetConfig = "earliest" 36 | } 37 | 38 | producer { 39 | broker = "localhost:9092" 40 | topic = "test-sink-topic" 41 | } 42 | } 43 | 44 | metrics { 45 | tags = { 46 | _blossom_id = "CI15247253" 47 | team = "reference" 48 | } 49 | } -------------------------------------------------------------------------------- /libs/kafka/src/main/kotlin/vanillakotlin/kafka/transformer/TransformerEventHandler.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.kafka.transformer 2 | 3 | import vanillakotlin.kafka.models.KafkaMessage 4 | import vanillakotlin.kafka.models.KafkaOutputMessage 5 | 6 | fun interface TransformerEventHandler { 7 | fun transform(kafkaMessage: KafkaMessage): TransformerMessages 8 | } 9 | 10 | data class TransformerMessage( 11 | val kafkaOutputMessage: KafkaOutputMessage, 12 | val metricTags: Map = emptyMap(), 13 | ) 14 | 15 | sealed class TransformerMessages(val messages: List>) { 16 | class Single( 17 | kafkaOutputMessage: KafkaOutputMessage, 18 | metricTags: Map = emptyMap(), 19 | ) : TransformerMessages(listOf(TransformerMessage(kafkaOutputMessage, metricTags))) 20 | 21 | class Multiple(messages: List>) : TransformerMessages(messages) 22 | 23 | class Dropped(metricTags: Map = emptyMap()) : TransformerMessages(listOf(TransformerMessage(KafkaOutputMessage(null, null), metricTags))) 24 | } 25 | -------------------------------------------------------------------------------- /libs/db/src/main/kotlin/vanillakotlin/db/JdbiConfig.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.db 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | import org.jdbi.v3.core.Jdbi 5 | import org.jdbi.v3.postgres.PostgresPlugin 6 | import vanillakotlin.db.repository.FavoriteThingMapper 7 | import vanillakotlin.db.repository.OutboxMapper 8 | import vanillakotlin.models.FavoriteThing 9 | import vanillakotlin.models.Outbox 10 | 11 | data class DbConfig( 12 | val username: String, 13 | val password: String, 14 | val databaseName: String, 15 | val host: String = "localhost", 16 | val port: Int = 5432, 17 | ) { 18 | val jdbcUrl: String 19 | get() = "jdbc:postgresql://$host:$port/$databaseName" 20 | } 21 | 22 | fun createJdbi( 23 | config: DbConfig, 24 | objectMapper: ObjectMapper, 25 | ): Jdbi { 26 | val jdbi = Jdbi.create(config.jdbcUrl, config.username, config.password) 27 | .installPlugin(PostgresPlugin()) 28 | 29 | // Register row mappers 30 | jdbi.registerRowMapper(FavoriteThing::class.java, FavoriteThingMapper()) 31 | jdbi.registerRowMapper(Outbox::class.java, OutboxMapper()) 32 | 33 | return jdbi 34 | } 35 | -------------------------------------------------------------------------------- /settings.gradle.kts: -------------------------------------------------------------------------------- 1 | rootProject.name = "vanilla-kotlin-public" 2 | include( 3 | "libs:common", 4 | "libs:db", 5 | "libs:client", 6 | "libs:metrics", 7 | "libs:kafka", 8 | "libs:rocksdb", 9 | "libs:http4k", 10 | "apps", 11 | "apps:api", 12 | "apps:bulk-inserter", 13 | "apps:kafka-transformer", 14 | "apps:outbox-processor", 15 | "db-migration", 16 | ) 17 | 18 | pluginManagement { 19 | repositories { 20 | gradlePluginPortal() 21 | } 22 | } 23 | 24 | dependencyResolutionManagement { 25 | @Suppress("UnstableApiUsage") 26 | repositories { 27 | mavenLocal() 28 | mavenCentral() 29 | } 30 | } 31 | 32 | // This processing allows us to use the submodule name for submodule build files. 33 | // Else they all need to be named build.gradle.kts, which is inconvenient when searching for the file 34 | rootProject.children.forEach { 35 | renameBuildFiles(it) 36 | } 37 | 38 | fun renameBuildFiles(descriptor: ProjectDescriptor) { 39 | descriptor.buildFileName = "${descriptor.name}.gradle.kts" 40 | descriptor.children.forEach { 41 | renameBuildFiles(it) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /libs/common/src/testFixtures/kotlin/vanillakotlin/random/Random.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.random 2 | 3 | import io.kotest.property.Arb 4 | import io.kotest.property.RandomSource 5 | import io.kotest.property.arbitrary.Codepoint 6 | import io.kotest.property.arbitrary.alphanumeric 7 | import io.kotest.property.arbitrary.byte 8 | import io.kotest.property.arbitrary.byteArray 9 | import io.kotest.property.arbitrary.int 10 | import io.kotest.property.arbitrary.long 11 | import io.kotest.property.arbitrary.string 12 | import kotlin.math.pow 13 | 14 | fun randomString(size: Int = 10) = Arb.string(size, Codepoint.alphanumeric()).sample(RandomSource.default()).value 15 | 16 | fun randomInt(range: IntRange = 1..Int.MAX_VALUE) = Arb.int(range).sample(RandomSource.default()).value 17 | 18 | fun randomLong(range: LongRange = 1L..Long.MAX_VALUE) = Arb.long(range).sample(RandomSource.default()).value 19 | 20 | fun randomByteArray(size: Int = 10) = Arb.byteArray(Arb.int(size..size), Arb.byte()).sample(RandomSource.default()).value 21 | 22 | fun randomThing() = randomInt(0 until 10.0.pow(8).toInt()).toString().padStart(8, '0') 23 | 24 | fun randomUsername() = randomString(size = 7) 25 | -------------------------------------------------------------------------------- /apps/outbox-processor/src/test/kotlin/vanillakotlin/outboxprocessor/OutboxTestFunctions.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.outboxprocessor 2 | 3 | import org.jdbi.v3.core.Jdbi 4 | import vanillakotlin.models.Outbox 5 | import vanillakotlin.random.randomByteArray 6 | import vanillakotlin.random.randomString 7 | 8 | fun getRowCount( 9 | jdbi: Jdbi, 10 | messageKey: String, 11 | ) = // you can't combine an aggregate function with skip locked, so we use a CTE. 12 | // `SKIP LOCKED` is important to avoid test race conditions. 13 | jdbi.withHandle { handle -> 14 | handle.createQuery( 15 | """ 16 | SELECT COUNT(id) FROM ( 17 | SELECT id FROM outbox WHERE message_key = :messageKey FOR UPDATE SKIP LOCKED 18 | ) count 19 | """.trimIndent(), 20 | ) 21 | .bind("messageKey", messageKey) 22 | .mapTo(Long::class.java) 23 | .single() 24 | } 25 | 26 | fun buildOutbox(messageKey: String = randomString()): Outbox = Outbox( 27 | messageKey = messageKey, 28 | headers = mapOf(randomString() to randomByteArray()), 29 | body = randomByteArray(), 30 | ) 31 | -------------------------------------------------------------------------------- /libs/kafka/src/main/kotlin/vanillakotlin/kafka/provenance/Provenance.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.kafka.provenance 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator 4 | import com.fasterxml.jackson.annotation.JsonProperty 5 | import java.time.Instant 6 | import java.util.UUID 7 | 8 | const val SPAN_ID_HEADER_NAME = "span_id" 9 | const val PROVENANCES_HEADER_NAME = "provenances" 10 | 11 | /** 12 | * Represents the provenance of a message. 13 | * @property spanId Unique identifier for the provenance if provided. e.g. 68B20473-0CE5-4650-A0EE-EB4477EAAE67 14 | * @property timestamp Source timestamp, ideally the business event timestamp if it exists, or the message creation timestamp if no 15 | * business timestamp exists. e.g. 1622464871000 16 | * @property entity the entity that this provenance is associated with. e.g. a Kafka message coordinate, or a DB row coordinate 17 | */ 18 | data class Provenance @JsonCreator constructor( 19 | @JsonProperty("spanId") val spanId: String? = null, 20 | @JsonProperty("timestamp") val timestamp: Instant, 21 | @JsonProperty("entity") val entity: String, 22 | ) 23 | 24 | fun generateSpanId(): String = UUID.randomUUID().toString() 25 | -------------------------------------------------------------------------------- /libs/common/src/test/kotlin/vanillakotlin/domain/OutboxTest.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.domain 2 | 3 | import io.kotest.matchers.equals.shouldBeEqual 4 | import org.junit.jupiter.api.Test 5 | import vanillakotlin.models.Outbox 6 | import vanillakotlin.random.randomByteArray 7 | import vanillakotlin.random.randomString 8 | 9 | fun randomOutbox(): Outbox = Outbox( 10 | messageKey = randomString(), 11 | headers = mapOf("header1" to randomByteArray(), "header2" to randomByteArray()), 12 | body = randomByteArray(), 13 | ) 14 | 15 | class OutboxTest { 16 | 17 | @Test fun equals() { 18 | val outbox = randomOutbox() 19 | 20 | val other = outbox.copy( 21 | headers = outbox.headers.mapValues { (_, v) -> v.copyOf() }, 22 | body = outbox.body?.copyOf(), 23 | ) 24 | 25 | outbox shouldBeEqual other 26 | } 27 | 28 | @Test fun hashcode() { 29 | val outbox = randomOutbox() 30 | 31 | val other = outbox.copy( 32 | headers = outbox.headers.mapValues { (_, v) -> v.copyOf() }, 33 | body = outbox.body?.copyOf(), 34 | ) 35 | 36 | outbox.hashCode() shouldBeEqual other.hashCode() 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /apps/api/src/test/kotlin/vanillakotlin/api/MockThingServer.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.api 2 | 3 | import okhttp3.mockwebserver.Dispatcher 4 | import okhttp3.mockwebserver.MockResponse 5 | import okhttp3.mockwebserver.MockWebServer 6 | import okhttp3.mockwebserver.RecordedRequest 7 | 8 | class MockThingServer { 9 | private val mockWebServer = MockWebServer() 10 | 11 | init { 12 | mockWebServer.dispatcher = ThingDispatcher() 13 | } 14 | 15 | private val sampleThingResponse = 16 | """ 17 | { 18 | "data": { 19 | "thing": { 20 | "id": "1", 21 | "product_name": "Test Product", 22 | "selling_price": 19.99 23 | } 24 | } 25 | } 26 | """.trimIndent() 27 | 28 | private inner class ThingDispatcher : Dispatcher() { 29 | override fun dispatch(request: RecordedRequest): MockResponse = when { 30 | request.path?.startsWith("/things/") == true -> MockResponse().setResponseCode(200).setBody(sampleThingResponse) 31 | else -> MockResponse().setResponseCode(404) 32 | } 33 | } 34 | 35 | val port: Int get() = mockWebServer.port 36 | 37 | fun close() { 38 | mockWebServer.close() 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /apps/kafka-transformer/src/main/kotlin/vanillakotlin/kafkatransformer/Config.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.kafkatransformer 2 | 3 | import vanillakotlin.http.clients.ConnectionConfig 4 | import vanillakotlin.http.clients.thing.ThingGateway 5 | import vanillakotlin.http.interceptors.RetryInterceptor 6 | import vanillakotlin.kafka.consumer.KafkaConsumer 7 | import vanillakotlin.kafka.producer.KafkaProducer 8 | import vanillakotlin.metrics.OtelMetrics 9 | 10 | // see docs/configuration.md for more details 11 | 12 | data class Config( 13 | val http: HttpConfig, 14 | val metrics: OtelMetrics.Config, 15 | val kafka: KafkaConfig, 16 | ) { 17 | data class HttpConfig( 18 | val server: HttpServerConfig, 19 | val client: ClientConfig, 20 | ) { 21 | data class HttpServerConfig(val port: Int) 22 | 23 | data class ClientConfig(val thing: ThingConfig) { 24 | data class ThingConfig( 25 | val gateway: ThingGateway.Config, 26 | val connection: ConnectionConfig, 27 | val retry: RetryInterceptor.Config, 28 | ) 29 | } 30 | } 31 | 32 | data class KafkaConfig( 33 | val consumer: KafkaConsumer.Config, 34 | val producer: KafkaProducer.Config, 35 | ) 36 | } 37 | -------------------------------------------------------------------------------- /libs/kafka/src/main/kotlin/vanillakotlin/kafka/Metrics.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.kafka 2 | 3 | // metric names 4 | internal const val KAFKA_SEND_METRIC = "kafka.send" 5 | internal const val KAFKA_PROCESS_TIMER_METRIC = "kafka.process.time" 6 | internal const val LAG_METRIC = "kafka.topic.consumer.lag" 7 | internal const val KAFKA_WORKER_CHANNEL_SIZE_METRIC = "kafka.worker.channel" 8 | internal const val KAFKA_PUBLISHER_CHANNEL_SIZE_METRIC = "kafka.publisher.channel" 9 | internal const val KAFKA_ACK_CHANNEL_SIZE_METRIC = "kafka.ack.channel" 10 | internal const val KAFKA_POLLER_TIMER_METRIC = "kafka.poller.time" 11 | internal const val KAFKA_PUBLISHER_TIMER_METRIC = "kafka.publisher.time" 12 | internal const val KAFKA_ACK_TIMER_METRIC = "kafka.ack.time" 13 | internal const val TRANSFORMER_EVENT = "transformer.event" 14 | internal const val AGE_METRIC = "kafka.message.age" 15 | 16 | // tag names 17 | internal const val APP_TAG = "app" 18 | internal const val TOPIC_TAG = "topic" 19 | internal const val PARTITION_TAG = "partition" 20 | internal const val SUCCESS_TAG = "success" 21 | internal const val STATUS_TAG = "status" 22 | internal const val FORWARDED_TAG = "forwarded" 23 | internal const val DROPPED_TAG = "dropped" 24 | internal const val WORKER_TAG = "worker" 25 | internal const val SKIPPED_TAG = "skipped" 26 | -------------------------------------------------------------------------------- /libs/common/src/main/kotlin/vanillakotlin/extensions/TimeLimiterExtensions.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.extensions 2 | 3 | import io.github.resilience4j.timelimiter.TimeLimiter 4 | import java.lang.Thread.sleep 5 | import java.time.Duration 6 | import java.util.concurrent.CompletableFuture 7 | 8 | inline fun attempt( 9 | duration: Duration = 10 | Duration.ofSeconds( 11 | 10L, 12 | ), 13 | crossinline block: () -> R, 14 | ): R = TimeLimiter.of(duration).attempt(block) 15 | 16 | inline fun TimeLimiter.attempt(crossinline block: () -> R): R = executeFutureSupplier { CompletableFuture.supplyAsync { block() } } 17 | 18 | // mostly useful in tests, the thing is expected to fail sometimes (like grabbing a port) but we want to retry 19 | inline fun eventually( 20 | duration: Duration = Duration.ofSeconds(10), 21 | delay: Long = 1, 22 | crossinline block: () -> R, 23 | ): R = attempt(duration) { untilErrorFree(block, delay) } 24 | 25 | inline fun untilErrorFree( 26 | crossinline block: () -> R, 27 | delay: Long, 28 | ): R { 29 | while (true) { 30 | try { 31 | return block() 32 | } catch (ex: Throwable) { 33 | // errors are "expected" just sleep a bit 34 | sleep(delay) 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /libs/kafka/src/main/kotlin/vanillakotlin/kafka/producer/PartitionCalculator.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.kafka.producer 2 | 3 | import org.apache.kafka.common.utils.Utils 4 | 5 | /** 6 | * Helper class for calculating which partition to put a message based on the partition key. 7 | * This is to be used when an application would like to separate out the message key from the partition key. 8 | * For non-compacted topics, a message key is usually all that is required to send a message, however for 9 | * compacted topics you will likely have a use-case where you want all messages for a specific dataset 10 | * (user, account, store, etc...) on the same partition, but because of the compaction need a unique key 11 | * for the message. 12 | * 13 | * If an app has that requirement, there is an optional `partition` on the KafkaOutput message where 14 | * this function could be used: 15 | * 16 | * KafkaOutputMessage( 17 | * key = "uniqueKey", 18 | * value = "value", 19 | * partition = PartitionCalculator.partitionFor("partitionKey", producer.partitionCount) 20 | * ) 21 | */ 22 | class PartitionCalculator { 23 | companion object { 24 | fun partitionFor( 25 | partitionKey: String?, 26 | partitionCount: Int, 27 | ): Int? = partitionKey?.let { Utils.toPositive(Utils.murmur2(it.toByteArray())) % partitionCount } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /libs/common/src/main/kotlin/vanillakotlin/models/HealthCheck.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.models 2 | 3 | import kotlinx.coroutines.Dispatchers 4 | import kotlinx.coroutines.async 5 | import kotlinx.coroutines.awaitAll 6 | import kotlinx.coroutines.runBlocking 7 | import kotlin.coroutines.CoroutineContext 8 | 9 | interface HealthMonitor { 10 | val name: String 11 | fun check(): HealthCheckResponse 12 | } 13 | 14 | data class HealthCheckResponse( 15 | val name: String, 16 | val isHealthy: Boolean, 17 | val details: String, 18 | ) 19 | 20 | fun healthCheckAll( 21 | healthMonitors: List, 22 | coroutineContext: CoroutineContext = Dispatchers.IO, 23 | ): List = runBlocking(context = coroutineContext) { 24 | healthMonitors.map { monitor -> 25 | async { 26 | try { 27 | monitor.check() 28 | } catch (t: Throwable) { 29 | HealthCheckResponse( 30 | name = monitor.name, 31 | isHealthy = false, 32 | details = "${t::class.qualifiedName} - ${t.message}", 33 | ) 34 | } 35 | } 36 | }.awaitAll() 37 | } 38 | 39 | fun allFailedChecks( 40 | healthMonitors: List, 41 | coroutineContext: CoroutineContext = Dispatchers.IO, 42 | ): List = healthCheckAll(healthMonitors, coroutineContext).filter { !it.isHealthy } 43 | -------------------------------------------------------------------------------- /libs/client/src/main/kotlin/vanillakotlin/http/interceptors/TelemetryInterceptor.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.http.interceptors 2 | 3 | import okhttp3.Interceptor 4 | import okhttp3.Request 5 | import okhttp3.Response 6 | import org.slf4j.LoggerFactory 7 | import vanillakotlin.metrics.PublishTimerMetric 8 | import kotlin.time.Duration.Companion.milliseconds 9 | 10 | class TelemetryInterceptor(private val publishTimerMetric: PublishTimerMetric) : Interceptor { 11 | private val log = LoggerFactory.getLogger(javaClass) 12 | 13 | override fun intercept(chain: Interceptor.Chain): Response { 14 | val request = chain.request() 15 | val startTimeMillis = System.currentTimeMillis() 16 | 17 | val response = try { 18 | chain.proceed(request) 19 | } catch (e: Throwable) { 20 | log.warn("http client call exception occurred", e) 21 | 22 | publishTimerMetric( 23 | "http.request", 24 | mapOf("status" to e.javaClass.simpleName, "source" to "remote") + request.metricTags, 25 | (System.currentTimeMillis() - startTimeMillis).milliseconds, 26 | ) 27 | throw e 28 | } 29 | 30 | publishTimerMetric( 31 | "http.request", 32 | mapOf("status" to response.code.toString(), "source" to "remote") + request.metricTags, 33 | (System.currentTimeMillis() - startTimeMillis).milliseconds, 34 | ) 35 | 36 | return response 37 | } 38 | 39 | private val Request.metricTags: Map 40 | get() = this.tag(TelemetryTag::class.java)?.metricTags ?: emptyMap() 41 | } 42 | -------------------------------------------------------------------------------- /apps/api/src/test/kotlin/vanillakotlin/api/ApiTestUtilities.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.api 2 | 3 | import org.http4k.contract.ContractRoute 4 | import org.http4k.contract.openapi.ApiInfo 5 | import org.http4k.core.HttpHandler 6 | import org.http4k.core.Request 7 | import vanillakotlin.http4k.CorsMode 8 | import vanillakotlin.http4k.buildServerHandler 9 | import vanillakotlin.models.HealthMonitor 10 | import vanillakotlin.random.randomUsername 11 | 12 | // Utility to build a server handler for testing. 13 | // This builds the same HTTP server used in the real app, using a simpler 14 | // interface for testing by allowing an optional list of healthMonitors and an 15 | // optional single route parameter. 16 | fun buildTestHandler( 17 | healthMonitors: List = emptyList(), 18 | contractRoute: ContractRoute? = null, 19 | ): HttpHandler = buildServerHandler(port = 0) { 20 | healthMonitors { +healthMonitors } 21 | corsMode = CorsMode.ALLOW_ALL 22 | 23 | if (contractRoute != null) { 24 | contract { 25 | apiInfo = ApiInfo("Test API", "1.0") 26 | routes { 27 | +contractRoute 28 | } 29 | } 30 | } 31 | } 32 | 33 | // Convenience function for tests to append authentication headers to requests 34 | fun Request.addAuth(username: String = randomUsername()): Request = this 35 | .header("X-ID", username) 36 | .header("X-EMAIL", "$username@host.com") 37 | .header("X-MEMBER-OF", "standard-user") 38 | 39 | fun Request.addAdminAuth(username: String = randomUsername()): Request = this 40 | .header("X-ID", username) 41 | .header("X-EMAIL", "$username@host.com") 42 | .header("X-MEMBER-OF", "admin-user") 43 | -------------------------------------------------------------------------------- /libs/rocksdb/src/main/kotlin/vanillakotlin/rocksdb/core/Disposable.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.rocksdb.core 2 | 3 | import java.io.Closeable 4 | import java.util.concurrent.atomic.AtomicBoolean 5 | 6 | // This allows for cleanup (ie. close) logic to be implemented 7 | // separately from a related value 8 | interface Disposable : Closeable { 9 | companion object { 10 | fun from(value: T): Disposable = NoOpDisposable(value) 11 | 12 | fun from( 13 | value: T, 14 | onClose: () -> Unit, 15 | ): Disposable = WrappedDisposable(value, onClose) 16 | 17 | fun from(value: T) = from(value) { value.close() } 18 | 19 | fun from(value: T) = from(value) { value.close() } 20 | } 21 | 22 | fun value(): T 23 | } 24 | 25 | private sealed class SafeDisposable : Disposable { 26 | private val hasClosed = AtomicBoolean(false) 27 | 28 | protected fun isClosed() = hasClosed.get() 29 | 30 | protected fun safeClose(block: () -> Unit) { 31 | val shouldClose = hasClosed.compareAndSet(false, true) 32 | 33 | if (shouldClose) { 34 | block() 35 | } 36 | } 37 | } 38 | 39 | private class WrappedDisposable( 40 | private val wrapped: T, 41 | private val onClose: () -> Unit, 42 | ) : SafeDisposable() { 43 | override fun value() = wrapped 44 | 45 | override fun close() = safeClose { onClose() } 46 | } 47 | 48 | private class NoOpDisposable(private val wrapped: T) : Disposable { 49 | override fun value() = wrapped 50 | 51 | override fun close() = Unit 52 | } 53 | 54 | fun Disposable.map(transform: (T) -> R): Disposable = Disposable.from(transform(value())) { close() } 55 | -------------------------------------------------------------------------------- /libs/client/src/main/kotlin/vanillakotlin/http/interceptors/Models.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.http.interceptors 2 | 3 | /** 4 | * Tag used for caching. Everything here will be used to generate a cache key. 5 | */ 6 | data class CacheTag( 7 | val context: String, 8 | val key: String, 9 | ) { 10 | fun cacheKey(): String = "$context:$key" 11 | } 12 | 13 | /** 14 | * Tag used for metrics and logging. Everything here might be logged; only service and endpoint will be influx'd. 15 | * @param service The top-level api name being called. e.g for /digital_items/v1/lite/ the service might be "digital_items" 16 | * @param endpoint The low-level api name being called. e.g. "lite" using the above example. 17 | */ 18 | data class TelemetryTag( 19 | val service: String, 20 | val endpoint: String, 21 | val logMap: Map = emptyMap(), 22 | val metricTags: Map = mapOf("service" to service, "endpoint" to endpoint), 23 | ) { 24 | init { 25 | (listOf(service, endpoint) + metricTags.values).forEach { tag -> 26 | check(tag == tag.cleanedForMetrics()) { "invalid metrics tag: $service" } 27 | } 28 | } 29 | 30 | // https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/#special-characters-and-keywords 31 | // we take it one step farther and strip out anything that's not alphanumeric, '.', '/', '-', or '_' 32 | private fun String.cleanedForMetrics(): String = replace(Regex("[^a-zA-Z0-9._\\-/]"), "_") 33 | } 34 | 35 | /** 36 | * Build this up with each request to provide metadata used to control the request behavior, and add metrics/logging identifiers 37 | */ 38 | data class RequestContext( 39 | val telemetryTag: TelemetryTag, 40 | val cacheTag: CacheTag? = null, 41 | ) 42 | -------------------------------------------------------------------------------- /libs/common/src/test/kotlin/vanillakotlin/extensions/TimeLimiterExtensionsTest.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.extensions 2 | 3 | import io.github.resilience4j.timelimiter.TimeLimiter 4 | import io.kotest.assertions.throwables.shouldThrowExactly 5 | import io.kotest.matchers.comparables.shouldBeLessThan 6 | import io.kotest.matchers.shouldBe 7 | import org.junit.jupiter.api.Test 8 | import java.util.concurrent.TimeoutException 9 | import kotlin.system.measureTimeMillis 10 | 11 | private const val SUCCESS: Int = 1 12 | const val TEN_MILLIS = 10L 13 | const val TEN_SECONDS = 10_000L 14 | 15 | fun Long.sleep() = Thread.sleep(this) 16 | 17 | class TimeLimiterExtensionsTest { 18 | private fun takesTenSeconds(): Int { 19 | TEN_SECONDS.sleep() 20 | return SUCCESS 21 | } 22 | 23 | private fun takesTenMillis(): Int { 24 | TEN_MILLIS.sleep() 25 | return SUCCESS 26 | } 27 | 28 | @Test fun `time limiter should timeout if it takes too long`() { 29 | measureTimeMillis { 30 | shouldThrowExactly { 31 | val timeLimiter: TimeLimiter = TimeLimiter.of(10.milliseconds) 32 | timeLimiter.attempt { takesTenSeconds() } 33 | } 34 | } shouldBeLessThan TEN_SECONDS 35 | } 36 | 37 | @Test fun `time limiter should return value if quick enough`() { 38 | measureTimeMillis { 39 | val timeLimiter: TimeLimiter = TimeLimiter.of(10.seconds) 40 | timeLimiter.attempt { takesTenMillis() } shouldBe SUCCESS 41 | } shouldBeLessThan TEN_SECONDS 42 | } 43 | 44 | @Test fun `attempt with defaults succeeds`() { 45 | measureTimeMillis { 46 | attempt { takesTenMillis() } shouldBe SUCCESS 47 | } shouldBeLessThan TEN_SECONDS 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /apps/api/src/test/kotlin/vanillakotlin/api/ConfigTest.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.api 2 | 3 | import io.kotest.extensions.system.withEnvironment 4 | import io.kotest.extensions.system.withSystemProperty 5 | import io.kotest.matchers.shouldBe 6 | import org.junit.jupiter.api.Test 7 | import vanillakotlin.config.loadConfig 8 | import vanillakotlin.random.randomString 9 | 10 | // configuration can be both strongly typed and tested 11 | class ConfigTest { 12 | 13 | @Test fun `default config`() { 14 | withEnvironment("CI", "true") { 15 | val config = loadConfig() 16 | config.metrics.tags["team"] shouldBe "test" // default value in the default.conf file 17 | } 18 | } 19 | 20 | @Test fun `system property config override`() { 21 | val expectedValue = randomString() 22 | // hoplite's environment override capability has two notable features: 23 | // 1. it uses a `config.override.` prefix to avoid collisions with existing environment variables whose names might match 24 | // the application's variable names. e.g. in TAP this can happen with `METRICS_URI` (and did happen previously) 25 | // 2. it doesn't do any case translation. this prevents problems around naming convention translation. 26 | // see the hoplite docs for more details 27 | withSystemProperty("config.override.db.host", expectedValue) { 28 | val config = loadConfig() 29 | config.db.host shouldBe expectedValue 30 | } 31 | } 32 | 33 | @Test fun `environment property config override`() { 34 | val expectedValue = randomString() 35 | withEnvironment("config.override.db.host", expectedValue) { 36 | val config = loadConfig() 37 | config.db.host shouldBe expectedValue 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /libs/client/src/main/kotlin/vanillakotlin/http/interceptors/RetryInterceptor.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.http.interceptors 2 | 3 | import io.github.resilience4j.core.IntervalFunction 4 | import io.github.resilience4j.kotlin.retry.RetryConfig 5 | import io.github.resilience4j.kotlin.retry.RetryRegistry 6 | import io.github.resilience4j.retry.Retry 7 | import okhttp3.Interceptor 8 | import okhttp3.Response 9 | import vanillakotlin.metrics.PublishCounterMetric 10 | 11 | class RetryInterceptor( 12 | private val config: Config, 13 | private val publishCounterMetric: PublishCounterMetric, 14 | ) : Interceptor { 15 | data class Config( 16 | val maxAttempts: Int = 3, 17 | val initialRetryDelayMs: Long = 1000L, 18 | val maxRetryDelayMs: Long = 10000L, 19 | val retryNonErrorCodes: Set = emptySet(), 20 | ) 21 | 22 | private val retrySupplier = RetryConfig { 23 | maxAttempts(config.maxAttempts) 24 | intervalFunction(IntervalFunction.ofExponentialBackoff(config.initialRetryDelayMs, 2.0, config.maxRetryDelayMs)) 25 | retryOnResult { response -> with(response) { code >= 500 || code in config.retryNonErrorCodes } } 26 | consumeResultBeforeRetryAttempt { _, response -> 27 | val telemetryTag = response.request.tag(TelemetryTag::class.java) 28 | if (telemetryTag != null) { 29 | publishCounterMetric("http.request.retry.count", telemetryTag.metricTags) 30 | } 31 | response.close() 32 | } 33 | }.let { retryConfig -> 34 | RetryRegistry { withRetryConfig(retryConfig) }.retry("okhttp") 35 | }.let { retry -> 36 | Retry.decorateCheckedFunction(retry) { chain: Interceptor.Chain -> with(chain) { proceed(request()) } } 37 | } 38 | 39 | override fun intercept(chain: Interceptor.Chain): Response = retrySupplier.apply(chain) 40 | } 41 | -------------------------------------------------------------------------------- /libs/rocksdb/src/main/kotlin/vanillakotlin/rocksdb/core/RocksDbColumnFamilyCompressionOptions.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.rocksdb.core 2 | 3 | import org.rocksdb.CompressionOptions 4 | import org.rocksdb.CompressionType 5 | import org.rocksdb.Options 6 | 7 | sealed class RocksDbColumnFamilyCompressionOptions { 8 | class Global : RocksDbColumnFamilyCompressionOptions() { 9 | var type: String = "lz4" 10 | } 11 | 12 | class PerLevel : RocksDbColumnFamilyCompressionOptions() { 13 | var defaultType: String = "lz4" 14 | val types = mutableListOf() 15 | } 16 | 17 | var level: Int = 4 18 | var dictionaryBytesKb: Int? = null 19 | var maxTrainBytesKb: Int? = null 20 | 21 | fun configureCompressionOptions(options: Options) { 22 | when (this) { 23 | is Global -> { 24 | val compressionType = CompressionType.getCompressionType(type) 25 | options.setCompressionType(compressionType) 26 | } 27 | is PerLevel -> { 28 | val compressionTypes = 29 | (0..options.numLevels()) 30 | .map { types.elementAtOrNull(it) ?: defaultType } 31 | .map { CompressionType.getCompressionType(it) } 32 | .toList() 33 | 34 | options.setCompressionPerLevel(compressionTypes) 35 | } 36 | } 37 | 38 | options.setCompressionOptions( 39 | CompressionOptions() 40 | .setLevel(level) 41 | .apply { 42 | val bytesKb = dictionaryBytesKb 43 | 44 | if (bytesKb != null) { 45 | setMaxDictBytes(bytesKb * 1024) 46 | } 47 | 48 | val maxTrainBytesKb = maxTrainBytesKb 49 | if (maxTrainBytesKb != null) { 50 | setZStdMaxTrainBytes(maxTrainBytesKb * 1024) 51 | } 52 | }, 53 | ) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /libs/db/src/test/kotlin/vanillakotlin/db/repository/OutboxRepositoryTest.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.db.repository 2 | 3 | import io.kotest.assertions.assertSoftly 4 | import io.kotest.matchers.shouldBe 5 | import io.kotest.matchers.shouldNotBe 6 | import org.junit.jupiter.api.Test 7 | import vanillakotlin.models.Outbox 8 | import vanillakotlin.random.randomByteArray 9 | import vanillakotlin.random.randomString 10 | 11 | class OutboxRepositoryTest { 12 | private val jdbi by lazy { buildTestDb() } 13 | 14 | @Test fun `pop outbox`() { 15 | // we'll use the same message key for our test messages to uniquely identify them within other possible test data 16 | // that might exist 17 | val messageKey1 = randomString() 18 | val messageKey2 = randomString() 19 | val testHeaders = mapOf(randomString() to randomByteArray()) 20 | val testBody = randomByteArray() 21 | val outbox = 22 | Outbox( 23 | messageKey = messageKey1, 24 | headers = testHeaders, 25 | body = testBody, 26 | ) 27 | 28 | jdbi.inTransaction { handle -> 29 | insertOutbox(handle, outbox) 30 | insertOutbox(handle, outbox.copy(messageKey = messageKey2)) 31 | } 32 | 33 | // pop all the messages off the outbox and find the one we're interested in 34 | jdbi.inTransaction { handle -> 35 | val outboxes = popOutbox(handle).filter { it.messageKey in listOf(messageKey1, messageKey2) } 36 | outboxes.size shouldBe 2 37 | assertSoftly(outboxes.first()) { 38 | messageKey shouldBe messageKey1 39 | createdTs shouldNotBe null 40 | headers shouldBe testHeaders 41 | body shouldBe testBody 42 | } 43 | } 44 | 45 | // there shouldn't be any more messages in the outbox 46 | jdbi.inTransaction { handle -> 47 | popOutbox(handle).size shouldBe 0 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /libs/client/src/main/kotlin/vanillakotlin/http/clients/thing/ThingGateway.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.http.clients.thing 2 | 3 | import com.fasterxml.jackson.module.kotlin.readValue 4 | import okhttp3.HttpUrl.Companion.toHttpUrl 5 | import okhttp3.OkHttpClient 6 | import okhttp3.Request 7 | import vanillakotlin.http.interceptors.CacheTag 8 | import vanillakotlin.http.interceptors.TelemetryTag 9 | import vanillakotlin.models.Thing 10 | import vanillakotlin.models.ThingIdentifier 11 | import vanillakotlin.serde.mapper 12 | 13 | /** 14 | * using a SAM interface here allows for easier testing, especially of the FavoriteThingsService coroutines activity 15 | * 16 | * SAM interface for [ThingGateway.getThingDetails] 17 | */ 18 | fun interface GetThingDetails { 19 | operator fun invoke(thingIdentifier: ThingIdentifier): Thing? 20 | } 21 | 22 | class ThingGateway( 23 | private val httpClient: OkHttpClient, 24 | private val config: Config, 25 | ) { 26 | data class Config( 27 | val apiKey: String, 28 | val baseUrl: String, 29 | ) 30 | 31 | fun getThingDetails(thingIdentifier: ThingIdentifier): Thing? { 32 | val url = config.baseUrl.toHttpUrl().newBuilder() 33 | .addPathSegments("things/v4/graphql/compact/thing") 34 | .addQueryParameter("thing", thingIdentifier).build() 35 | 36 | val request = Request.Builder() 37 | .url(url) 38 | .addHeader("Authorization", "Bearer ${config.apiKey}") 39 | .tag(CacheTag::class.java, CacheTag(context = "thingDetails", key = thingIdentifier)) 40 | .tag(TelemetryTag::class.java, TelemetryTag(service = "thing", endpoint = "details")) 41 | .build() 42 | 43 | val response = httpClient.newCall(request).execute() 44 | response.use { resp -> 45 | if (!resp.isSuccessful) return null 46 | val body = resp.body ?: return null 47 | val bodyBytes = body.bytes() 48 | if (bodyBytes.isEmpty()) return null 49 | return mapper.readValue(bodyBytes).toThing() 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /libs/kafka/src/main/kotlin/vanillakotlin/kafka/models/Models.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.kafka.models 2 | 3 | import org.apache.kafka.clients.consumer.OffsetAndMetadata 4 | import org.apache.kafka.common.TopicPartition 5 | import vanillakotlin.kafka.provenance.Provenance 6 | import vanillakotlin.kafka.provenance.SPAN_ID_HEADER_NAME 7 | import java.time.Instant 8 | 9 | typealias Partition = Int 10 | typealias Offset = Long 11 | 12 | data class TopicPartitionOffset( 13 | val topic: String, 14 | val partition: Int, 15 | val offset: Long, 16 | ) { 17 | fun getTopicPartition() = TopicPartition(topic, partition) 18 | 19 | fun getOffsetAndMetadata() = OffsetAndMetadata(offset) 20 | } 21 | 22 | fun interface KafkaConsumerSequenceHandler { 23 | fun processSequence(messages: Sequence) 24 | } 25 | 26 | data class KafkaMessage( 27 | val broker: String, 28 | val topic: String, 29 | val key: String?, 30 | val partition: Partition, 31 | val offset: Offset, 32 | val headers: Map, 33 | val timestamp: Long, 34 | val body: ByteArray?, 35 | val endOfBatch: Boolean = false, 36 | ) { 37 | fun buildProvenance(): Provenance = Provenance( 38 | spanId = headers[SPAN_ID_HEADER_NAME]?.let { String(it) }, 39 | timestamp = Instant.ofEpochMilli(timestamp), 40 | entity = "kafka://${broker.replace(Regex("(:9092|:9093)"), "")}/$topic/$partition/$offset", 41 | ) 42 | } 43 | 44 | data class KafkaOutputMessage( 45 | val key: String?, 46 | val value: V?, 47 | // You can provide a list of provenances to be attached to the message as headers instead of manually populating them in the headers 48 | // the provenances are mutable to allow adding more provenances after the message is created (used in the transformer) 49 | val provenances: MutableList = mutableListOf(), 50 | val headers: Map = emptyMap(), 51 | // if no partition or partitionKey is provided, the default partition will be used based on the message key 52 | val partition: Int? = null, 53 | val partitionKey: String? = null, 54 | ) 55 | -------------------------------------------------------------------------------- /apps/bulk-inserter/src/main/kotlin/vanillakotlin/bulkinserter/App.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.bulkinserter 2 | 3 | import org.slf4j.LoggerFactory 4 | import vanillakotlin.app.VanillaApp 5 | import vanillakotlin.app.runApplication 6 | import vanillakotlin.config.loadConfig 7 | import vanillakotlin.db.createJdbi 8 | import vanillakotlin.db.repository.FavoriteThingRepository 9 | import vanillakotlin.http4k.buildServer 10 | import vanillakotlin.kafka.consumer.KafkaConsumer 11 | import vanillakotlin.models.HealthMonitor 12 | import vanillakotlin.serde.mapper 13 | 14 | class App : VanillaApp { 15 | private val log = LoggerFactory.getLogger(javaClass) 16 | 17 | private val config = loadConfig() 18 | 19 | // internal so we can leverage it in tests 20 | internal val repository = FavoriteThingRepository(createJdbi(config.db, mapper)) 21 | 22 | private val kafkaConsumer = KafkaConsumer( 23 | config = config.kafka.consumer, 24 | eventHandler = BulkInserterHandler( 25 | addToBatch = repository.addToBatch, 26 | runBatch = repository.runBatch, 27 | ), 28 | ) 29 | 30 | private val healthMonitors: List = emptyList() 31 | 32 | // since this app doesn't provide any APIs other than a health endpoint, use a more convenient function to build an httpserver 33 | private val httpServer = buildServer(port = config.http.server.port) { 34 | healthMonitors { +healthMonitors } 35 | } 36 | 37 | // httpServerPort is used for testing 38 | val httpServerPort: Int 39 | get() = httpServer.port() 40 | 41 | override fun start() { 42 | // in this function we start up all the processes that are needed for the application. 43 | // they should all be non-blocking / daemons 44 | log.atInfo().log("Starting app") 45 | kafkaConsumer.start() 46 | httpServer.start() 47 | log.atInfo().log("Started app") 48 | } 49 | 50 | override fun close() { 51 | log.atInfo().log("Closing app") 52 | kafkaConsumer.stop() 53 | httpServer.stop() 54 | log.atInfo().log("Closed app") 55 | } 56 | } 57 | 58 | fun main() = runApplication { App() } 59 | -------------------------------------------------------------------------------- /libs/common/src/main/kotlin/vanillakotlin/models/Outbox.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.models 2 | 3 | import java.time.Instant 4 | 5 | data class Outbox( 6 | val id: Long? = null, 7 | val messageKey: String, 8 | val headers: Map = emptyMap(), 9 | val body: ByteArray? = null, 10 | val createdTs: Instant? = null, 11 | ) { 12 | // `==` and `!=` only performs referential equality checks for ByteArray rather than structural. 13 | // This is a consequence of the fact that the JVM implements `.equals()` for arrays differently than other collections. 14 | // Custom equals and hashcode implementations are required here to ensure duplicates are handled correctly. 15 | override fun equals(other: Any?): Boolean { 16 | if (this === other) return true 17 | if (javaClass != other?.javaClass) return false 18 | 19 | other as Outbox 20 | 21 | if (id != other.id) return false 22 | if (messageKey != other.messageKey) return false 23 | if (!headers.contentEquals(other.headers)) return false 24 | if (body != null) { 25 | if (other.body == null) return false 26 | if (!body.contentEquals(other.body)) return false 27 | } else if (other.body != null) { 28 | return false 29 | } 30 | if (createdTs != other.createdTs) return false 31 | 32 | return true 33 | } 34 | 35 | override fun hashCode(): Int { 36 | var result = id?.hashCode() ?: 0 37 | result = 31 * result + messageKey.hashCode() 38 | result = 31 * result + (headers.contentHashCode() ?: 0) 39 | result = 31 * result + (body?.contentHashCode() ?: 0) 40 | result = 31 * result + (createdTs?.hashCode() ?: 0) 41 | return result 42 | } 43 | 44 | // these perform structural equals and hashcode for maps containing ByteArray values 45 | private fun Map.contentEquals(other: Map): Boolean = size == other.size && entries.all { (k, v) -> other[k].contentEquals(v) } 46 | 47 | private fun Map.contentHashCode(): Int = entries.fold(0) { sum, (k, v) -> 31 * (sum + k.hashCode() + v.contentHashCode()) } 48 | } 49 | -------------------------------------------------------------------------------- /apps/kafka-transformer/src/main/resources/default.conf: -------------------------------------------------------------------------------- 1 | http { 2 | server { 3 | port = 8080 4 | } 5 | 6 | client { 7 | 8 | # the thought around having a separate config section for the thing gateway is that in the case of having multiple http 9 | # integrations, you might want to have different configurations for each. for example you may want a smaller number of pooled client 10 | # connections for one system versus another. Also, the caching configuration could be very different if you're caching a few large 11 | # objects versus a lot of smaller objects. 12 | # you may find it better to maintain a single configuration section used by multiple things though. whatever fits your situation. 13 | thing { 14 | gateway { 15 | # defaulting baseUrl to an address compatible for testing. 16 | # many apps would default this to a local stubbed address, but to make local experimentation easier, we'll default it to a working 17 | # integration location 18 | baseUrl = "https://test.com" 19 | 20 | apiKey = "2349fd95702e18a4615207b60f15a4dd5a5ffa40" 21 | } 22 | 23 | # thing gateway http client connection settings 24 | connection { 25 | connectTimeoutMillis = 1000, 26 | readTimeoutMillis = 3000, 27 | maxConnections = 10, 28 | keepAliveDurationMinutes = 5 29 | } 30 | 31 | # retry interceptor 32 | retry { 33 | maxAttempts = 3, 34 | initialRetryDelayMs = 1000, 35 | maxRetryDelayMs = 10000 36 | } 37 | 38 | # cache interceptor 39 | cache { 40 | cacheTimeout = 1h, 41 | cacheSize = 100000, 42 | cacheClientErrorCodes = [] 43 | } 44 | } 45 | 46 | } 47 | } 48 | 49 | kafka { 50 | consumer { 51 | appName = "vanillakotlinkafkatransformer" 52 | broker = "localhost:9092" 53 | topics = ["reference-favorite-things"] 54 | group = "vanillakotlin" 55 | autoOffsetResetConfig = "earliest" 56 | } 57 | 58 | producer { 59 | broker = "localhost:9092" 60 | topic = "reference-favorite-things" 61 | } 62 | } 63 | 64 | metrics { 65 | tags = { 66 | _blossom_id = "CI15247253" 67 | team = "reference" 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /libs/common/src/test/kotlin/vanillakotlin/random/RandomTest.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.random 2 | 3 | import io.kotest.assertions.assertSoftly 4 | import io.kotest.assertions.withClue 5 | import io.kotest.matchers.shouldBe 6 | import org.junit.jupiter.api.Test 7 | 8 | class RandomTest { 9 | 10 | @Test fun `randomThing should return an 8-digit string`() { 11 | randomThing().length shouldBe 8 12 | } 13 | 14 | @Test fun `random string`() { 15 | randomString(10).length shouldBe 10 16 | } 17 | 18 | @Test fun `random byte array`() { 19 | randomByteArray(10).size shouldBe 10 20 | } 21 | 22 | @Test fun `kotlin math works`() { 23 | listOf( 24 | Triple("1 * 1 should be 4", { 1 * 1 }, 4), 25 | Triple("1 * 2 should be 4", { 1 * 2 }, 4), 26 | Triple("1 * 3 should be 4", { 1 * 3 }, 4), 27 | Triple("1 * 4 should be 4", { 1 * 4 }, 4), 28 | Triple("1 * 5 should be 4", { 1 * 5 }, 4), 29 | Triple("1 * 6 should be 4", { 1 * 6 }, 4), 30 | Triple("1 * 7 should be 4", { 1 * 7 }, 4), 31 | Triple("1 * 8 should be 4", { 1 * 8 }, 4), 32 | Triple("1 * 9 should be 4", { 1 * 9 }, 4), 33 | Triple("1 * 10 should be 10", { 1 * 10 }, 10), 34 | ).forEach { (testName, block, expectedValue) -> 35 | withClue(testName) { 36 | block() shouldBe expectedValue 37 | } 38 | } 39 | } 40 | 41 | @Test fun `kotlin math works - but better`() { 42 | assertSoftly { 43 | listOf( 44 | Triple("1 * 1 should be 4", { 1 * 1 }, 4), 45 | Triple("1 * 2 should be 4", { 1 * 2 }, 4), 46 | Triple("1 * 3 should be 4", { 1 * 3 }, 4), 47 | Triple("1 * 4 should be 4", { 1 * 4 }, 4), 48 | Triple("1 * 5 should be 4", { 1 * 5 }, 4), 49 | Triple("1 * 6 should be 4", { 1 * 6 }, 4), 50 | Triple("1 * 7 should be 4", { 1 * 7 }, 4), 51 | Triple("1 * 8 should be 4", { 1 * 8 }, 4), 52 | Triple("1 * 9 should be 4", { 1 * 9 }, 4), 53 | Triple("1 * 10 should be 10", { 1 * 10 }, 10), 54 | ).forEach { (testName, block, expectedValue) -> 55 | withClue(testName) { 56 | block() shouldBe expectedValue 57 | } 58 | } 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /apps/outbox-processor/src/test/kotlin/vanillakotlin/outboxprocessor/OutboxProcessorTest.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.outboxprocessor 2 | 3 | import io.kotest.assertions.throwables.shouldThrow 4 | import io.kotest.matchers.shouldBe 5 | import org.apache.kafka.clients.producer.RecordMetadata 6 | import org.apache.kafka.common.TopicPartition 7 | import org.junit.jupiter.api.Test 8 | import vanillakotlin.db.repository.buildTestDb 9 | import vanillakotlin.db.repository.insertOutbox 10 | import vanillakotlin.random.randomInt 11 | import vanillakotlin.random.randomString 12 | import java.time.Duration 13 | import java.time.Instant 14 | import java.util.concurrent.CompletableFuture 15 | 16 | class OutboxProcessorTest { 17 | private val jdbi by lazy { buildTestDb() } 18 | 19 | @Test fun `pop and send normal path`() { 20 | val messageKey = randomString() 21 | 22 | jdbi.inTransaction { handle -> insertOutbox(handle, buildOutbox(messageKey = messageKey)) } 23 | getRowCount(jdbi, messageKey) shouldBe 1 24 | 25 | val processor = OutboxProcessor( 26 | config = OutboxProcessor.Config(pollEvery = Duration.ofSeconds(1), popMessageLimit = 100), 27 | jdbi = jdbi, 28 | kafkaSendAsync = { CompletableFuture.supplyAsync { buildTestRecordMetadata() } }, 29 | ) 30 | 31 | processor.popAndSend() 32 | 33 | getRowCount(jdbi, messageKey) shouldBe 0 34 | } 35 | 36 | @Test fun `pop and send failure should roll back the transaction`() { 37 | val messageKey = randomString() 38 | 39 | jdbi.inTransaction { handle -> insertOutbox(handle, buildOutbox(messageKey = messageKey)) } 40 | getRowCount(jdbi, messageKey) shouldBe 1 41 | 42 | val processor = OutboxProcessor( 43 | config = OutboxProcessor.Config(pollEvery = Duration.ofSeconds(1), popMessageLimit = 100), 44 | jdbi = jdbi, 45 | kafkaSendAsync = { 46 | CompletableFuture.supplyAsync { 47 | throw Exception("expected") 48 | } 49 | }, 50 | ) 51 | 52 | shouldThrow { 53 | processor.popAndSend() 54 | } 55 | 56 | getRowCount(jdbi, messageKey) shouldBe 1 57 | } 58 | 59 | private fun buildTestRecordMetadata() = RecordMetadata( 60 | TopicPartition(randomString(), 0), 61 | 0, 62 | 0, 63 | Instant.now().toEpochMilli(), 64 | randomInt(), 65 | randomInt(), 66 | ) 67 | } 68 | -------------------------------------------------------------------------------- /libs/client/src/main/kotlin/vanillakotlin/http/clients/ClientInitializer.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.http.clients 2 | 3 | import okhttp3.ConnectionPool 4 | import okhttp3.Interceptor 5 | import okhttp3.OkHttpClient 6 | import vanillakotlin.metrics.PublishGaugeMetric 7 | import java.util.concurrent.TimeUnit 8 | 9 | data class ConnectionConfig( 10 | val connectTimeoutMillis: Long, 11 | val readTimeoutMillis: Long, 12 | val maxIdleConnections: Int = 1, 13 | val maxConnections: Int, 14 | val keepAliveDurationMinutes: Long, 15 | val clientName: String = "default", 16 | ) 17 | 18 | /** 19 | * Initialize an OkHttpClient with the given configuration and interceptors. 20 | * The client will be configured with the given connection pool settings and interceptors. 21 | * The client will also publish metrics for the connection pool and dispatcher. 22 | */ 23 | fun initializeHttpClient( 24 | config: ConnectionConfig, 25 | publishGaugeMetric: PublishGaugeMetric, 26 | vararg interceptors: Interceptor, 27 | ): OkHttpClient = OkHttpClient.Builder() 28 | .readTimeout(config.readTimeoutMillis, TimeUnit.MILLISECONDS) 29 | .connectTimeout(config.connectTimeoutMillis, TimeUnit.MILLISECONDS) 30 | .connectionPool( 31 | ConnectionPool( 32 | maxIdleConnections = config.maxIdleConnections, 33 | keepAliveDuration = config.keepAliveDurationMinutes, 34 | timeUnit = TimeUnit.MINUTES, 35 | ), 36 | ) 37 | .apply { interceptors.forEach { addInterceptor(it) } } 38 | .build() 39 | .apply { 40 | dispatcher.maxRequestsPerHost = config.maxConnections 41 | dispatcher.maxRequests = config.maxConnections 42 | publishGaugeMetric("http.client.connections.total", mapOf("client" to config.clientName)) { 43 | connectionPool.connectionCount() 44 | } 45 | publishGaugeMetric("http.client.connections.active", mapOf("client" to config.clientName)) { 46 | connectionPool.connectionCount() - connectionPool.idleConnectionCount() 47 | } 48 | publishGaugeMetric("http.client.connections.idle", mapOf("client" to config.clientName)) { 49 | connectionPool.idleConnectionCount() 50 | } 51 | publishGaugeMetric("http.client.calls.running", mapOf("client" to config.clientName)) { 52 | dispatcher.runningCallsCount() 53 | } 54 | publishGaugeMetric("http.client.calls.queued", mapOf("client" to config.clientName)) { 55 | dispatcher.queuedCallsCount() 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /libs/common/src/main/kotlin/vanillakotlin/config/Configuration.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.config 2 | 3 | import com.sksamuel.hoplite.ConfigLoaderBuilder 4 | import com.sksamuel.hoplite.PropertySource 5 | import org.slf4j.LoggerFactory 6 | import java.io.File 7 | 8 | val log = LoggerFactory.getLogger("vanillakotlin.config.Configuration") 9 | 10 | /** 11 | * The sources are first-in-wins, so define them in order of highest priority descending. 12 | */ 13 | fun buildOrderedSources(): List { 14 | // See the [hoplite docs](https://github.com/sksamuel/hoplite#property-sources) on property sources that are enabled by default. 15 | // For example, you can set a `config.override.http.server.port=8090` environment variable or system property to override the 16 | // `server.port` configuration value. 17 | val sources = mutableListOf() 18 | 19 | if (System.getenv().containsKey("CI")) { 20 | // we're running in the CI environment - load the ci.conf file 21 | sources.add(PropertySource.resource("/ci.conf", optional = true)) 22 | } else if (System.getenv().containsKey("CLOUD_ENVIRONMENT")) { 23 | // we're running in a deployed environment. Load secrets and configuration specific to the environment. 24 | sources.add(PropertySource.file(File("/path/to/deployed/secret/restricted/restricted-secret.conf"), optional = true)) 25 | sources.add(PropertySource.file(File("/path/to/deployed/secret/secret.conf"), optional = true)) 26 | sources.add(PropertySource.file(File("/path/to/deployed/config/environment.conf"), optional = true)) 27 | } else { 28 | // running locally - reference a local.conf file for local development configuration overrides. 29 | sources.add(PropertySource.resource("/local.conf", optional = true)) 30 | } 31 | 32 | // finally add the default.conf file to the configuration sources 33 | sources.add(PropertySource.resource("/default.conf")) 34 | 35 | return sources 36 | } 37 | 38 | /** 39 | * Load config for the given class destination 40 | */ 41 | inline fun loadConfig(): T { 42 | val configBuilder = ConfigLoaderBuilder.default() 43 | buildOrderedSources().forEach { propertySource -> configBuilder.addSource(propertySource) } 44 | try { 45 | return configBuilder 46 | .build() 47 | .loadConfigOrThrow() 48 | } catch (t: Throwable) { 49 | // configuration errors were getting swallowed in CI, so adding some explicit logging 50 | log.atError().setCause(t).log("Error loading configuration") 51 | throw t 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /libs/rocksdb/src/main/kotlin/vanillakotlin/rocksdb/core/RocksDbStore.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.rocksdb.core 2 | 3 | import org.rocksdb.ColumnFamilyOptions 4 | import org.rocksdb.DBOptions 5 | import org.rocksdb.Options 6 | import org.rocksdb.RocksDB 7 | import org.rocksdb.Statistics 8 | import org.rocksdb.WriteBatch 9 | import org.rocksdb.WriteOptions 10 | 11 | data class KeyValuePair( 12 | val key: ByteArray, 13 | val value: ByteArray?, 14 | ) 15 | 16 | class RocksDbStore( 17 | private val dataDirectory: String, 18 | private val configureOptions: (options: Options) -> Unit = {}, 19 | private val defaultWriteOptions: WriteOptions? = null, 20 | ) : AutoCloseable { 21 | val db: RocksDB 22 | val options: Options 23 | private var writeOptions: WriteOptions 24 | 25 | init { 26 | RocksDB.loadLibrary() 27 | 28 | ColumnFamilyOptions().use { cfOpts -> 29 | options = Options(DBOptions(), cfOpts).apply { 30 | setCreateIfMissing(true) 31 | setCreateMissingColumnFamilies(true) 32 | setStatistics(Statistics()) 33 | } 34 | configureOptions(options) 35 | 36 | writeOptions = defaultWriteOptions ?: defaultWriteOptions() 37 | 38 | db = RocksDB.open(options, dataDirectory) 39 | } 40 | } 41 | 42 | private fun defaultWriteOptions(): WriteOptions = WriteOptions().apply { setDisableWAL(true) } 43 | 44 | fun write(writeBatch: WriteBatch) = db.write(writeOptions, writeBatch) 45 | 46 | fun put( 47 | key: ByteArray, 48 | value: ByteArray, 49 | ) = db.put(writeOptions, key, value) 50 | 51 | fun get(key: ByteArray): ByteArray? = db.get(key) 52 | 53 | fun delete(key: ByteArray) = db.delete(key) 54 | 55 | fun withRange( 56 | start: ByteArray, 57 | end: ByteArray, 58 | block: (sequence: Sequence) -> Unit, 59 | ) = db.newIterator().getRange(start, end).use { disposable -> 60 | block(disposable.value()) 61 | } 62 | 63 | fun withPrefixed( 64 | start: ByteArray, 65 | block: (sequence: Sequence) -> Unit, 66 | ) = db.newIterator().getPrefixed(start).use { disposable -> 67 | block(disposable.value()) 68 | } 69 | 70 | fun withSequence(block: (sequence: Sequence) -> Unit) { 71 | db.newIterator().sequence().use { disposable -> 72 | block(disposable.value()) 73 | } 74 | } 75 | 76 | fun enableAutoCompaction() { 77 | db.enableAutoCompaction(listOf(db.defaultColumnFamily)) 78 | } 79 | 80 | fun compact() { 81 | db.compactRange() 82 | } 83 | 84 | override fun close() { 85 | db.close() 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /libs/common/src/main/kotlin/vanillakotlin/serde/ObjectMapperSingleton.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.serde 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude 4 | import com.fasterxml.jackson.databind.DeserializationFeature 5 | import com.fasterxml.jackson.databind.ObjectMapper 6 | import com.fasterxml.jackson.databind.PropertyNamingStrategies 7 | import com.fasterxml.jackson.databind.SerializationFeature 8 | import com.fasterxml.jackson.datatype.jdk8.Jdk8Module 9 | import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule 10 | import com.fasterxml.jackson.module.kotlin.registerKotlinModule 11 | 12 | // Jackson's object mapper is meant to be used as a singleton 13 | // We use it all over the place, so making it a global singleton object is useful 14 | val mapper: ObjectMapper = ObjectMapper().initialize() 15 | 16 | fun ObjectMapper.initialize(): ObjectMapper = this 17 | // writing textual dates is a little more verbose, but it provides a big advantage for human readability and debugging. 18 | .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false) 19 | // This can be handy for human readability and consistency. It does add a small computational cost to serialization, but typically not 20 | // a substantial amount 21 | .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true) 22 | // This the default naming strategy that should be used for most JSON payloads at Target unless there's a reason otherwise. 23 | // For example, one exception to this rule is GraphQL, which uses a lowerCamelCase naming convention 24 | .setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE) 25 | // only include non-null properties in output is a typical preferred option. It results in a smaller payload 26 | .setDefaultPropertyInclusion(JsonInclude.Include.NON_NULL) // only include non-null properties in output. usually preferred. 27 | // not failing on unknown properties is a good default when you may want to only deserialize a subset of a JSON body 28 | // for example when you only need a few properties 29 | // it IS important to fail on unknown properties when you're working with objects for which you expect to receive 30 | // the entire object. for example configuration files or other payloads that must be completely specified. 31 | .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) // this is fine when you are ok with partial deserialization 32 | // all newer projects will set this, which sets some base minimum version feature settings 33 | .registerModule(Jdk8Module()) 34 | // enables support for date objects used in java.time, which is an extremely valuable addition 35 | .registerModule(JavaTimeModule()) 36 | // enable kotlin features and extensions - another standard feature for any kotlin project using jackson 37 | .registerKotlinModule() 38 | -------------------------------------------------------------------------------- /libs/db/src/main/kotlin/vanillakotlin/db/repository/OutboxRepository.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.db.repository 2 | 3 | import com.fasterxml.jackson.module.kotlin.readValue 4 | import org.jdbi.v3.core.Handle 5 | import org.jdbi.v3.core.mapper.RowMapper 6 | import org.jdbi.v3.core.statement.StatementContext 7 | import vanillakotlin.models.Outbox 8 | import vanillakotlin.serde.mapper 9 | import java.sql.ResultSet 10 | 11 | /** 12 | * These operations must be executed within the context of a transaction and can't be executed on their own, 13 | * so we're requiring a Handle argument (which represents an active transaction in JDBI). 14 | */ 15 | 16 | /** SAM interface for insertOutbox */ 17 | fun interface InsertOutbox { 18 | operator fun invoke(handle: Handle, outbox: Outbox): Outbox 19 | } 20 | 21 | /** SAM interface for popOutbox */ 22 | fun interface PopOutbox { 23 | operator fun invoke(handle: Handle, limit: Int): List 24 | operator fun invoke(handle: Handle): List = invoke(handle, Int.MAX_VALUE) 25 | } 26 | 27 | val insertOutbox = InsertOutbox { handle, outbox -> 28 | handle.createQuery( 29 | """ 30 | INSERT INTO outbox (message_key, headers, body) 31 | VALUES (:messageKey, :headers, :body) 32 | RETURNING * 33 | """.trimIndent(), 34 | ) 35 | .bind("messageKey", outbox.messageKey) 36 | .bind("headers", mapper.writeValueAsBytes(outbox.headers)) 37 | .bind("body", outbox.body) 38 | .mapTo(Outbox::class.java) 39 | .single() 40 | } 41 | 42 | val popOutbox = PopOutbox { handle, limit -> 43 | handle.createQuery( 44 | """ 45 | DELETE FROM outbox WHERE id = ANY ( 46 | SELECT id FROM outbox 47 | ORDER BY created_ts 48 | FOR UPDATE SKIP LOCKED 49 | LIMIT :limit 50 | ) RETURNING * 51 | """.trimIndent(), 52 | ) 53 | .bind("limit", limit) 54 | .mapTo(Outbox::class.java) 55 | .list() 56 | } 57 | 58 | // Custom row mapper for Outbox 59 | class OutboxMapper : RowMapper { 60 | override fun map( 61 | rs: ResultSet, 62 | ctx: StatementContext, 63 | ): Outbox = Outbox( 64 | id = rs.getLong("id"), 65 | messageKey = rs.getString("message_key"), 66 | headers = mapper.readValue(rs.getBytes("headers")), 67 | body = rs.getBytes("body"), 68 | createdTs = rs.getTimestamp("created_ts").toInstant(), 69 | ) 70 | } 71 | 72 | fun mapToOutbox(resultSet: ResultSet): Outbox = with(resultSet) { 73 | Outbox( 74 | id = getLong("id"), 75 | messageKey = getString("message_key"), 76 | headers = mapper.readValue(getBytes("headers")), 77 | body = getBytes("body"), 78 | createdTs = getTimestamp("created_ts").toInstant(), 79 | ) 80 | } 81 | -------------------------------------------------------------------------------- /libs/rocksdb/src/main/kotlin/vanillakotlin/rocksdb/freshfilter/FreshFilter.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.rocksdb.freshfilter 2 | 3 | import vanillakotlin.rocksdb.core.RocksDbStore 4 | import vanillakotlin.rocksdb.core.smallValueReadHeavyConfigArchetype 5 | import vanillakotlin.rocksdb.core.toByteArray 6 | import vanillakotlin.rocksdb.core.toLong 7 | import java.io.File 8 | 9 | private val nullHash = "null".toByteArray() 10 | 11 | interface FreshFilterable { 12 | val key: ByteArray 13 | val valueHash: ByteArray 14 | val timestamp: Long 15 | 16 | companion object { 17 | fun of( 18 | key: String, 19 | valueHash: ByteArray?, 20 | timestamp: Long, 21 | ): FreshFilterable = of(key.toByteArray(), valueHash, timestamp) 22 | 23 | fun of( 24 | key: ByteArray, 25 | valueHash: ByteArray?, 26 | timestamp: Long, 27 | ): FreshFilterable = DefaultFreshFilterable( 28 | key = key, 29 | valueHash = valueHash ?: nullHash, 30 | timestamp = timestamp, 31 | ) 32 | } 33 | } 34 | 35 | private class DefaultFreshFilterable( 36 | override val key: ByteArray, 37 | override val valueHash: ByteArray, 38 | override val timestamp: Long, 39 | ) : FreshFilterable 40 | 41 | enum class FilterStatus { 42 | NEW, 43 | STALE, 44 | REDUNDANT, 45 | } 46 | 47 | class FreshFilter(dataDirectory: String = "/tmp/rocks_db") : AutoCloseable { 48 | init { 49 | File(dataDirectory).deleteRecursively() 50 | File(dataDirectory).mkdirs() 51 | } 52 | 53 | private val rocksDbStore = RocksDbStore( 54 | dataDirectory = dataDirectory, 55 | configureOptions = smallValueReadHeavyConfigArchetype::configureOptions, 56 | ) 57 | 58 | fun filter(message: FreshFilterable): FilterStatus { 59 | val savedBytes = rocksDbStore.get(message.key) 60 | if (savedBytes == null) { 61 | updateStorage(message) 62 | return FilterStatus.NEW 63 | } 64 | val savedTimestamp = savedBytes.copyOfRange(0, Long.SIZE_BYTES).toLong() 65 | 66 | if (savedTimestamp > message.timestamp) { 67 | return FilterStatus.STALE 68 | } 69 | 70 | val savedValueHash = savedBytes.takeLast(savedBytes.size - Long.SIZE_BYTES).toByteArray() 71 | 72 | // Check if this is the same message (same timestamp and hash) 73 | if (savedTimestamp == message.timestamp && message.valueHash.contentEquals(savedValueHash)) { 74 | return FilterStatus.REDUNDANT 75 | } 76 | 77 | // This is a new message (newer timestamp or different hash) 78 | updateStorage(message) 79 | return FilterStatus.NEW 80 | } 81 | 82 | private fun updateStorage(message: FreshFilterable) { 83 | val timestampBytes = message.timestamp.toByteArray() 84 | val bytes = timestampBytes + message.valueHash 85 | rocksDbStore.put(message.key, bytes) 86 | } 87 | 88 | override fun close() { 89 | rocksDbStore.close() 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /apps/kafka-transformer/src/main/kotlin/vanillakotlin/kafkatransformer/App.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.kafkatransformer 2 | 3 | import org.slf4j.LoggerFactory 4 | import vanillakotlin.app.VanillaApp 5 | import vanillakotlin.app.runApplication 6 | import vanillakotlin.config.loadConfig 7 | import vanillakotlin.http.clients.initializeHttpClient 8 | import vanillakotlin.http.clients.thing.ThingGateway 9 | import vanillakotlin.http.interceptors.RetryInterceptor 10 | import vanillakotlin.http.interceptors.TelemetryInterceptor 11 | import vanillakotlin.http4k.buildServer 12 | import vanillakotlin.kafka.transformer.KafkaTransformer 13 | import vanillakotlin.metrics.OtelMetrics 14 | import vanillakotlin.models.HealthMonitor 15 | 16 | // See vanillakotlin.api.App for an annotated version of this type of class 17 | 18 | class App : VanillaApp { 19 | private val log = LoggerFactory.getLogger(javaClass) 20 | 21 | private val config = loadConfig() 22 | 23 | private val metricsPublisher = OtelMetrics(config.metrics) 24 | 25 | private val thingClient = initializeHttpClient( 26 | config = config.http.client.thing.connection, 27 | publishGaugeMetric = metricsPublisher::publishGaugeMetric, 28 | RetryInterceptor(config.http.client.thing.retry, metricsPublisher::publishCounterMetric), 29 | TelemetryInterceptor(metricsPublisher::publishTimerMetric), 30 | ) 31 | 32 | private val thingGateway = ThingGateway( 33 | httpClient = thingClient, 34 | config = config.http.client.thing.gateway, 35 | ) 36 | 37 | private val kafkaTransformer = KafkaTransformer( 38 | consumerConfig = config.kafka.consumer, 39 | producerConfig = config.kafka.producer, 40 | eventHandler = 41 | FavoriteThingsEventHandler( 42 | getThingDetails = thingGateway::getThingDetails, 43 | ), 44 | publishTimerMetric = metricsPublisher::publishTimerMetric, 45 | publishCounterMetric = metricsPublisher::publishCounterMetric, 46 | ) 47 | 48 | private val healthMonitors: List = emptyList() 49 | 50 | // since this app doesn't provide any APIs other than a health endpoint, use a more convenient function to build a httpserver 51 | private val httpServer = buildServer(port = config.http.server.port) { 52 | healthMonitors { +healthMonitors } 53 | } 54 | 55 | // httpServerPort is used for testing 56 | val httpServerPort: Int 57 | get() = httpServer.port() 58 | 59 | override fun start() { 60 | // in this function we start up all the processes that are needed for the application. 61 | // they should all be non-blocking / daemons 62 | log.atInfo().log("Starting app") 63 | kafkaTransformer.start() 64 | httpServer.start() 65 | log.atInfo().log("Started app") 66 | } 67 | 68 | override fun close() { 69 | log.atInfo().log("Closing app") 70 | kafkaTransformer.close() 71 | httpServer.stop() 72 | log.atInfo().log("Closed app") 73 | } 74 | } 75 | 76 | fun main() = runApplication { App() } 77 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /libs/rocksdb/src/main/kotlin/vanillakotlin/rocksdb/core/Extensions.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.rocksdb.core 2 | 3 | import org.rocksdb.RocksIterator 4 | import java.nio.ByteBuffer 5 | import vanillakotlin.rocksdb.core.map as disposableMap 6 | 7 | internal fun RocksIterator.getPrefixed(prefix: ByteArray): Disposable> = Disposable.from(this).disposableMap { iterator -> 8 | sequence { 9 | iterator.seek(prefix) 10 | 11 | while (iterator.isValid) { 12 | val key = iterator.key() 13 | if (!key.startsWith(prefix)) { 14 | break 15 | } 16 | 17 | yield(KeyValuePair(key, iterator.value())) 18 | iterator.next() 19 | } 20 | } 21 | } 22 | 23 | internal fun RocksIterator.getRange( 24 | startKey: ByteArray, 25 | endKey: ByteArray, 26 | ): Disposable> = Disposable.from(this).disposableMap { iterator -> 27 | sequence { 28 | iterator.seek(startKey) 29 | var key = iterator.key() 30 | 31 | while (iterator.isValid && key <= endKey) { 32 | yield(KeyValuePair(key, iterator.value())) 33 | iterator.next() 34 | key = iterator.key() 35 | } 36 | } 37 | } 38 | 39 | internal fun RocksIterator.sequence(): Disposable> = Disposable.from(this).disposableMap { iterator -> 40 | sequence { 41 | iterator.seekToFirst() 42 | 43 | while (iterator.isValid) { 44 | val key = iterator.key() 45 | yield(KeyValuePair(key, iterator.value())) 46 | iterator.next() 47 | } 48 | } 49 | } 50 | 51 | fun ByteBuffer.size() = this.remaining() 52 | 53 | fun ByteArray.startsWith(other: ByteArray): Boolean { 54 | if (other.size > this.size) { 55 | return false 56 | } 57 | 58 | for (i in other.indices) { 59 | if (this[i] != other[i]) { 60 | return false 61 | } 62 | } 63 | 64 | return true 65 | } 66 | 67 | operator fun ByteArray?.compareTo(other: ByteArray?): Int { 68 | @Suppress("ReplaceCallWithBinaryOperator") 69 | if ((this == null && other == null) || (this != null && this.equals(other))) { 70 | return 0 71 | } 72 | 73 | if (this == null) { 74 | return -1 75 | } else if (other == null) { 76 | return 1 77 | } 78 | 79 | val last = kotlin.math.min(this.size, other.size) 80 | for (i in 0 until last) { 81 | val v1 = this[i] 82 | val v2 = other[i] 83 | 84 | if (v1 == v2) { 85 | continue 86 | } 87 | 88 | val comp = v1.compareTo(v2) 89 | if (comp != 0) { 90 | return comp 91 | } 92 | } 93 | 94 | if (this.size < other.size) { 95 | return -1 96 | } else if (this.size > other.size) { 97 | return 1 98 | } 99 | 100 | return 0 101 | } 102 | 103 | fun Long.toByteArray(): ByteArray = ByteBuffer.allocate(Long.SIZE_BYTES).putLong(this).array() 104 | 105 | fun ByteArray.toLong(): Long = ByteBuffer.wrap(this).long 106 | -------------------------------------------------------------------------------- /apps/outbox-processor/src/main/kotlin/vanillakotlin/outboxprocessor/OutboxProcessor.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.outboxprocessor 2 | 3 | import org.jdbi.v3.core.Jdbi 4 | import org.slf4j.LoggerFactory 5 | import vanillakotlin.db.repository.popOutbox 6 | import vanillakotlin.kafka.models.KafkaOutputMessage 7 | import vanillakotlin.kafka.producer.KafkaSendAsync 8 | import java.time.Duration 9 | import java.util.Timer 10 | import kotlin.concurrent.timerTask 11 | import kotlin.system.exitProcess 12 | 13 | typealias UncaughtErrorHandler = (t: Throwable) -> Unit 14 | 15 | private val log = LoggerFactory.getLogger(OutboxProcessor::class.java) 16 | 17 | // Running in TAP, the default error handler is to halt the process, letting TAP restart the container. This handles occasional glitches 18 | // like broker and network errors. Any more permanent issues should trigger an alert for the engineers to investigate and fix. 19 | // By allowing a function for error handling, we can more easily test and allow different behavior for different scenarios. 20 | val shutdownErrorHandler = { throwable: Throwable -> 21 | log.atError().setCause(throwable).log("Fatal error encountered during outbox processing. Will halt.") 22 | exitProcess(1) 23 | } 24 | 25 | class OutboxProcessor( 26 | private val config: Config, 27 | private val jdbi: Jdbi, 28 | private val kafkaSendAsync: KafkaSendAsync, 29 | private val uncaughtErrorHandler: UncaughtErrorHandler = shutdownErrorHandler, 30 | ) { 31 | data class Config( 32 | val pollEvery: Duration, 33 | val popMessageLimit: Int, 34 | ) 35 | 36 | fun start() { 37 | // this timer is currently running every X seconds. it could be made to be a lot smarter, 38 | // for example if it's behind, it could poll faster, and if it's idle, it could poll more slowly 39 | Timer(true).scheduleAtFixedRate( 40 | timerTask { 41 | try { 42 | popAndSend() 43 | } catch (t: Throwable) { 44 | uncaughtErrorHandler(t) 45 | } 46 | }, 47 | 0, 48 | config.pollEvery.toMillis(), 49 | ) 50 | } 51 | 52 | /** 53 | * In a transaction, pop a set of messages from the outbox and send them all. 54 | * If the kafka send completes successfully, the transaction will commit. 55 | * If there's an exception, the transaction will rollback and throw the exception. 56 | */ 57 | fun popAndSend() { 58 | jdbi.inTransaction { handle -> 59 | val messages = popOutbox(handle, config.popMessageLimit) 60 | messages.map { message -> 61 | log.atDebug().log("Sending message") 62 | 63 | // send all the messages asynchronously 64 | kafkaSendAsync( 65 | KafkaOutputMessage( 66 | key = message.messageKey, 67 | value = message.body, 68 | headers = message.headers, 69 | ), 70 | ) 71 | }.forEach { 72 | // then wait for all the futures to complete. any errors will throw and rollback the transaction. 73 | it.join() 74 | } 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /apps/outbox-processor/src/test/kotlin/vanillakotlin/outboxprocessor/AppTest.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.outboxprocessor 2 | 3 | import io.kotest.extensions.system.withSystemProperties 4 | import io.kotest.matchers.shouldBe 5 | import org.apache.kafka.clients.admin.AdminClient 6 | import org.apache.kafka.clients.admin.NewTopic 7 | import org.http4k.client.JavaHttpClient 8 | import org.http4k.core.Method 9 | import org.http4k.core.Request 10 | import org.http4k.core.Status 11 | import org.junit.jupiter.api.AfterAll 12 | import org.junit.jupiter.api.BeforeAll 13 | import org.junit.jupiter.api.Test 14 | import org.junit.jupiter.api.TestInstance 15 | import vanillakotlin.db.repository.buildTestDb 16 | import vanillakotlin.db.repository.insertOutbox 17 | import vanillakotlin.kafka.collectMessages 18 | import vanillakotlin.kafka.models.KafkaMessage 19 | import vanillakotlin.random.randomString 20 | 21 | @TestInstance(TestInstance.Lifecycle.PER_CLASS) 22 | class AppTest { 23 | private var app: App? = null 24 | private val jdbi by lazy { buildTestDb() } 25 | 26 | private val sinkTopicName = randomString() 27 | private val broker = if (System.getenv().containsKey("CI")) "kafka:9092" else "localhost:9092" 28 | 29 | @BeforeAll fun beforeAll() { 30 | // create random sink topic 31 | val adminClient = AdminClient.create(mapOf("bootstrap.servers" to broker)) 32 | adminClient.createTopics(listOf(NewTopic(sinkTopicName, 1, 1))) 33 | 34 | // override some of the app configuration 35 | val overriddenConfiguration = 36 | mapOf( 37 | "config.override.kafka.producer.topic" to sinkTopicName, 38 | "config.override.outbox.pollEvery" to "1s", 39 | ) 40 | withSystemProperties(overriddenConfiguration) { 41 | app = App() 42 | app?.start() 43 | } 44 | } 45 | 46 | @AfterAll fun afterAll() { 47 | app?.close() 48 | } 49 | 50 | @Test fun `health check ok`() { 51 | val request = Request(Method.GET, "http://localhost:${app?.httpServerPort}/health") 52 | 53 | val response = JavaHttpClient()(request) 54 | 55 | response.status shouldBe Status.OK 56 | } 57 | 58 | @Test fun `the outbox processor should pop and send successfully`() { 59 | // the processor should be running already because the app has been started 60 | 61 | val messageKey = randomString() 62 | val bodyString = randomString() 63 | 64 | // insert an outbox item 65 | jdbi.inTransaction { handle -> insertOutbox(handle, buildOutbox(messageKey = messageKey).copy(body = bodyString.toByteArray())) } 66 | 67 | // validate the message is in the topic 68 | val receivedMessages = collectMessages( 69 | broker = broker, 70 | topic = sinkTopicName, 71 | filter = { kafkaMessage: KafkaMessage -> kafkaMessage.key == messageKey }, 72 | stopWhen = { messages: MutableList -> messages.size == 1 }, 73 | ) 74 | 75 | // we should have received a message 76 | receivedMessages.size shouldBe 1 77 | String(receivedMessages.first().body!!) shouldBe bodyString 78 | 79 | // the db row should be gone 80 | getRowCount(jdbi, messageKey) shouldBe 0 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /apps/outbox-processor/src/main/kotlin/vanillakotlin/outboxprocessor/App.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.outboxprocessor 2 | 3 | import org.slf4j.LoggerFactory 4 | import vanillakotlin.app.VanillaApp 5 | import vanillakotlin.app.runApplication 6 | import vanillakotlin.config.loadConfig 7 | import vanillakotlin.db.createJdbi 8 | import vanillakotlin.http4k.buildServer 9 | import vanillakotlin.kafka.producer.KafkaProducer 10 | import vanillakotlin.metrics.OtelMetrics 11 | import vanillakotlin.models.HealthCheckResponse 12 | import vanillakotlin.models.HealthMonitor 13 | import vanillakotlin.serde.mapper 14 | 15 | class App : VanillaApp { 16 | private val log = LoggerFactory.getLogger(javaClass) 17 | 18 | private val config = loadConfig() 19 | 20 | private val jdbi = createJdbi(config.db, mapper) 21 | 22 | private val metricsPublisher = OtelMetrics(config.metrics) 23 | 24 | private val kafkaProducer = 25 | KafkaProducer( 26 | config = config.kafka.producer, 27 | publishTimerMetric = metricsPublisher::publishTimerMetric, 28 | ) 29 | 30 | // Create a simple health monitor that checks database connectivity 31 | private val healthMonitors: List = listOf( 32 | object : HealthMonitor { 33 | override val name = "database" 34 | override fun check(): HealthCheckResponse = try { 35 | val isHealthy = jdbi.withHandle { handle -> 36 | handle.createQuery("SELECT 1").mapTo(Int::class.java).single() == 1 37 | } 38 | HealthCheckResponse( 39 | name = name, 40 | isHealthy = isHealthy, 41 | details = "Database connection successful", 42 | ) 43 | } catch (e: Exception) { 44 | log.warn("Database health check failed", e) 45 | HealthCheckResponse( 46 | name = name, 47 | isHealthy = false, 48 | details = "Database connection failed: ${e.message}", 49 | ) 50 | } 51 | }, 52 | ) 53 | 54 | // since this app doesn't provide any APIs other than a health endpoint, use a more convenient function to build a httpserver 55 | private val httpServer = buildServer(port = config.http.server.port) { 56 | healthMonitors { +healthMonitors } 57 | } 58 | 59 | // this port is made available for testing, because in testing we'll grab the first available open port and need to know what that is 60 | val httpServerPort: Int 61 | get() = httpServer.port() 62 | 63 | private val outboxProcessor = OutboxProcessor( 64 | config = config.outbox, 65 | jdbi = jdbi, 66 | kafkaSendAsync = kafkaProducer::sendAsync, 67 | ) 68 | 69 | override fun start() { 70 | // in this function we start up all the processes that are needed for the application. 71 | // they should all be non-blocking / daemons 72 | log.atInfo().log("Starting app") 73 | kafkaProducer.start() 74 | httpServer.start() 75 | outboxProcessor.start() 76 | log.atInfo().log("Started app") 77 | } 78 | 79 | override fun close() { 80 | log.atInfo().log("Closing app") 81 | httpServer.stop() 82 | log.atInfo().log("Closed app") 83 | } 84 | } 85 | 86 | // the main function is invoked via the gradle application plugin, and is configured in the `api.gradle.kts` file with 87 | // `mainClass.set("vanillakotlin.outboxprocessor.AppKt")` 88 | fun main() = runApplication { App() } 89 | -------------------------------------------------------------------------------- /apps/api/src/main/kotlin/vanillakotlin/api/App.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.api 2 | 3 | import org.slf4j.LoggerFactory 4 | import vanillakotlin.api.favoritethings.FavoriteThingsService 5 | import vanillakotlin.api.favoritethings.deleteFavoriteThingsRoute 6 | import vanillakotlin.api.favoritethings.getAdminFavoriteThingsRoute 7 | import vanillakotlin.api.favoritethings.getFavoriteThingIdsRoute 8 | import vanillakotlin.api.favoritethings.getFavoriteThingsRoute 9 | import vanillakotlin.api.favoritethings.postFavoriteThingsRoute 10 | import vanillakotlin.app.VanillaApp 11 | import vanillakotlin.app.runApplication 12 | import vanillakotlin.config.loadConfig 13 | import vanillakotlin.db.createJdbi 14 | import vanillakotlin.db.repository.FavoriteThingRepository 15 | import vanillakotlin.http.clients.initializeHttpClient 16 | import vanillakotlin.http.clients.thing.ThingGateway 17 | import vanillakotlin.http.interceptors.RetryInterceptor 18 | import vanillakotlin.http.interceptors.TelemetryInterceptor 19 | import vanillakotlin.http4k.buildServer 20 | import vanillakotlin.metrics.OtelMetrics 21 | import vanillakotlin.serde.mapper 22 | import java.lang.invoke.MethodHandles 23 | 24 | private val logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()) 25 | 26 | class App : VanillaApp { 27 | 28 | private val config = loadConfig() 29 | private val jdbi = createJdbi(config.db, mapper) 30 | 31 | // Initialize metrics 32 | private val metrics = OtelMetrics(config.metrics) 33 | 34 | private val httpClient = initializeHttpClient( 35 | config = config.http.client.connectionConfig, 36 | publishGaugeMetric = metrics::publishGaugeMetric, 37 | RetryInterceptor( 38 | config = config.http.client.retryConfig, 39 | publishCounterMetric = metrics::publishCounterMetric, 40 | ), 41 | TelemetryInterceptor( 42 | publishTimerMetric = metrics::publishTimerMetric, 43 | ), 44 | ) 45 | 46 | // Initialize ThingGateway 47 | val thingGateway = ThingGateway( 48 | httpClient = httpClient, 49 | config = config.http.client.thing, 50 | ) 51 | 52 | // Initialize FavoriteThingsService with real ThingGateway 53 | val favoriteThingRepository = FavoriteThingRepository(jdbi) 54 | val favoriteThingsService = FavoriteThingsService( 55 | upsertFavoriteThing = favoriteThingRepository.upsert, 56 | deleteFavoriteThingRepository = favoriteThingRepository.deleteItem, 57 | findAllFavoriteThings = favoriteThingRepository.findAll, 58 | getThingDetails = thingGateway::getThingDetails, 59 | ) 60 | 61 | private val httpServer = buildServer( 62 | host = config.http.server.host, 63 | port = config.http.server.port, 64 | ) { 65 | routeHandlers { 66 | +postFavoriteThingsRoute(favoriteThingsService.saveFavoriteThing) 67 | +deleteFavoriteThingsRoute(favoriteThingsService.deleteFavoriteThing) 68 | +getFavoriteThingIdsRoute(favoriteThingsService.getFavoriteThingIds) 69 | +getFavoriteThingsRoute(favoriteThingsService.getFavoriteThings) 70 | +getAdminFavoriteThingsRoute(favoriteThingsService.getFavoriteThingIds) 71 | } 72 | } 73 | 74 | val httpServerPort: Int 75 | get() = httpServer.port() 76 | 77 | override fun start() { 78 | logger.info("Starting app") 79 | httpServer.start() 80 | logger.info("Started app on port ${httpServer.port()}") 81 | } 82 | 83 | override fun close() { 84 | logger.info("Closing app") 85 | httpServer.stop() 86 | logger.info("Closed app") 87 | } 88 | } 89 | 90 | fun main() = runApplication { App() } 91 | -------------------------------------------------------------------------------- /apps/api/src/main/kotlin/vanillakotlin/api/favoritethings/FavoriteThingsService.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.api.favoritethings 2 | 3 | import org.slf4j.LoggerFactory 4 | import vanillakotlin.db.repository.FindAllFavoriteThings 5 | import vanillakotlin.db.repository.UpsertFavoriteThing 6 | import vanillakotlin.http.clients.thing.GetThingDetails 7 | import vanillakotlin.models.FavoriteThing 8 | import vanillakotlin.models.Thing 9 | import vanillakotlin.models.ThingIdentifier 10 | import vanillakotlin.db.repository.DeleteFavoriteThing as DeleteFavoriteThingRepository 11 | 12 | /** SAM interface for [FavoriteThingsService.saveFavoriteThing] */ 13 | fun interface SaveFavoriteThing { 14 | operator fun invoke(userFavoriteThing: FavoriteThing): SaveResult 15 | } 16 | 17 | /** SAM interface for [FavoriteThingsService.deleteFavoriteThing] */ 18 | fun interface DeleteFavoriteThing { 19 | operator fun invoke(thingIdentifier: ThingIdentifier): DeleteResult 20 | } 21 | 22 | /** SAM interface for [FavoriteThingsService.getFavoriteThings] */ 23 | fun interface GetFavoriteThingIds { 24 | operator fun invoke(): List 25 | } 26 | 27 | /** SAM interface for [FavoriteThingsService.getFavoriteThings] */ 28 | fun interface GetFavoriteThings { 29 | operator fun invoke(): List 30 | } 31 | 32 | class FavoriteThingsService( 33 | private val upsertFavoriteThing: UpsertFavoriteThing, 34 | private val deleteFavoriteThingRepository: DeleteFavoriteThingRepository, 35 | private val findAllFavoriteThings: FindAllFavoriteThings, 36 | private val getThingDetails: GetThingDetails, 37 | ) { 38 | private val logger = LoggerFactory.getLogger(javaClass) 39 | 40 | val saveFavoriteThing: SaveFavoriteThing = SaveFavoriteThing { userFavoriteThing -> 41 | try { 42 | upsertFavoriteThing(userFavoriteThing) 43 | SaveResult.Success 44 | } catch (e: Exception) { 45 | logger.warn("Failed to save record", e) 46 | SaveResult.Error(SaveErrorType.DATABASE_ERROR) 47 | } 48 | } 49 | 50 | val deleteFavoriteThing: DeleteFavoriteThing = DeleteFavoriteThing { thingIdentifier -> 51 | try { 52 | val rowCount = deleteFavoriteThingRepository(thingIdentifier) 53 | if (rowCount > 0) { 54 | DeleteResult.Success 55 | } else { 56 | logger.warn("Favorite not found during delete.") 57 | DeleteResult.NotFound 58 | } 59 | } catch (e: Exception) { 60 | logger.warn("Failed to delete record", e) 61 | DeleteResult.Error(DeleteErrorType.DATABASE_ERROR) 62 | } 63 | } 64 | 65 | val getFavoriteThingIds: GetFavoriteThingIds = GetFavoriteThingIds { 66 | findAllFavoriteThings().map { it.thingIdentifier } 67 | } 68 | 69 | val getFavoriteThings: GetFavoriteThings = GetFavoriteThings { 70 | val ids = getFavoriteThingIds() 71 | 72 | ids.mapNotNull { id -> 73 | try { 74 | getThingDetails(id) 75 | } catch (e: Exception) { 76 | logger.warn("Failed to get thing details for $id", e) 77 | null 78 | } 79 | } 80 | } 81 | } 82 | 83 | sealed class SaveResult { 84 | object Success : SaveResult() 85 | data class Error(val errorType: SaveErrorType) : SaveResult() 86 | } 87 | 88 | sealed class DeleteResult { 89 | object Success : DeleteResult() 90 | object NotFound : DeleteResult() 91 | data class Error(val errorType: DeleteErrorType) : DeleteResult() 92 | } 93 | 94 | enum class SaveErrorType { 95 | DATABASE_ERROR, 96 | } 97 | 98 | enum class DeleteErrorType { 99 | DATABASE_ERROR, 100 | } 101 | -------------------------------------------------------------------------------- /apps/kafka-transformer/src/main/kotlin/vanillakotlin/kafkatransformer/FavoriteThingsEventHandler.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.kafkatransformer 2 | 3 | import com.fasterxml.jackson.module.kotlin.readValue 4 | import org.slf4j.LoggerFactory 5 | import vanillakotlin.http.clients.thing.GetThingDetails 6 | import vanillakotlin.kafka.models.KafkaMessage 7 | import vanillakotlin.kafka.models.KafkaOutputMessage 8 | import vanillakotlin.kafka.transformer.TransformerEventHandler 9 | import vanillakotlin.kafka.transformer.TransformerMessages 10 | import vanillakotlin.models.UserFavoriteThing 11 | import vanillakotlin.serde.mapper 12 | 13 | /** 14 | * This handler expects to receive a KafkaMessage whose body is a ThingIdentifier string. 15 | * It then gets additional thing information and publishes a message to another topic with the augmented data. 16 | * Follows fail-fast architecture - any errors will cause the app to shut down. 17 | */ 18 | class FavoriteThingsEventHandler(private val getThingDetails: GetThingDetails) : TransformerEventHandler { 19 | private val log = LoggerFactory.getLogger(javaClass) 20 | 21 | override fun transform(kafkaMessage: KafkaMessage): TransformerMessages { 22 | // Note that this function and its input/output are not directly tied to kafka. 23 | // The major benefit of separating processing logic from kafka is that it is much easier to thoroughly test. 24 | // e.g. We can write unit tests that populate a KafkaMessage data class with the intended test values and directly call this 25 | // function without needing to spin up a kafka consumer and deal with its baggage. 26 | // We still want integration/functional tests, but we don't need as many. 27 | 28 | // See the `Concurrency Considerations` doc for more information on design 29 | 30 | // Always validate key format first - fail fast if invalid format 31 | val key = requireNotNull(kafkaMessage.key) { "Message key is required but was null" } 32 | require(key.contains(":")) { "Message key '$key' must be in format 'username:thingIdentifier'" } 33 | val userName = key.substringBeforeLast(":") 34 | require(userName.isNotBlank()) { "Username cannot be blank in key '$key'" } 35 | 36 | // A null body means the user favorite was deleted. Send a message downstream with the same pattern. 37 | val outputBody = kafkaMessage.body?.let { body -> 38 | // Parse the thingIdentifier from the message body - will fail fast if invalid JSON 39 | val thingIdentifier = mapper.readValue(body) 40 | 41 | // Get thing details - will fail fast if service call fails 42 | val thing = getThingDetails(thingIdentifier) 43 | 44 | // Require that we got thing details - fail fast if thing not found 45 | requireNotNull(thing) { "Thing details not found for identifier: $thingIdentifier" } 46 | 47 | UserFavoriteThing( 48 | userName = userName, 49 | thingIdentifier = thing.id, 50 | ) 51 | } 52 | 53 | // Here, we are returning a single message to be published to the next topic. If we wanted 54 | // to publish multiple messages, we would return a TransformerMessages.Multiple object, and if 55 | // we wanted to drop the message we would return a TransformerMessages.Dropped object. Once returned, 56 | // the KafkaTransformer will handle sending the message downstream in the order it was consumed. 57 | // Committing the offset is also handled by the KafkaTransformer and done periodically every 2 seconds. 58 | return TransformerMessages.Single( 59 | kafkaOutputMessage = KafkaOutputMessage( 60 | key = kafkaMessage.key, 61 | value = outputBody, 62 | ), 63 | ).also { log.atDebug().log("Processed event") } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /apps/bulk-inserter/src/main/kotlin/vanillakotlin/bulkinserter/BulkInserterHandler.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.bulkinserter 2 | 3 | import com.fasterxml.jackson.module.kotlin.readValue 4 | import org.slf4j.LoggerFactory 5 | import vanillakotlin.db.repository.AddFavoriteThingToBatch 6 | import vanillakotlin.db.repository.RunFavoriteThingBatch 7 | import vanillakotlin.kafka.models.KafkaConsumerSequenceHandler 8 | import vanillakotlin.kafka.models.KafkaMessage 9 | import vanillakotlin.models.FavoriteThing 10 | import vanillakotlin.models.UserFavoriteThing 11 | import vanillakotlin.serde.mapper 12 | 13 | /** 14 | * This handler expects to receive a KafkaMessage whose body is a UserFavoriteThing. 15 | * It then converts it to a FavoriteThing and adds it to a batch, running the batch when the end of the batch is reached. 16 | * 17 | * This handler will fail fast on any errors - malformed messages, null keys, or JSON parsing errors will cause the app to shut down. 18 | */ 19 | class BulkInserterHandler( 20 | private val addToBatch: AddFavoriteThingToBatch, 21 | private val runBatch: RunFavoriteThingBatch, 22 | ) : KafkaConsumerSequenceHandler { 23 | private val log = LoggerFactory.getLogger(javaClass) 24 | 25 | override fun processSequence(messages: Sequence) = messages.forEach { message -> 26 | // Note that this function and its input/output are not directly tied to kafka. 27 | // The major benefit of separating processing logic from kafka is that it is much easier to thoroughly test. 28 | // e.g. We can write unit tests that populate a KafkaMessage data class with the intended test values and directly call this 29 | // function without needing to spin up a kafka consumer and deal with its baggage. 30 | // We still want integration/functional tests, but we don't need as many. 31 | 32 | // See the `Concurrency Considerations` doc for more information on design 33 | 34 | processMessage(message) 35 | }.also { 36 | // Once the end of the consumer batch has been reached, run the batch. There is no concurrency here, so be sure that if your 37 | // application does have concurrency, you handle it properly. 38 | runBatch().also { batchSize -> log.atDebug().log { "batch successfully run, processed $batchSize records" } } 39 | } 40 | 41 | private fun processMessage(message: KafkaMessage) { 42 | // Require valid key - app will fail if key is null or blank 43 | val key = requireNotNull(message.key) { "Message key is required but was null" } 44 | require(key.isNotBlank()) { "Message key is required but was blank" } 45 | 46 | // Extract thingIdentifier from key format "username:thingIdentifier" 47 | // App will fail if key doesn't contain a colon 48 | require(key.contains(":")) { "Message key '$key' must be in format 'username:thingIdentifier'" } 49 | 50 | val thingIdentifier = key.substringAfterLast(":") // Use substringAfterLast to handle multiple colons 51 | require(thingIdentifier.isNotBlank()) { "Thing identifier cannot be blank in key '$key'" } 52 | 53 | // A null body means the user favorite was deleted, so we'll create a new FavoriteThing with isDeleted = true to indicate to the 54 | // batch processor that it should delete that record 55 | val favoriteThing = message.body?.let { body -> 56 | // Parse JSON - will throw exception if malformed, causing app to fail 57 | val userFavoriteThing = mapper.readValue(body) 58 | FavoriteThing( 59 | thingIdentifier = userFavoriteThing.thingIdentifier, 60 | isDeleted = userFavoriteThing.isDeleted, 61 | ) 62 | } ?: FavoriteThing( 63 | thingIdentifier = thingIdentifier, 64 | isDeleted = true, 65 | ) 66 | 67 | // Add the FavoriteThing to the current batch 68 | addToBatch(favoriteThing).also { log.atDebug().log { "adding $favoriteThing to batch" } } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /apps/api/src/test/kotlin/vanillakotlin/api/favoritethings/FavoriteThingsServiceTest.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.api.favoritethings 2 | 3 | import io.kotest.assertions.assertSoftly 4 | import io.kotest.matchers.collections.shouldContainAll 5 | import io.kotest.matchers.shouldBe 6 | import org.junit.jupiter.api.Test 7 | import vanillakotlin.api.buildTestThing 8 | import vanillakotlin.db.repository.DeleteFavoriteThing 9 | import vanillakotlin.db.repository.FindAllFavoriteThings 10 | import vanillakotlin.db.repository.UpsertFavoriteThing 11 | import vanillakotlin.http.clients.thing.GetThingDetails 12 | import vanillakotlin.models.FavoriteThing 13 | import vanillakotlin.random.randomThing 14 | import java.sql.SQLException 15 | import java.time.LocalDateTime 16 | 17 | private val testThings: List = listOf(randomThing(), randomThing()) 18 | private val testTimestamp = LocalDateTime.of(2023, 1, 1, 12, 0, 0) 19 | private val testFavorites: List = 20 | listOf( 21 | FavoriteThing(id = 1, thingIdentifier = testThings[0], createdTs = testTimestamp, updatedTs = testTimestamp), 22 | FavoriteThing(id = 2, thingIdentifier = testThings[1], createdTs = testTimestamp, updatedTs = testTimestamp), 23 | ) 24 | 25 | class FavoriteThingsServiceTest { 26 | 27 | @Test fun `saveFavoriteThing success`() { 28 | val service = FavoriteThingsService( 29 | upsertFavoriteThing = UpsertFavoriteThing { testFavorites[0] }, 30 | deleteFavoriteThingRepository = DeleteFavoriteThing { 1 }, 31 | findAllFavoriteThings = FindAllFavoriteThings { testFavorites }, 32 | getThingDetails = GetThingDetails { buildTestThing(it) }, 33 | ) 34 | 35 | val result = service.saveFavoriteThing(FavoriteThing(thingIdentifier = testThings[0])) 36 | 37 | assertSoftly(result) { 38 | this shouldBe SaveResult.Success 39 | } 40 | } 41 | 42 | @Test fun `deleteFavoriteThing success`() { 43 | val service = FavoriteThingsService( 44 | upsertFavoriteThing = UpsertFavoriteThing { testFavorites[0] }, 45 | deleteFavoriteThingRepository = DeleteFavoriteThing { thingIdentifier -> 46 | if (thingIdentifier == testFavorites[0].thingIdentifier) 1 else 0 47 | }, 48 | findAllFavoriteThings = FindAllFavoriteThings { testFavorites }, 49 | getThingDetails = GetThingDetails { thingId -> buildTestThing(thingId) }, 50 | ) 51 | 52 | val result = service.deleteFavoriteThing(testFavorites[0].thingIdentifier) 53 | 54 | assertSoftly(result) { 55 | this shouldBe DeleteResult.Success 56 | } 57 | } 58 | 59 | @Test fun `deleteFavoriteThing record doesn't exist`() { 60 | val service = FavoriteThingsService( 61 | upsertFavoriteThing = UpsertFavoriteThing { testFavorites[0] }, 62 | deleteFavoriteThingRepository = DeleteFavoriteThing { 0 }, 63 | findAllFavoriteThings = FindAllFavoriteThings { testFavorites }, 64 | getThingDetails = GetThingDetails { thingId -> buildTestThing(thingId) }, 65 | ) 66 | 67 | val result = service.deleteFavoriteThing(testFavorites[0].thingIdentifier) 68 | 69 | result shouldBe DeleteResult.NotFound 70 | } 71 | 72 | @Test fun `deleteFavoriteThing SQL Exception`() { 73 | val service = FavoriteThingsService( 74 | upsertFavoriteThing = UpsertFavoriteThing { testFavorites[0] }, 75 | deleteFavoriteThingRepository = DeleteFavoriteThing { throw SQLException("Failure") }, 76 | findAllFavoriteThings = FindAllFavoriteThings { testFavorites }, 77 | getThingDetails = GetThingDetails { thingId -> buildTestThing(thingId) }, 78 | ) 79 | 80 | val result = service.deleteFavoriteThing(testFavorites[0].thingIdentifier) 81 | 82 | result shouldBe DeleteResult.Error(DeleteErrorType.DATABASE_ERROR) 83 | } 84 | 85 | @Test fun `getFavoriteThings success`() { 86 | val service = FavoriteThingsService( 87 | upsertFavoriteThing = UpsertFavoriteThing { testFavorites[0] }, 88 | deleteFavoriteThingRepository = DeleteFavoriteThing { 1 }, 89 | findAllFavoriteThings = FindAllFavoriteThings { testFavorites }, 90 | getThingDetails = GetThingDetails { thingId -> buildTestThing(thingId) }, 91 | ) 92 | 93 | val things = service.getFavoriteThings() 94 | 95 | things.size shouldBe testThings.size 96 | things shouldContainAll testThings 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /libs/metrics/src/main/kotlin/vanillakotlin/metrics/OtelMetrics.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.metrics 2 | 3 | import io.opentelemetry.api.GlobalOpenTelemetry 4 | import io.opentelemetry.api.common.AttributeKey 5 | import io.opentelemetry.api.common.Attributes 6 | import io.opentelemetry.api.metrics.DoubleCounter 7 | import io.opentelemetry.api.metrics.LongHistogram 8 | import io.opentelemetry.api.metrics.MeterProvider 9 | import io.opentelemetry.api.metrics.ObservableDoubleGauge 10 | import kotlin.system.measureTimeMillis 11 | import kotlin.time.Duration 12 | import kotlin.time.Duration.Companion.milliseconds 13 | 14 | typealias PublishCounterMetric = (name: String, tags: Map) -> Unit 15 | typealias PublishCounterMetricAmount = (name: String, tags: Map, amount: Double) -> Unit 16 | typealias PublishTimerMetric = (name: String, tags: Map, duration: Duration) -> Unit 17 | typealias PublishDistributionSummaryMetric = (name: String, tags: Map, value: Double) -> Unit 18 | typealias PublishGaugeMetric = (name: String, tags: Map, supplier: () -> Number?) -> ObservableDoubleGauge 19 | typealias Gauge = ObservableDoubleGauge 20 | 21 | class OtelMetrics( 22 | private val config: Config, 23 | meterProvider: MeterProvider = GlobalOpenTelemetry.getMeterProvider(), 24 | ) { 25 | data class Config(val tags: Map) 26 | 27 | companion object { 28 | const val PROVIDER_NAME = "vanillakotlin.metrics.OtelMetricsPublisher" 29 | } 30 | 31 | private val meter = meterProvider.get(PROVIDER_NAME) 32 | private val timers = mutableMapOf() 33 | private val counters = mutableMapOf() 34 | private val gauges = mutableMapOf() 35 | 36 | // used as drop-in replacement for the micrometer timer 37 | fun publishTimerMetric( 38 | name: String, 39 | tags: Map, 40 | duration: Duration, 41 | ) = timers.getOrPut(name) { 42 | meter.histogramBuilder(name) 43 | .setDescription("") 44 | .setUnit("ms") 45 | .ofLongs() 46 | .build() 47 | }.record( 48 | duration.inWholeMilliseconds, 49 | Attributes.builder().apply { 50 | (config.tags + tags).map { (k, v) -> put(AttributeKey.stringKey(k), v) } 51 | }.build(), 52 | ) 53 | 54 | // used as drop-in replacement for the micrometer counter 55 | fun publishCounterMetric( 56 | name: String, 57 | tags: Map, 58 | ) = counters.getOrPut(name) { 59 | meter.counterBuilder(name) 60 | .setDescription("") 61 | .setUnit("unit") 62 | .ofDoubles() 63 | .build() 64 | }.add( 65 | 1.0, 66 | Attributes.builder().apply { 67 | (config.tags + tags).map { (k, v) -> put(AttributeKey.stringKey(k), v) } 68 | }.build(), 69 | ) 70 | 71 | // used as drop-in replacement for the micrometer counter amount 72 | fun publishCounterMetricAmount( 73 | name: String, 74 | tags: Map, 75 | amount: Double, 76 | ) = counters.getOrPut(name) { 77 | meter.counterBuilder(name) 78 | .setDescription("") 79 | .setUnit("unit") 80 | .ofDoubles() 81 | .build() 82 | }.add( 83 | amount, 84 | Attributes.builder().apply { 85 | (config.tags + tags).map { (k, v) -> put(AttributeKey.stringKey(k), v) } 86 | }.build(), 87 | ) 88 | 89 | fun publishGaugeMetric( 90 | name: String, 91 | tags: Map, 92 | supplier: () -> Number?, 93 | ): ObservableDoubleGauge = gauges.getOrPut(name) { 94 | meter.gaugeBuilder(name) 95 | .setDescription("") 96 | .setUnit("unit") 97 | .buildWithCallback { 98 | it.record( 99 | supplier()?.toDouble() ?: Double.NaN, 100 | Attributes.builder().apply { (config.tags + tags).map { (k, v) -> put(AttributeKey.stringKey(k), v) } }.build(), 101 | ) 102 | } 103 | } 104 | } 105 | 106 | fun PublishTimerMetric.time( 107 | metricName: String, 108 | tags: Map = emptyMap(), 109 | block: () -> Unit, 110 | ) { 111 | measureTimeMillis { block() }.also { 112 | this(metricName, tags, it.milliseconds) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /libs/db/src/test/kotlin/vanillakotlin/db/repository/FavoriteThingRepositoryTest.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.db.repository 2 | 3 | import io.kotest.assertions.assertSoftly 4 | import io.kotest.matchers.shouldBe 5 | import io.kotest.matchers.shouldNotBe 6 | import org.junit.jupiter.api.Test 7 | import vanillakotlin.models.FavoriteThing 8 | import vanillakotlin.random.randomThing 9 | 10 | class FavoriteThingRepositoryTest { 11 | private val jdbi by lazy { buildTestDb() } 12 | private val repository by lazy { FavoriteThingRepository(jdbi) } 13 | 14 | @Test fun `upsert favorite thing - insert new thing`() { 15 | val favoriteThing = FavoriteThing( 16 | id = null, 17 | thingIdentifier = randomThing(), 18 | ) 19 | 20 | val result = repository.upsert(favoriteThing) 21 | 22 | assertSoftly { 23 | result.id shouldNotBe null 24 | result.thingIdentifier shouldBe favoriteThing.thingIdentifier 25 | result.createdTs shouldNotBe null 26 | result.updatedTs shouldNotBe null 27 | } 28 | } 29 | 30 | @Test fun `upsert favorite thing - update existing thing`() { 31 | // First insert a thing 32 | val originalThing = FavoriteThing( 33 | id = null, 34 | thingIdentifier = randomThing(), 35 | ) 36 | val insertedThing = repository.upsert(originalThing) 37 | 38 | // Wait a moment to ensure different timestamp 39 | Thread.sleep(100) 40 | 41 | // Now upsert the same thing again (should update timestamp) 42 | val result = repository.upsert(insertedThing) 43 | 44 | assertSoftly { 45 | result.id shouldBe insertedThing.id 46 | result.thingIdentifier shouldBe insertedThing.thingIdentifier 47 | result.createdTs shouldBe insertedThing.createdTs 48 | result.updatedTs shouldNotBe insertedThing.updatedTs // Should be updated 49 | } 50 | } 51 | 52 | @Test fun `delete thing - thing exists`() { 53 | val favoriteThing = FavoriteThing( 54 | id = null, 55 | thingIdentifier = randomThing(), 56 | ) 57 | val insertedThing = repository.upsert(favoriteThing) 58 | 59 | val rowCount = repository.deleteItem(insertedThing.thingIdentifier) 60 | 61 | assertSoftly { 62 | rowCount shouldBe 1 63 | // Verify thing is deleted by checking findAll doesn't contain it 64 | val allThings = repository.findAll() 65 | allThings.none { it.thingIdentifier == insertedThing.thingIdentifier } shouldBe true 66 | } 67 | } 68 | 69 | @Test fun `delete thing - thing does not exist`() { 70 | val nonExistentThing = randomThing() 71 | 72 | val rowCount = repository.deleteItem(nonExistentThing) 73 | 74 | rowCount shouldBe 0 75 | } 76 | 77 | @Test fun `find all things`() { 78 | // Clear any existing things by checking current state 79 | val initialCount = repository.findAll().size 80 | 81 | // Insert a few things 82 | val thing1 = FavoriteThing(thingIdentifier = randomThing()) 83 | val thing2 = FavoriteThing(thingIdentifier = randomThing()) 84 | 85 | repository.upsert(thing1) 86 | repository.upsert(thing2) 87 | 88 | val allThings = repository.findAll() 89 | 90 | assertSoftly { 91 | allThings.size shouldBe (initialCount + 2) 92 | allThings.any { it.thingIdentifier == thing1.thingIdentifier } shouldBe true 93 | allThings.any { it.thingIdentifier == thing2.thingIdentifier } shouldBe true 94 | } 95 | } 96 | 97 | @Test fun `batch operations - add to batch and run batch`() { 98 | val thing1 = FavoriteThing(thingIdentifier = randomThing()) 99 | val thing2 = FavoriteThing(thingIdentifier = randomThing()) 100 | val thing3 = FavoriteThing(thingIdentifier = randomThing()) 101 | 102 | // Add things to batch 103 | repository.addToBatch(thing1) 104 | repository.addToBatch(thing2) 105 | repository.addToBatch(thing3) 106 | 107 | // Run batch 108 | val result = repository.runBatch() 109 | 110 | assertSoftly { 111 | result shouldBe 3 112 | 113 | // Verify things were inserted 114 | val allThings = repository.findAll() 115 | allThings.any { it.thingIdentifier == thing1.thingIdentifier } shouldBe true 116 | allThings.any { it.thingIdentifier == thing2.thingIdentifier } shouldBe true 117 | allThings.any { it.thingIdentifier == thing3.thingIdentifier } shouldBe true 118 | } 119 | } 120 | 121 | @Test fun `batch operations - empty batch returns zero`() { 122 | val result = repository.runBatch() 123 | 124 | result shouldBe 0 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /apps/api/src/main/kotlin/vanillakotlin/api/favoritethings/FavoriteThingsRoutes.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.api.favoritethings 2 | 3 | import org.http4k.core.Method 4 | import org.http4k.core.Request 5 | import org.http4k.core.Response 6 | import org.http4k.core.Status 7 | import org.http4k.routing.RoutingHttpHandler 8 | import org.http4k.routing.bind 9 | import vanillakotlin.extensions.toJsonString 10 | import vanillakotlin.models.FavoriteThing 11 | import vanillakotlin.models.Thing 12 | 13 | /** 14 | * Each route's responsibility is to: 15 | * - read the request attributes 16 | * - call the underlying function 17 | * - create the response based on the outcome of the function 18 | * 19 | * It should not contain business logic for the function itself. This is handled by the underlying function. 20 | * 21 | * These routes are example of HTTP4K route handlers that integrates with OpenAPI. 22 | * Docs available at https://www.http4k.org/guide/reference/contracts/ 23 | * and https://www.http4k.org/guide/howto/integrate_with_openapi/ 24 | * and also here: https://www.http4k.org/blog/documenting_apis_with_openapi/#4_modelling_http_body_messages 25 | * They use a code-first perspective. i.e. The swagger spec is generated from the properties specified here. 26 | */ 27 | 28 | // Response DTO 29 | data class FavoriteThingResponse( 30 | val thingIdentifier: String, 31 | val sellingPrice: Double, 32 | val productName: String, 33 | ) 34 | 35 | // Extension function to convert Thing to FavoriteThingResponse 36 | fun Thing.toFavoriteThingResponse(): FavoriteThingResponse = FavoriteThingResponse( 37 | thingIdentifier = this.id, 38 | sellingPrice = this.sellingPrice, 39 | productName = this.productName, 40 | ) 41 | 42 | // Extension functions for auth checking 43 | private fun Request.hasAuth(): Boolean = this.header("X-ID") != null && this.header("X-EMAIL") != null && this.header("X-MEMBER-OF") != null 44 | 45 | private fun Request.hasAdminAuth(): Boolean = hasAuth() && this.header("X-MEMBER-OF") == "admin-user" 46 | 47 | // Individual route functions 48 | fun postFavoriteThingsRoute(saveFavoriteThing: SaveFavoriteThing): RoutingHttpHandler { 49 | return org.http4k.routing.routes( 50 | "/api/v1/favorite_things/{thingIdentifier}" bind Method.POST to { request -> 51 | if (!request.hasAuth()) return@to Response(Status.UNAUTHORIZED) 52 | val thingIdentifier = request.uri.path.substringAfterLast("/") 53 | when (val result = saveFavoriteThing(FavoriteThing(thingIdentifier = thingIdentifier))) { 54 | is SaveResult.Success -> Response(Status.OK) 55 | is SaveResult.Error -> Response(Status.INTERNAL_SERVER_ERROR).body(result.errorType.toString()) 56 | } 57 | }, 58 | ) 59 | } 60 | 61 | fun deleteFavoriteThingsRoute(deleteFavoriteThing: DeleteFavoriteThing): RoutingHttpHandler { 62 | return org.http4k.routing.routes( 63 | "/api/v1/favorite_things/{thingIdentifier}" bind Method.DELETE to { request -> 64 | if (!request.hasAuth()) return@to Response(Status.UNAUTHORIZED) 65 | val thingIdentifier = request.uri.path.substringAfterLast("/") 66 | when (val result = deleteFavoriteThing(thingIdentifier)) { 67 | is DeleteResult.Success -> Response(Status.OK) 68 | is DeleteResult.NotFound -> Response(Status.NOT_FOUND) 69 | is DeleteResult.Error -> Response(Status.INTERNAL_SERVER_ERROR).body(result.errorType.toString()) 70 | } 71 | }, 72 | ) 73 | } 74 | 75 | fun getFavoriteThingIdsRoute(getFavoriteThingIds: GetFavoriteThingIds): RoutingHttpHandler { 76 | return org.http4k.routing.routes( 77 | "/api/v1/favorite_things_ids" bind Method.GET to { request -> 78 | if (!request.hasAuth()) return@to Response(Status.UNAUTHORIZED) 79 | val favoriteThings = getFavoriteThingIds() 80 | Response(Status.OK).body(favoriteThings.toJsonString()) 81 | }, 82 | ) 83 | } 84 | 85 | fun getFavoriteThingsRoute(getFavoriteThings: GetFavoriteThings): RoutingHttpHandler { 86 | return org.http4k.routing.routes( 87 | "/api/v1/favorite_things" bind Method.GET to { request -> 88 | if (!request.hasAuth()) return@to Response(Status.UNAUTHORIZED) 89 | val favoriteThingResponses = getFavoriteThings().map { it.toFavoriteThingResponse() } 90 | Response(Status.OK).body(favoriteThingResponses.toJsonString()) 91 | }, 92 | ) 93 | } 94 | 95 | fun getAdminFavoriteThingsRoute(getFavoriteThingIds: GetFavoriteThingIds): RoutingHttpHandler { 96 | return org.http4k.routing.routes( 97 | "/api/v1/admin/favorite_things" bind Method.GET to { request -> 98 | if (!request.hasAdminAuth()) return@to Response(Status.UNAUTHORIZED) 99 | val favoriteThings = getFavoriteThingIds() 100 | Response(Status.OK).body(favoriteThings.toJsonString()) 101 | }, 102 | ) 103 | } 104 | -------------------------------------------------------------------------------- /apps/api/src/test/kotlin/vanillakotlin/api/favoritethings/FavoriteThingsRoutesTest.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.api.favoritethings 2 | 3 | import io.kotest.assertions.assertSoftly 4 | import io.kotest.assertions.fail 5 | import io.kotest.assertions.json.shouldEqualJson 6 | import io.kotest.matchers.shouldBe 7 | import org.http4k.core.Method 8 | import org.http4k.core.Request 9 | import org.http4k.core.Status 10 | import org.junit.jupiter.api.Test 11 | import vanillakotlin.api.addAuth 12 | import vanillakotlin.api.buildTestThing 13 | 14 | class FavoriteThingsRoutesTest { 15 | 16 | @Test fun `postFavoriteThingsRoute OK`() { 17 | val saveFavoriteThing = SaveFavoriteThing { _ -> SaveResult.Success } 18 | val handler = postFavoriteThingsRoute(saveFavoriteThing) 19 | 20 | val response = handler(Request(Method.POST, "/api/v1/favorite_things/1").addAuth()) 21 | assertSoftly(response) { 22 | status shouldBe Status.OK 23 | } 24 | } 25 | 26 | @Test fun `postFavoriteThingsRoute missing auth`() { 27 | val saveFavoriteThing = SaveFavoriteThing { _ -> fail("unexpected function called") } 28 | val handler = postFavoriteThingsRoute(saveFavoriteThing) 29 | 30 | val response = handler(Request(Method.POST, "/api/v1/favorite_things/1")) 31 | response.status shouldBe Status.UNAUTHORIZED 32 | } 33 | 34 | @Test fun `deleteFavoriteThingsRoute OK`() { 35 | val deleteFavoriteThing = DeleteFavoriteThing { _ -> DeleteResult.Success } 36 | val handler = deleteFavoriteThingsRoute(deleteFavoriteThing) 37 | 38 | val response = handler(Request(Method.DELETE, "/api/v1/favorite_things/1").addAuth()) 39 | assertSoftly(response) { 40 | status shouldBe Status.OK 41 | } 42 | } 43 | 44 | @Test fun `deleteFavoriteThingsRoute missing auth`() { 45 | val deleteFavoriteThing = DeleteFavoriteThing { _ -> fail("unexpected function called") } 46 | val handler = deleteFavoriteThingsRoute(deleteFavoriteThing) 47 | 48 | val response = handler(Request(Method.DELETE, "/api/v1/favorite_things/1")) 49 | response.status shouldBe Status.UNAUTHORIZED 50 | } 51 | 52 | @Test fun `getFavoriteThingIdsRoute OK`() { 53 | val favoriteThingIds = listOf("1", "2") 54 | val getFavoriteThingIds = GetFavoriteThingIds { favoriteThingIds } 55 | val handler = getFavoriteThingIdsRoute(getFavoriteThingIds) 56 | 57 | val response = handler(Request(Method.GET, "/api/v1/favorite_things_ids").addAuth()) 58 | assertSoftly(response) { 59 | status shouldBe Status.OK 60 | bodyString() shouldBe """["1","2"]""" 61 | } 62 | } 63 | 64 | @Test fun `getFavoriteThingIdsRoute missing auth`() { 65 | val getFavoriteThingIds = GetFavoriteThingIds { fail("unexpected function called") } 66 | val handler = getFavoriteThingIdsRoute(getFavoriteThingIds) 67 | 68 | val response = handler(Request(Method.GET, "/api/v1/favorite_things_ids")) 69 | response.status shouldBe Status.UNAUTHORIZED 70 | } 71 | 72 | @Test fun `getFavoriteThingsRoute OK`() { 73 | val favoriteThings = listOf(buildTestThing("1"), buildTestThing("2")) 74 | val getFavoriteThings = GetFavoriteThings { favoriteThings } 75 | val handler = getFavoriteThingsRoute(getFavoriteThings) 76 | 77 | val response = handler(Request(Method.GET, "/api/v1/favorite_things").addAuth()) 78 | assertSoftly(response) { 79 | status shouldBe Status.OK 80 | 81 | // On validating JSON, there is some personal preference involved, and it depends on the context of what's being tested too, 82 | // but it's good to have at least some tests that validate at the string level as opposed to constructing objects and validating 83 | // their properties. Some reasons: 84 | // 1) You can see what the JSON output actually looks like, so takes less brainpower to inspect and debug 85 | // 2) It avoids bugs where serialization masks a problem in the testing 86 | 87 | // https://kotest.io/docs/assertions/json-matchers.html#shouldequaljson 88 | bodyString() shouldEqualJson """ 89 | [ 90 | { 91 | "thingIdentifier": "1", 92 | "sellingPrice": 19.99, 93 | "productName": "Test Product" 94 | }, 95 | { 96 | "thingIdentifier": "2", 97 | "sellingPrice": 19.99, 98 | "productName": "Test Product" 99 | } 100 | ] 101 | """ 102 | } 103 | } 104 | 105 | @Test fun `getFavoriteThingsRoute missing auth`() { 106 | val getFavoriteThings = GetFavoriteThings { fail("unexpected function called") } 107 | val handler = getFavoriteThingsRoute(getFavoriteThings) 108 | 109 | val response = handler(Request(Method.GET, "/api/v1/favorite_things")) 110 | response.status shouldBe Status.UNAUTHORIZED 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /libs/kafka/src/testFixtures/kotlin/vanillakotlin/kafka/KafkaTestUtilities.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.kafka 2 | 3 | import org.apache.kafka.clients.admin.AdminClient 4 | import org.apache.kafka.clients.admin.NewTopic 5 | import org.apache.kafka.clients.consumer.ConsumerConfig 6 | import org.apache.kafka.clients.consumer.KafkaConsumer 7 | import org.apache.kafka.common.serialization.ByteArrayDeserializer 8 | import org.apache.kafka.common.serialization.StringDeserializer 9 | import vanillakotlin.kafka.models.KafkaMessage 10 | import vanillakotlin.metrics.PublishTimerMetric 11 | import vanillakotlin.random.randomString 12 | import java.time.Duration 13 | import java.util.Properties 14 | import java.util.concurrent.TimeUnit 15 | 16 | /** 17 | * Test utility to collect messages from a Kafka topic for testing purposes. 18 | * This function polls the topic for a specified duration and returns all messages found. 19 | */ 20 | fun collectMessages( 21 | broker: String, 22 | topic: String, 23 | metricsPublisher: PublishTimerMetric? = null, 24 | filter: (KafkaMessage) -> Boolean = { true }, 25 | stopWhen: (MutableList) -> Boolean = { false }, 26 | timeoutMs: Long = 10000L, 27 | pollTimeoutMs: Long = 1000L, 28 | ): List { 29 | val messages = mutableListOf() 30 | val consumerGroup = "test-consumer-${randomString()}" 31 | 32 | val consumerProperties = Properties().apply { 33 | put(ConsumerConfig.GROUP_ID_CONFIG, consumerGroup) 34 | put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, broker) 35 | put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer::class.java.name) 36 | put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer::class.java.name) 37 | put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") 38 | put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false) 39 | put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 10000) 40 | put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, 3000) 41 | put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 30000) 42 | put(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG, 500) 43 | } 44 | 45 | var consecutiveEmptyPolls = 0 46 | val maxEmptyPolls = 5 // Stop after 5 consecutive empty polls to prevent infinite loops 47 | 48 | KafkaConsumer(consumerProperties).use { consumer -> 49 | try { 50 | consumer.subscribe(listOf(topic)) 51 | 52 | val startTime = System.currentTimeMillis() 53 | while (System.currentTimeMillis() - startTime < timeoutMs && consecutiveEmptyPolls < maxEmptyPolls) { 54 | val records = consumer.poll(Duration.ofMillis(pollTimeoutMs)) 55 | 56 | if (records.isEmpty) { 57 | consecutiveEmptyPolls++ 58 | } else { 59 | consecutiveEmptyPolls = 0 60 | records.forEach { record -> 61 | val kafkaMessage = KafkaMessage( 62 | broker = broker, 63 | topic = record.topic(), 64 | key = record.key(), 65 | partition = record.partition(), 66 | offset = record.offset(), 67 | headers = record.headers().associate { it.key() to it.value() }, 68 | timestamp = record.timestamp(), 69 | body = record.value(), 70 | endOfBatch = false, 71 | ) 72 | 73 | if (filter(kafkaMessage)) { 74 | messages.add(kafkaMessage) 75 | } 76 | } 77 | } 78 | 79 | if (stopWhen(messages)) { 80 | break 81 | } 82 | } 83 | } catch (e: Exception) { 84 | // Log the exception but don't let it break the test 85 | println("Warning: Error collecting messages from topic $topic: ${e.message}") 86 | } 87 | } 88 | 89 | return messages 90 | } 91 | 92 | /** 93 | * Test utility to create a Kafka topic for testing 94 | */ 95 | fun AdminClient.createTestTopic( 96 | topicName: String, 97 | partitions: Int = 1, 98 | replicationFactor: Short = 1, 99 | ) { 100 | createTopics(listOf(NewTopic(topicName, partitions, replicationFactor))) 101 | .all() 102 | .get(10, TimeUnit.SECONDS) 103 | } 104 | 105 | /** 106 | * Test utility to delete a Kafka topic for testing 107 | */ 108 | fun AdminClient.deleteTestTopic(topicName: String) { 109 | deleteTopics(listOf(topicName)) 110 | .all() 111 | .get(10, TimeUnit.SECONDS) 112 | } 113 | 114 | /** 115 | * Mock metrics publisher for testing 116 | */ 117 | class MockMetricsPublisher : PublishTimerMetric { 118 | val publishedMetrics = mutableListOf>>() 119 | 120 | override fun invoke( 121 | metricName: String, 122 | tags: Map, 123 | duration: kotlin.time.Duration, 124 | ) { 125 | publishedMetrics.add(metricName to tags) 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /libs/client/src/test/kotlin/vanillakotlin/http/client/thing/ThingGraphqlResponseTest.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.http.client.thing 2 | 3 | import org.junit.jupiter.api.Assertions.assertEquals 4 | import org.junit.jupiter.api.Assertions.assertNotNull 5 | import org.junit.jupiter.api.Assertions.assertNull 6 | import org.junit.jupiter.api.Test 7 | import vanillakotlin.http.clients.thing.ThingResponse 8 | import vanillakotlin.models.Thing 9 | 10 | class ThingGraphqlResponseTest { 11 | 12 | @Test fun `test ThingResponse with valid data`() { 13 | val thing = Thing( 14 | id = "123", 15 | productName = "Test Product", 16 | sellingPrice = 19.99, 17 | ) 18 | val thingData = ThingResponse.ThingData(thing = thing) 19 | val response = ThingResponse(data = thingData) 20 | 21 | val result = response.toThing() 22 | assertNotNull(result) 23 | assertEquals("123", result?.id) 24 | assertEquals("Test Product", result?.productName) 25 | assertEquals(19.99, result?.sellingPrice) 26 | } 27 | 28 | @Test fun `test ThingResponse with null data`() { 29 | val response = ThingResponse(data = null) 30 | 31 | val result = response.toThing() 32 | assertNull(result) 33 | } 34 | 35 | @Test fun `test ThingResponse toThing returns correct thing`() { 36 | val thing = Thing( 37 | id = "456", 38 | productName = "Another Product", 39 | sellingPrice = 29.99, 40 | ) 41 | val thingData = ThingResponse.ThingData(thing = thing) 42 | val response = ThingResponse(data = thingData) 43 | 44 | val result = response.toThing() 45 | assertEquals(thing, result) 46 | } 47 | 48 | @Test fun `test ThingResponse with different thing properties`() { 49 | val thing = Thing( 50 | id = "789", 51 | productName = "Special Product", 52 | sellingPrice = 99.99, 53 | ) 54 | val thingData = ThingResponse.ThingData(thing = thing) 55 | val response = ThingResponse(data = thingData) 56 | 57 | val result = response.toThing() 58 | assertNotNull(result) 59 | assertEquals("789", result?.id) 60 | assertEquals("Special Product", result?.productName) 61 | assertEquals(99.99, result?.sellingPrice) 62 | } 63 | 64 | @Test fun `test ThingResponse with zero price`() { 65 | val thing = Thing( 66 | id = "000", 67 | productName = "Free Product", 68 | sellingPrice = 0.0, 69 | ) 70 | val thingData = ThingResponse.ThingData(thing = thing) 71 | val response = ThingResponse(data = thingData) 72 | 73 | val result = response.toThing() 74 | assertNotNull(result) 75 | assertEquals("000", result?.id) 76 | assertEquals("Free Product", result?.productName) 77 | assertEquals(0.0, result?.sellingPrice) 78 | } 79 | 80 | @Test fun `test ThingResponse with empty product name`() { 81 | val thing = Thing( 82 | id = "111", 83 | productName = "", 84 | sellingPrice = 5.0, 85 | ) 86 | val thingData = ThingResponse.ThingData(thing = thing) 87 | val response = ThingResponse(data = thingData) 88 | 89 | val result = response.toThing() 90 | assertNotNull(result) 91 | assertEquals("111", result?.id) 92 | assertEquals("", result?.productName) 93 | assertEquals(5.0, result?.sellingPrice) 94 | } 95 | 96 | @Test fun `test ThingResponse with negative price`() { 97 | val thing = Thing( 98 | id = "222", 99 | productName = "Discounted Product", 100 | sellingPrice = -10.0, 101 | ) 102 | val thingData = ThingResponse.ThingData(thing = thing) 103 | val response = ThingResponse(data = thingData) 104 | 105 | val result = response.toThing() 106 | assertNotNull(result) 107 | assertEquals("222", result?.id) 108 | assertEquals("Discounted Product", result?.productName) 109 | assertEquals(-10.0, result?.sellingPrice) 110 | } 111 | 112 | @Test fun `test multiple ThingResponse objects with same data`() { 113 | val thing1 = Thing(id = "same", productName = "Same Product", sellingPrice = 15.0) 114 | val thing2 = Thing(id = "same", productName = "Same Product", sellingPrice = 15.0) 115 | 116 | val data1 = ThingResponse.ThingData(thing = thing1) 117 | val data2 = ThingResponse.ThingData(thing = thing2) 118 | 119 | val response1 = ThingResponse(data = data1) 120 | val response2 = ThingResponse(data = data2) 121 | 122 | assertEquals(response1.toThing(), response2.toThing()) 123 | } 124 | 125 | @Test fun `test ThingResponse with large price value`() { 126 | val thing = Thing( 127 | id = "999", 128 | productName = "Expensive Product", 129 | sellingPrice = 999999.99, 130 | ) 131 | val thingData = ThingResponse.ThingData(thing = thing) 132 | val response = ThingResponse(data = thingData) 133 | 134 | val result = response.toThing() 135 | assertNotNull(result) 136 | assertEquals("999", result?.id) 137 | assertEquals("Expensive Product", result?.productName) 138 | assertEquals(999999.99, result?.sellingPrice) 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /libs/http4k/src/main/kotlin/vanillakotlin/http4k/HttpServer.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.http4k 2 | 3 | import net.pearx.kasechange.toSnakeCase 4 | import org.http4k.contract.ContractRoute 5 | import org.http4k.contract.jsonschema.v3.AutoJsonToJsonSchema 6 | import org.http4k.contract.jsonschema.v3.FieldRetrieval 7 | import org.http4k.contract.jsonschema.v3.JacksonJsonNamingAnnotated 8 | import org.http4k.contract.jsonschema.v3.JacksonJsonPropertyAnnotated 9 | import org.http4k.contract.jsonschema.v3.SimpleLookup 10 | import org.http4k.core.ContentType 11 | import org.http4k.core.Filter 12 | import org.http4k.core.HttpHandler 13 | import org.http4k.core.HttpTransaction 14 | import org.http4k.core.Method 15 | import org.http4k.core.NoOp 16 | import org.http4k.core.Response 17 | import org.http4k.core.Status 18 | import org.http4k.core.then 19 | import org.http4k.filter.AllowAll 20 | import org.http4k.filter.CorsPolicy 21 | import org.http4k.filter.OriginPolicy 22 | import org.http4k.filter.ResponseFilters 23 | import org.http4k.filter.ServerFilters 24 | import org.http4k.format.ConfigurableJackson 25 | import org.http4k.routing.RoutingHttpHandler 26 | import org.http4k.server.Http4kServer 27 | import org.http4k.server.Undertow 28 | import org.http4k.server.asServer 29 | import org.slf4j.LoggerFactory 30 | import vanillakotlin.extensions.toJsonString 31 | import vanillakotlin.models.HealthMonitor 32 | import vanillakotlin.models.healthCheckAll 33 | import vanillakotlin.serde.mapper 34 | import java.lang.invoke.MethodHandles 35 | 36 | private val log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass().name) 37 | 38 | val DEFAULT_HEALTH_CHECKS: List = emptyList() 39 | val DEFAULT_ROUTE_HANDLERS: List = emptyList() 40 | val DEFAULT_CONTRACT_ROUTES: List = emptyList() 41 | val DEFAULT_CORS_MODE = CorsMode.NO_OP 42 | val DEFAULT_LOGGING_FILTER = simpleLoggingFilter() 43 | val DEFAULT_EXCEPTION_FILTER = simpleExceptionFilter() 44 | val DEFAULT_GLOBAL_FILTERS = listOf(DEFAULT_LOGGING_FILTER, DEFAULT_EXCEPTION_FILTER) 45 | const val DEFAULT_SWAGGER_PATH = "/swagger.json" 46 | const val DEFAULT_BASE_PATH = "/" 47 | const val DEFAULT_PORT = 8080 48 | const val DEFAULT_HOST = "127.0.0.1" 49 | 50 | private fun simpleLoggingFilter() = ResponseFilters.ReportHttpTransaction { tx: HttpTransaction -> 51 | log.atDebug().log { "uri=${tx.request.uri} status=${tx.response.status} elapsed_ms=${tx.duration.toMillis()}" } 52 | } 53 | 54 | private fun simpleExceptionFilter() = Filter { next -> 55 | { 56 | try { 57 | next(it) 58 | } catch (t: Throwable) { 59 | log.error("Failed to process request", t) 60 | Response(Status.INTERNAL_SERVER_ERROR) 61 | } 62 | } 63 | } 64 | 65 | internal fun autoJsonToJsonSchema(json: ConfigurableJackson) = AutoJsonToJsonSchema( 66 | json, 67 | FieldRetrieval.compose( 68 | SimpleLookup(renamingStrategy = { it.toSnakeCase() }), 69 | FieldRetrieval.compose(JacksonJsonPropertyAnnotated, JacksonJsonNamingAnnotated(json)), 70 | ), 71 | ) 72 | 73 | /** 74 | * There is an UnsafeGlobalPermissive policy given by http4k, but unfortunately the only header allowed is 75 | * content-type. For any server that is serving a UI backend this is very constricting as common Target 76 | * headers such as x-api-key, x-profile-id, and any custom headers are not allowed. Therefore, this custom 77 | * ALLOW_ALL policy is created to allow all headers, methods, and credentials. This is not recommended for 78 | * deployed use and go-proxy should be used instead for CORS, but is useful for local development. 79 | */ 80 | enum class CorsMode(val filter: Filter) { 81 | ALLOW_ALL( 82 | ServerFilters.Cors( 83 | CorsPolicy( 84 | originPolicy = OriginPolicy.AllowAll(), 85 | headers = listOf("*"), 86 | methods = Method.entries, 87 | credentials = true, 88 | ), 89 | ), 90 | ), 91 | NO_OP(Filter.NoOp), 92 | } 93 | 94 | /** 95 | * This is a small customization over one of the choices available in http4k. 96 | * The key difference is that it allows for a `host` parameter, which lets you bind to a loopback address (127.0.0.1) instead of 0.0.0.0 97 | * to prevent direct external access. This is primarily when using the go-proxy, which handles external access and auth, and then 98 | * forwards unsecured auth headers to the app. If you aren't using the go-proxy and want your application host to be directly addressable, 99 | * you would want to set the host to 0.0.0.0. Be sure that none of your endpoints need auth if you do this. 100 | */ 101 | class Server( 102 | val port: Int, 103 | val host: String, 104 | ) { 105 | fun toServer(http: HttpHandler): Http4kServer = http.asServer(Undertow(port)) 106 | } 107 | 108 | fun Filter.then(filters: List): Filter = filters.fold(this) { acc, filter -> acc.then(filter) } 109 | 110 | fun healthCheckHandler(healthMonitors: List): HttpHandler = { _ -> 111 | val healthCheckResponses = healthCheckAll(healthMonitors) 112 | val status = if (healthCheckResponses.any { !it.isHealthy }) Status.INTERNAL_SERVER_ERROR else Status.OK 113 | Response(status).body(healthCheckResponses.toJsonString()) 114 | } 115 | 116 | val CatchAllFailure = Filter { next -> 117 | { 118 | try { 119 | next(it) 120 | } catch (throwable: Throwable) { 121 | Response(Status.INTERNAL_SERVER_ERROR) 122 | .header("Content-Type", ContentType.APPLICATION_JSON.toHeaderValue()) 123 | .body(mapper.writeValueAsString(mapOf("error" to throwable.message))) 124 | } 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /apps/kafka-transformer/src/test/kotlin/vanillakotlin/kafkatransformer/FavoriteThingsEventHandlerTest.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.kafkatransformer 2 | 3 | import com.fasterxml.jackson.core.JsonParseException 4 | import io.kotest.assertions.assertSoftly 5 | import io.kotest.assertions.fail 6 | import io.kotest.assertions.throwables.shouldThrow 7 | import io.kotest.matchers.shouldBe 8 | import io.kotest.matchers.shouldNotBe 9 | import org.junit.jupiter.api.Test 10 | import vanillakotlin.kafka.models.KafkaMessage 11 | import vanillakotlin.models.UserFavoriteThing 12 | import vanillakotlin.random.randomInt 13 | import vanillakotlin.random.randomLong 14 | import vanillakotlin.random.randomString 15 | import vanillakotlin.random.randomThing 16 | import vanillakotlin.random.randomUsername 17 | import vanillakotlin.serde.mapper 18 | import java.time.Instant 19 | 20 | class FavoriteThingsEventHandlerTest { 21 | 22 | @Test fun `a populated message should be enriched and forwarded`() { 23 | val userName = randomUsername() 24 | val thingIdentifier = randomThing() 25 | val thing = buildTestThing(thingIdentifier = thingIdentifier) 26 | val kafkaMessage = buildTestMessage(username = userName, thingIdentifier = thingIdentifier) 27 | val kafkaMessageKey = "$userName:$thingIdentifier" 28 | 29 | val handler = FavoriteThingsEventHandler( 30 | getThingDetails = { thing }, 31 | ) 32 | 33 | val result = handler.transform(kafkaMessage) 34 | assertSoftly(result.messages) { 35 | size shouldBe 1 36 | val outputMessage = first().kafkaOutputMessage 37 | outputMessage.key shouldBe kafkaMessageKey 38 | outputMessage.value shouldBe UserFavoriteThing(userName = userName, thingIdentifier = thingIdentifier) 39 | outputMessage.headers shouldNotBe null 40 | } 41 | } 42 | 43 | @Test fun `a message with a null body should forward a message with the same key and null body`() { 44 | // A message with a null body would correlate with the system deleting a user favorite. 45 | // The message key and any headers would still be populated, but the body would be null. 46 | 47 | val userName = randomUsername() 48 | val thingIdentifier = randomThing() 49 | val kafkaMessage = buildTestMessage(username = userName, thingIdentifier = thingIdentifier).copy(body = null) 50 | val kafkaMessageKey = "$userName:$thingIdentifier" 51 | 52 | val handler = FavoriteThingsEventHandler( 53 | getThingDetails = { fail("unexpected method call") }, 54 | ) 55 | 56 | val result = handler.transform(kafkaMessage) 57 | assertSoftly(result.messages) { 58 | size shouldBe 1 59 | val outputMessage = first().kafkaOutputMessage 60 | outputMessage.key shouldBe kafkaMessageKey 61 | outputMessage.value shouldBe null 62 | outputMessage.headers shouldNotBe null 63 | } 64 | } 65 | 66 | @Test fun `invalid message body should throw an exception`() { 67 | // This test shows fail-fast behavior - malformed JSON will cause the app to shut down 68 | val kafkaMessage = buildTestMessage().copy(body = "oh no".toByteArray()) 69 | 70 | val handler = FavoriteThingsEventHandler( 71 | getThingDetails = { fail("unexpected method call") }, 72 | ) 73 | 74 | shouldThrow { 75 | handler.transform(kafkaMessage) 76 | } 77 | } 78 | 79 | @Test fun `null key should throw exception`() { 80 | val kafkaMessage = buildTestMessage().copy(key = null) 81 | 82 | val handler = FavoriteThingsEventHandler( 83 | getThingDetails = { fail("unexpected method call") }, 84 | ) 85 | 86 | shouldThrow { 87 | handler.transform(kafkaMessage) 88 | } 89 | } 90 | 91 | @Test fun `malformed key should throw exception`() { 92 | val kafkaMessage = buildTestMessage().copy(key = "malformed-key-without-colon") 93 | 94 | val handler = FavoriteThingsEventHandler( 95 | getThingDetails = { fail("unexpected method call") }, 96 | ) 97 | 98 | shouldThrow { 99 | handler.transform(kafkaMessage) 100 | } 101 | } 102 | 103 | @Test fun `empty username in key should throw exception`() { 104 | val kafkaMessage = buildTestMessage().copy(key = ":${randomThing()}") 105 | 106 | val handler = FavoriteThingsEventHandler( 107 | getThingDetails = { fail("unexpected method call") }, 108 | ) 109 | 110 | shouldThrow { 111 | handler.transform(kafkaMessage) 112 | } 113 | } 114 | 115 | @Test fun `thing not found should throw exception`() { 116 | val kafkaMessage = buildTestMessage() 117 | 118 | val handler = FavoriteThingsEventHandler( 119 | getThingDetails = { null }, // Return null to simulate thing not found 120 | ) 121 | 122 | shouldThrow { 123 | handler.transform(kafkaMessage) 124 | } 125 | } 126 | 127 | private fun buildTestMessage( 128 | username: String = randomUsername(), 129 | thingIdentifier: String = randomThing(), 130 | ): KafkaMessage = KafkaMessage( 131 | broker = randomString(), 132 | topic = randomString(), 133 | key = "$username:$thingIdentifier", 134 | partition = randomInt(1..3), 135 | offset = randomLong(), 136 | headers = emptyMap(), 137 | timestamp = Instant.now().toEpochMilli(), 138 | body = mapper.writeValueAsBytes(thingIdentifier), 139 | ) 140 | } 141 | -------------------------------------------------------------------------------- /libs/kafka/src/test/kotlin/vanillakotlin/kafka/producer/PartitionCalculatorTest.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.kafka.producer 2 | 3 | import org.junit.jupiter.api.Assertions.assertEquals 4 | import org.junit.jupiter.api.Assertions.assertNull 5 | import org.junit.jupiter.api.Assertions.assertTrue 6 | import org.junit.jupiter.api.Test 7 | 8 | class PartitionCalculatorTest { 9 | 10 | @Test fun `should return null for null partition key`() { 11 | val result = PartitionCalculator.partitionFor(null, 10) 12 | assertNull(result) 13 | } 14 | 15 | @Test fun `should calculate partition for given key and partition count`() { 16 | val partitionKey = "test-key" 17 | val partitionCount = 5 18 | 19 | val partition = PartitionCalculator.partitionFor(partitionKey, partitionCount) 20 | 21 | assertTrue(partition?.let { it >= 0 } ?: false, "Partition should be non-negative") 22 | assertTrue(partition?.let { it < partitionCount } ?: false, "Partition should be less than partition count") 23 | } 24 | 25 | @Test fun `should consistently return same partition for same key`() { 26 | val partitionKey = "consistent-key" 27 | val partitionCount = 10 28 | 29 | val partition1 = PartitionCalculator.partitionFor(partitionKey, partitionCount) 30 | val partition2 = PartitionCalculator.partitionFor(partitionKey, partitionCount) 31 | 32 | assertEquals(partition1, partition2, "Same key should always map to same partition") 33 | } 34 | 35 | @Test fun `should distribute keys across partitions`() { 36 | val partitionCount = 5 37 | val partitions = mutableSetOf() 38 | 39 | // Generate many keys to test distribution 40 | repeat(100) { i -> 41 | val partition = PartitionCalculator.partitionFor("key-$i", partitionCount) 42 | partition?.let { partitions.add(it) } 43 | } 44 | 45 | // Should use multiple partitions (at least 3 out of 5) 46 | assertTrue(partitions.size >= 3, "Keys should be distributed across multiple partitions") 47 | 48 | // All partitions should be within valid range 49 | partitions.forEach { partition -> 50 | assertTrue(partition >= 0 && partition < partitionCount) 51 | } 52 | } 53 | 54 | @Test fun `should handle different partition counts`() { 55 | val partitionKey = "test-key" 56 | 57 | val partition3 = PartitionCalculator.partitionFor(partitionKey, 3) 58 | val partition10 = PartitionCalculator.partitionFor(partitionKey, 10) 59 | val partition100 = PartitionCalculator.partitionFor(partitionKey, 100) 60 | 61 | assertTrue(partition3?.let { it < 3 } ?: false) 62 | assertTrue(partition10?.let { it < 10 } ?: false) 63 | assertTrue(partition100?.let { it < 100 } ?: false) 64 | } 65 | 66 | @Test fun `should handle single partition`() { 67 | val partitionKey = "single-partition-key" 68 | 69 | val partition = PartitionCalculator.partitionFor(partitionKey, 1) 70 | 71 | assertEquals(0, partition, "Single partition should always be 0") 72 | } 73 | 74 | @Test fun `should handle empty string key`() { 75 | val partition = PartitionCalculator.partitionFor("", 5) 76 | 77 | assertTrue(partition?.let { it >= 0 } ?: false, "Empty string should still produce valid partition") 78 | assertTrue(partition?.let { it < 5 } ?: false, "Empty string partition should be within range") 79 | } 80 | 81 | @Test fun `should handle special characters in key`() { 82 | val specialKeys = listOf( 83 | "key-with-dashes", 84 | "key_with_underscores", 85 | "key.with.dots", 86 | "key@with@symbols", 87 | "key with spaces", 88 | "key/with/slashes", 89 | ) 90 | 91 | specialKeys.forEach { key -> 92 | val partition = PartitionCalculator.partitionFor(key, 10) 93 | assertTrue( 94 | partition?.let { it >= 0 && it < 10 } ?: false, 95 | "Special character key '$key' should produce valid partition", 96 | ) 97 | } 98 | } 99 | 100 | @Test fun `should handle unicode characters`() { 101 | val unicodeKeys = listOf( 102 | "키-한국어", 103 | "clé-français", 104 | "ключ-русский", 105 | "键-中文", 106 | "🔑-emoji", 107 | ) 108 | 109 | unicodeKeys.forEach { key -> 110 | val partition = PartitionCalculator.partitionFor(key, 7) 111 | assertTrue( 112 | partition?.let { it >= 0 && it < 7 } ?: false, 113 | "Unicode key '$key' should produce valid partition", 114 | ) 115 | } 116 | } 117 | 118 | @Test fun `should demonstrate consistent hashing behavior`() { 119 | val partitionCount = 8 120 | val keyToPartitionMap = mutableMapOf() 121 | 122 | // Map a set of keys to partitions 123 | val testKeys = (1..20).map { "user-$it" } 124 | testKeys.forEach { key -> 125 | val partition = PartitionCalculator.partitionFor(key, partitionCount) 126 | partition?.let { 127 | keyToPartitionMap[key] = it 128 | } 129 | } 130 | 131 | // Verify consistency over multiple calls 132 | repeat(5) { 133 | testKeys.forEach { key -> 134 | val partition = PartitionCalculator.partitionFor(key, partitionCount) 135 | partition?.let { 136 | assertEquals( 137 | keyToPartitionMap[key], 138 | it, 139 | "Key '$key' should always map to same partition", 140 | ) 141 | } 142 | } 143 | } 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /gradle/libs.versions.toml: -------------------------------------------------------------------------------- 1 | [versions] 2 | flyway = "10.11.0" 3 | hoplite = "2.9.0" 4 | http4k = "6.15.0.1" 5 | jackson = "2.19.1" 6 | jdbi = "3.49.5" 7 | jupiter = "5.13.1" 8 | kasechange = "1.4.1" 9 | kotest = "5.9.1" 10 | kotlin-core = "2.1.21" 11 | kotlin-coroutines = "1.10.2" 12 | ktlint = "0.48.2" 13 | okhttp = "4.12.0" 14 | postgresql = "42.7.7" 15 | resilience4j = "2.3.0" 16 | spotless = "7.0.4" 17 | otel = "1.51.0" 18 | slf4j = "2.0.17" 19 | logback = "1.5.18" 20 | logback-encoder = "8.1" 21 | kafka = "3.9.0" 22 | rocksdb = "10.2.1" 23 | versions = "0.52.0" 24 | 25 | [libraries] 26 | flyway-database-postgresql = { module = "org.flywaydb:flyway-database-postgresql", version.ref = "flyway" } 27 | 28 | postgresql = { module = "org.postgresql:postgresql", version.ref = "postgresql" } 29 | 30 | hoplite-core = { module = "com.sksamuel.hoplite:hoplite-core", version.ref = "hoplite" } 31 | hoplite-hocon = { module = "com.sksamuel.hoplite:hoplite-hocon", version.ref = "hoplite" } 32 | 33 | http4k-core = { module = "org.http4k:http4k-core", version.ref = "http4k" } 34 | http4k-open-api = { module = "org.http4k:http4k-api-openapi", version.ref = "http4k" } 35 | http4k-format-jackson = { module = "org.http4k:http4k-format-jackson", version.ref = "http4k" } 36 | http4k-server-undertow = { module = "org.http4k:http4k-server-undertow", version.ref = "http4k" } 37 | http4k-websocket-client = { module = "org.http4k:http4k-client-websocket", version.ref = "http4k" } 38 | 39 | jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson" } 40 | jackson-jdk8 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jdk8", version.ref = "jackson" } 41 | jackson-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", version.ref = "jackson" } 42 | jackson-kotlin = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version.ref = "jackson" } 43 | 44 | jdbi-core = { module = "org.jdbi:jdbi3-core", version.ref = "jdbi" } 45 | jdbi-kotlin = { module = "org.jdbi:jdbi3-kotlin", version.ref = "jdbi" } 46 | jdbi-postgres = { module = "org.jdbi:jdbi3-postgres", version.ref = "jdbi" } 47 | 48 | jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "jupiter" } 49 | jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "jupiter" } 50 | jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "jupiter" } 51 | 52 | kasechange = { module = "net.pearx.kasechange:kasechange", version.ref = "kasechange" } 53 | 54 | kotest-assertions-core = { module = "io.kotest:kotest-assertions-core", version.ref = "kotest" } 55 | kotest-assertions-json = { module = "io.kotest:kotest-assertions-json", version.ref = "kotest" } 56 | kotest-extensions = { module = "io.kotest:kotest-extensions", version.ref = "kotest" } 57 | kotest-runner-junit5 = { module = "io.kotest:kotest-runner-junit5", version.ref = "kotest" } 58 | kotest-property = { module = "io.kotest:kotest-property-jvm", version.ref = "kotest" } 59 | 60 | kotlin-coroutines = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "kotlin-coroutines" } 61 | 62 | okhttp = { module = "com.squareup.okhttp3:okhttp", version.ref = "okhttp" } 63 | okhttp-mock-server = { module = "com.squareup.okhttp3:mockwebserver", version.ref = "okhttp" } 64 | 65 | resilience4j-kotlin = { module = "io.github.resilience4j:resilience4j-kotlin", version.ref = "resilience4j" } 66 | resilience4j-timelimiter = { module = "io.github.resilience4j:resilience4j-timelimiter", version.ref = "resilience4j" } 67 | resilience4j-retry = { module = "io.github.resilience4j:resilience4j-retry", version.ref = "resilience4j" } 68 | 69 | otel-api = { module = "io.opentelemetry:opentelemetry-api", version.ref = "otel" } 70 | otel-sdk = { module = "io.opentelemetry:opentelemetry-sdk", version.ref = "otel" } 71 | otel-autoconfigure = { module = "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure", version.ref = "otel" } 72 | otel-logging = { module = "io.opentelemetry:opentelemetry-exporter-logging", version.ref = "otel" } 73 | otel-logging-otlp = { module = "io.opentelemetry:opentelemetry-exporter-logging-otlp", version.ref = "otel" } 74 | otel-sdk-metrics = { module = "io.opentelemetry:opentelemetry-sdk-metrics", version.ref = "otel" } 75 | otel-sdk-testing = { module = "io.opentelemetry:opentelemetry-sdk-testing", version.ref = "otel" } 76 | 77 | slf4j = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } 78 | logback-classic = { module = "ch.qos.logback:logback-classic", version.ref = "logback" } 79 | logback-encoder = { module = "net.logstash.logback:logstash-logback-encoder", version.ref = "logback-encoder" } 80 | 81 | kafka-client = { module = "org.apache.kafka:kafka-clients", version.ref = "kafka" } 82 | 83 | rocksdb = { module = "org.rocksdb:rocksdbjni", version.ref = "rocksdb" } 84 | 85 | [bundles] 86 | hoplite = [ 87 | "hoplite-core", 88 | "hoplite-hocon", 89 | ] 90 | 91 | http4k = [ 92 | "http4k-core", 93 | "http4k-open-api", 94 | "http4k-format-jackson", 95 | "http4k-server-undertow", 96 | "http4k-websocket-client", 97 | ] 98 | 99 | jackson = [ 100 | "jackson-databind", 101 | "jackson-jdk8", 102 | "jackson-jsr310", 103 | "jackson-kotlin", 104 | ] 105 | 106 | jdbi = [ 107 | "jdbi-core", 108 | "jdbi-kotlin", 109 | "jdbi-postgres", 110 | ] 111 | 112 | jupiter = [ 113 | "jupiter-api", 114 | "jupiter-engine", 115 | "jupiter-params", 116 | ] 117 | 118 | testing = [ 119 | "jupiter-api", 120 | "jupiter-engine", 121 | "jupiter-params", 122 | "kotest-assertions-core", 123 | "kotest-runner-junit5", 124 | "kotest-property", 125 | "kotest-assertions-json", 126 | "kotest-extensions", 127 | ] 128 | 129 | resilience4j = [ 130 | "resilience4j-timelimiter", 131 | "resilience4j-kotlin" 132 | ] 133 | 134 | otel-testing = [ 135 | "otel-sdk", 136 | "otel-autoconfigure", 137 | "otel-logging", 138 | "otel-logging-otlp", 139 | "otel-sdk-metrics", 140 | "otel-sdk-testing" 141 | ] 142 | 143 | logging = [ 144 | "logback-classic", 145 | "logback-encoder", 146 | "slf4j" 147 | ] 148 | 149 | [plugins] 150 | flyway = { id = "org.flywaydb.flyway", version.ref = "flyway" } 151 | versions = { id = "com.github.ben-manes.versions", version.ref = "versions" } 152 | -------------------------------------------------------------------------------- /apps/kafka-transformer/src/test/kotlin/vanillakotlin/kafkatransformer/AppTest.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.kafkatransformer 2 | 3 | import io.kotest.assertions.assertSoftly 4 | import io.kotest.extensions.system.withSystemProperties 5 | import io.kotest.matchers.shouldBe 6 | import io.kotest.matchers.shouldNotBe 7 | import okhttp3.mockwebserver.Dispatcher 8 | import okhttp3.mockwebserver.MockResponse 9 | import okhttp3.mockwebserver.MockWebServer 10 | import okhttp3.mockwebserver.RecordedRequest 11 | import org.apache.kafka.clients.admin.AdminClient 12 | import org.apache.kafka.clients.admin.NewTopic 13 | import org.http4k.client.JavaHttpClient 14 | import org.http4k.core.Method 15 | import org.http4k.core.Request 16 | import org.http4k.core.Status 17 | import org.intellij.lang.annotations.Language 18 | import org.junit.jupiter.api.AfterAll 19 | import org.junit.jupiter.api.BeforeAll 20 | import org.junit.jupiter.api.Test 21 | import org.junit.jupiter.api.TestInstance 22 | import vanillakotlin.kafka.MockMetricsPublisher 23 | import vanillakotlin.kafka.collectMessages 24 | import vanillakotlin.kafka.models.KafkaOutputMessage 25 | import vanillakotlin.kafka.producer.AGENT_HEADER_NAME 26 | import vanillakotlin.kafka.producer.KafkaProducer 27 | import vanillakotlin.kafka.provenance.PROVENANCES_HEADER_NAME 28 | import vanillakotlin.kafka.provenance.SPAN_ID_HEADER_NAME 29 | import vanillakotlin.models.UserFavoriteThing 30 | import vanillakotlin.random.randomString 31 | import vanillakotlin.random.randomThing 32 | import vanillakotlin.random.randomUsername 33 | import vanillakotlin.serde.mapper 34 | 35 | @TestInstance(TestInstance.Lifecycle.PER_CLASS) 36 | class AppTest { 37 | private var app: App? = null 38 | private val mockThingServer = MockWebServer() 39 | 40 | // Use dynamic topic names for test isolation 41 | private val sourceTopicName = randomString() 42 | private val sinkTopicName = randomString() 43 | private val broker = if (System.getenv().containsKey("CI")) "kafka:9092" else "localhost:9092" 44 | 45 | private val thingIdentifier = randomThing() 46 | private val testThing = buildTestThing(thingIdentifier = thingIdentifier) 47 | 48 | @Language("JSON") 49 | private val sampleThingGraphQlResponse = 50 | """ 51 | { 52 | "data": { 53 | "thing": { 54 | "id": "${testThing.id}", 55 | "product_name": "${testThing.productName}", 56 | "selling_price": ${testThing.sellingPrice} 57 | } 58 | } 59 | } 60 | """.trimIndent() 61 | 62 | @BeforeAll fun beforeAll() { 63 | // mock the thing endpoint to return the same sample responses based on the request path 64 | val dispatcher: Dispatcher = 65 | object : Dispatcher() { 66 | override fun dispatch(request: RecordedRequest): MockResponse { 67 | return when (request.path?.substringBefore("?")) { 68 | "/things/v4/graphql/compact/thing" -> return MockResponse().setResponseCode(200).setBody(sampleThingGraphQlResponse) 69 | else -> MockResponse().setResponseCode(404) 70 | } 71 | } 72 | } 73 | mockThingServer.dispatcher = dispatcher 74 | mockThingServer.start() 75 | 76 | // create dynamic source and sink topics 77 | val adminClient = AdminClient.create(mapOf("bootstrap.servers" to broker)) 78 | adminClient.createTopics(listOf(NewTopic(sourceTopicName, 1, 1))) 79 | adminClient.createTopics(listOf(NewTopic(sinkTopicName, 1, 1))) 80 | 81 | // Override configuration with dynamic topic names 82 | val overriddenConfiguration = mapOf( 83 | "config.override.http.client.thing.gateway.baseUrl" to "http://localhost:${mockThingServer.port}", 84 | "config.override.kafka.consumer.topics" to sourceTopicName, 85 | "config.override.kafka.producer.topic" to sinkTopicName, 86 | ) 87 | 88 | withSystemProperties(overriddenConfiguration) { 89 | app = App() 90 | app?.start() 91 | } 92 | } 93 | 94 | @AfterAll fun afterAll() { 95 | app?.close() 96 | mockThingServer.close() 97 | } 98 | 99 | @Test fun `health check ok`() { 100 | val request = Request(Method.GET, "http://localhost:${app?.httpServerPort}/health") 101 | 102 | val response = JavaHttpClient()(request) 103 | 104 | response.status shouldBe Status.OK 105 | } 106 | 107 | @Test fun `the kafkatransformer should consume, process, and forward a valid message successfully`() { 108 | // The beforeAll test setup function created a new source and sink topic, and configured the application to use those. 109 | // So at this point, the kafkatransformer's event handler should be running and waiting for a message. 110 | 111 | val metricsPublisher = MockMetricsPublisher() 112 | 113 | // produce a message on source topic 114 | val producer = KafkaProducer( 115 | config = KafkaProducer.Config( 116 | broker = broker, 117 | topic = sourceTopicName, 118 | ), 119 | publishTimerMetric = metricsPublisher, 120 | ) 121 | producer.start() 122 | 123 | val userName = randomUsername() 124 | 125 | producer.send( 126 | KafkaOutputMessage( 127 | key = "$userName:$thingIdentifier", 128 | value = thingIdentifier, 129 | ), 130 | ) 131 | 132 | val expectedUserFavoriteThing = UserFavoriteThing( 133 | userName = userName, 134 | thingIdentifier = testThing.id, 135 | ) 136 | 137 | val receivedMessages = collectMessages( 138 | broker = broker, 139 | topic = sinkTopicName, 140 | metricsPublisher = metricsPublisher, 141 | ) 142 | 143 | receivedMessages.size shouldBe 1 144 | assertSoftly(receivedMessages.first()) { 145 | this.key shouldBe "$userName:$thingIdentifier" 146 | this.body shouldBe mapper.writeValueAsBytes(expectedUserFavoriteThing) 147 | this.timestamp shouldNotBe null 148 | this.offset shouldBe 0 149 | this.partition shouldBe 0 150 | this.headers.size shouldBe 3 151 | this.headers[SPAN_ID_HEADER_NAME] shouldNotBe null 152 | this.headers[AGENT_HEADER_NAME] shouldNotBe null 153 | this.headers[PROVENANCES_HEADER_NAME] shouldNotBe null 154 | } 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /libs/db/src/main/kotlin/vanillakotlin/db/repository/FavoriteThingRepository.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.db.repository 2 | 3 | import org.intellij.lang.annotations.Language 4 | import org.jdbi.v3.core.Jdbi 5 | import org.jdbi.v3.core.mapper.RowMapper 6 | import org.jdbi.v3.core.statement.StatementContext 7 | import vanillakotlin.models.FavoriteThing 8 | import vanillakotlin.models.ThingIdentifier 9 | import vanillakotlin.models.buildDeletedOutbox 10 | import java.sql.ResultSet 11 | 12 | /** SAM interface for [FavoriteThingRepository.upsert] */ 13 | fun interface UpsertFavoriteThing { 14 | operator fun invoke(favorite: FavoriteThing): FavoriteThing 15 | } 16 | 17 | /** SAM interface for [FavoriteThingRepository.deleteItem] */ 18 | fun interface DeleteFavoriteThing { 19 | operator fun invoke(thingIdentifier: ThingIdentifier): Int 20 | } 21 | 22 | /** SAM interface for [FavoriteThingRepository.findAll] */ 23 | fun interface FindAllFavoriteThings { 24 | operator fun invoke(): List 25 | } 26 | 27 | /** SAM interface for [FavoriteThingRepository.addToBatch] */ 28 | fun interface AddFavoriteThingToBatch { 29 | operator fun invoke(favoriteThing: FavoriteThing): Unit 30 | } 31 | 32 | /** SAM interface for [FavoriteThingRepository.runBatch] */ 33 | fun interface RunFavoriteThingBatch { 34 | operator fun invoke(): Int 35 | } 36 | 37 | // This class encapsulates all the SQL operations for the FavoriteThing entity. 38 | class FavoriteThingRepository(private val jdbi: Jdbi) { 39 | 40 | val upsert = UpsertFavoriteThing { favorite -> 41 | jdbi.inTransaction { handle -> 42 | val favoriteThing = handle.createQuery( 43 | """ 44 | INSERT INTO favorite_thing (thing) 45 | VALUES (:thing) 46 | ON CONFLICT (thing) DO UPDATE SET 47 | updated_ts = CURRENT_TIMESTAMP 48 | RETURNING * 49 | """, 50 | ) 51 | .bind("thing", favorite.thingIdentifier) 52 | .mapTo(FavoriteThing::class.java) 53 | .one() 54 | 55 | insertOutbox(handle, favoriteThing.buildOutbox()) 56 | 57 | favoriteThing 58 | } 59 | } 60 | 61 | val deleteItem = DeleteFavoriteThing { thingIdentifier -> 62 | jdbi.withHandle { handle -> 63 | val rowCount = handle.createUpdate("DELETE FROM favorite_thing WHERE thing = :thing") 64 | .bind("thing", thingIdentifier) 65 | .execute() 66 | 67 | if (rowCount > 0) { 68 | insertOutbox(handle, buildDeletedOutbox(thingIdentifier)) 69 | } 70 | 71 | rowCount 72 | } 73 | } 74 | 75 | val findAll = FindAllFavoriteThings { 76 | jdbi.withHandle, Exception> { handle -> 77 | handle.createQuery("SELECT * FROM favorite_thing") 78 | .mapTo(FavoriteThing::class.java) 79 | .list() 80 | } 81 | } 82 | 83 | // The batch here is just a list because we don't need to worry about concurrency in this case - once the consumer has consumed an 84 | // entire batch of records, the thread will wait until the running of the batch has been completed. If we were to run this in an 85 | // environment where concurrency was an issue, we would need to use a thread-safe data structure or synchronize access to the batch. 86 | private val batch = mutableListOf() 87 | 88 | val addToBatch = AddFavoriteThingToBatch { favoriteThing -> 89 | batch.add(favoriteThing) 90 | } 91 | 92 | // This function will run two batch statements. The first will batch update the table with the updated timestamp, then it will see if 93 | // the entire batch was updates, and if not, will run a batch insert, doing nothing when there are conflicts. The reason this pattern is 94 | // necessary is that if we run an upsert for the batch, the postgresql driver will not actually run them as a batch, and will instead 95 | // turn the batch into a series of individual upserts. This is a workaround to that issue. 96 | val runBatch = RunFavoriteThingBatch { 97 | val batchSize = batch.size 98 | if (batch.isEmpty()) return@RunFavoriteThingBatch 0 99 | 100 | @Language("SQL") 101 | val updateQuery = 102 | """ 103 | UPDATE favorite_thing 104 | SET updated_ts = CURRENT_TIMESTAMP 105 | WHERE thing = ? 106 | """.trimIndent() 107 | 108 | @Language("SQL") 109 | val insertQuery = 110 | """ 111 | INSERT INTO favorite_thing (thing) 112 | VALUES (?) 113 | ON CONFLICT ON CONSTRAINT unique_thing DO NOTHING 114 | """.trimIndent() 115 | 116 | @Language("SQL") 117 | val deleteQuery = 118 | """ 119 | DELETE FROM favorite_thing 120 | WHERE thing = ? 121 | """.trimIndent() 122 | 123 | jdbi.useTransaction { handle -> 124 | val updateBatch = handle.prepareBatch(updateQuery) 125 | val insertBatch = handle.prepareBatch(insertQuery) 126 | val deleteBatch = handle.prepareBatch(deleteQuery) 127 | 128 | batch.forEach { favoriteThing -> 129 | if (favoriteThing.isDeleted) { 130 | deleteBatch.bind(0, favoriteThing.thingIdentifier).add() 131 | } else { 132 | updateBatch.bind(0, favoriteThing.thingIdentifier).add() 133 | insertBatch.bind(0, favoriteThing.thingIdentifier).add() 134 | } 135 | } 136 | 137 | val updateResults = updateBatch.execute() 138 | 139 | // Run delete batch for deleted items 140 | deleteBatch.execute() 141 | 142 | // Run insert batch only if there are rows that were not updated 143 | if (updateResults.sum() < batch.size) { 144 | insertBatch.execute() 145 | } 146 | 147 | batch.clear() 148 | } 149 | 150 | batchSize 151 | } 152 | } 153 | 154 | // Custom row mapper for FavoriteThing 155 | class FavoriteThingMapper : RowMapper { 156 | override fun map( 157 | rs: ResultSet, 158 | ctx: StatementContext, 159 | ): FavoriteThing = FavoriteThing( 160 | id = rs.getLong("id"), 161 | thingIdentifier = rs.getString("thing"), 162 | createdTs = rs.getTimestamp("created_ts").toLocalDateTime(), 163 | updatedTs = rs.getTimestamp("updated_ts").toLocalDateTime(), 164 | ) 165 | } 166 | -------------------------------------------------------------------------------- /apps/api/src/test/kotlin/vanillakotlin/api/AppTest.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.api 2 | 3 | import com.fasterxml.jackson.module.kotlin.readValue 4 | import io.kotest.extensions.system.withSystemProperties 5 | import io.kotest.matchers.collections.shouldBeIn 6 | import io.kotest.matchers.shouldBe 7 | import org.http4k.client.JavaHttpClient 8 | import org.http4k.core.HttpHandler 9 | import org.http4k.core.Method 10 | import org.http4k.core.Request 11 | import org.http4k.core.Status 12 | import org.junit.jupiter.api.AfterAll 13 | import org.junit.jupiter.api.BeforeAll 14 | import org.junit.jupiter.api.BeforeEach 15 | import org.junit.jupiter.api.Test 16 | import org.junit.jupiter.api.TestInstance 17 | import vanillakotlin.config.loadConfig 18 | import vanillakotlin.db.createJdbi 19 | import vanillakotlin.random.randomUsername 20 | import vanillakotlin.serde.mapper 21 | 22 | // This test contains setup and teardown functions that are meant to run once for this class (not for each test) 23 | // The `PER_CLASS` annotation enables that in combination with the @BeforeAll and @AfterAll annotated functions 24 | @TestInstance(TestInstance.Lifecycle.PER_CLASS) 25 | class AppTest { 26 | private val mockThingServer = MockThingServer() 27 | private var app: App? = null 28 | 29 | @BeforeEach fun beforeEach() { 30 | // Clean up database before each test 31 | val config = loadConfig() 32 | val jdbi = createJdbi(config.db, mapper) 33 | jdbi.useHandle { handle -> 34 | handle.execute("DELETE FROM favorite_thing") 35 | } 36 | } 37 | 38 | @BeforeAll fun beforeAll() { 39 | // override the http client baseUrl to the mocked server's address. The app should use that when configured. 40 | withSystemProperties( 41 | mapOf( 42 | "config.override.http.client.thing.gateway.baseUrl" to "http://localhost:${mockThingServer.port}", 43 | ), 44 | ) { 45 | app = App() 46 | app?.start() 47 | } 48 | } 49 | 50 | @AfterAll fun afterAll() { 51 | mockThingServer.close() 52 | app?.close() 53 | } 54 | 55 | @Test fun `health check ok`() { 56 | val request = Request(Method.GET, "http://localhost:${app?.httpServerPort}/health") 57 | 58 | val response = JavaHttpClient()(request) 59 | 60 | response.status shouldBe Status.OK 61 | } 62 | 63 | @Test fun `post favorite_things - missing auth`() { 64 | val request = Request(Method.POST, "http://localhost:${app?.httpServerPort}/api/v1/favorite_things/1") 65 | 66 | val response = JavaHttpClient()(request) 67 | 68 | response.status shouldBe Status.UNAUTHORIZED 69 | } 70 | 71 | @Test fun `delete favorite_things - missing auth`() { 72 | val request = Request(Method.DELETE, "http://localhost:${app?.httpServerPort}/api/v1/favorite_things/1") 73 | 74 | val response = JavaHttpClient()(request) 75 | 76 | response.status shouldBe Status.UNAUTHORIZED 77 | } 78 | 79 | @Test fun `get favorite_things - missing auth`() { 80 | val request = Request(Method.GET, "http://localhost:${app?.httpServerPort}/api/v1/favorite_things") 81 | 82 | val response = JavaHttpClient()(request) 83 | 84 | response.status shouldBe Status.UNAUTHORIZED 85 | } 86 | 87 | @Test fun `favorite_things - simple operations`() { 88 | val request = Request(Method.GET, "http://localhost:${app?.httpServerPort}/api/v1/favorite_things").addAuth() 89 | 90 | val response = JavaHttpClient()(request) 91 | 92 | response.status shouldBe Status.OK 93 | } 94 | 95 | @Test fun `admin get favorite_things authorized`() { 96 | val request = Request(Method.GET, "http://localhost:${app?.httpServerPort}/api/v1/admin/favorite_things?user_name=x").addAdminAuth() 97 | 98 | val response = JavaHttpClient()(request) 99 | 100 | response.status shouldBe Status.OK 101 | } 102 | 103 | @Test fun `admin get favorite_things missing required admin auth`() { 104 | val request = Request(Method.GET, "http://localhost:${app?.httpServerPort}/api/v1/admin/favorite_things").addAuth() 105 | 106 | val response = JavaHttpClient()(request) 107 | 108 | response.status shouldBe Status.UNAUTHORIZED 109 | } 110 | 111 | @Test fun `favorite_things - normal operations`() { 112 | val userName = randomUsername() 113 | val testThing1 = "thing1" 114 | val testThing2 = "thing2" 115 | val client = JavaHttpClient() 116 | 117 | testReadFavorite(userName, emptySet(), client) 118 | 119 | testPost(userName, testThing1, client) 120 | 121 | testReadFavorite(userName, setOf(testThing1), client) 122 | 123 | testPost(userName, testThing2, client) 124 | 125 | testReadFavorite(userName, setOf(testThing1, testThing2), client) 126 | 127 | testDelete(userName, testThing1, client) 128 | 129 | testReadFavorite(userName, setOf(testThing2), client) 130 | 131 | testDelete(userName, testThing2, client) 132 | 133 | testReadFavorite(userName, emptySet(), client) 134 | } 135 | 136 | private fun testReadFavorite( 137 | userName: String, 138 | things: Set, 139 | client: HttpHandler = JavaHttpClient(), 140 | ) { 141 | val request = Request(Method.GET, "http://localhost:${app?.httpServerPort}/api/v1/favorite_things_ids").addAuth(userName) 142 | 143 | val response = client(request) 144 | 145 | response.status shouldBe Status.OK 146 | val responseBody = response.body.toString() 147 | val getResult = mapper.readValue>(responseBody).toSet() 148 | getResult.size shouldBe things.size 149 | getResult.forEach { 150 | it shouldBeIn things 151 | } 152 | } 153 | 154 | private fun testPost( 155 | userName: String, 156 | thing: String, 157 | client: HttpHandler = JavaHttpClient(), 158 | ) { 159 | val request = 160 | Request(Method.POST, "http://localhost:${app?.httpServerPort}/api/v1/favorite_things/$thing") 161 | .addAuth(userName) 162 | val response = client(request) 163 | 164 | response.status shouldBe Status.OK 165 | } 166 | 167 | private fun testDelete( 168 | userName: String, 169 | thing: String, 170 | client: HttpHandler = JavaHttpClient(), 171 | ) { 172 | val request = 173 | Request(Method.DELETE, "http://localhost:${app?.httpServerPort}/api/v1/favorite_things/$thing") 174 | .addAuth(userName) 175 | val response = client(request) 176 | 177 | response.status shouldBe Status.OK 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /libs/client/src/test/kotlin/vanillakotlin/http/interceptors/ModelsTest.kt: -------------------------------------------------------------------------------- 1 | package vanillakotlin.http.interceptors 2 | 3 | import org.junit.jupiter.api.Assertions.assertEquals 4 | import org.junit.jupiter.api.Assertions.assertTrue 5 | import org.junit.jupiter.api.Test 6 | import org.junit.jupiter.api.assertThrows 7 | 8 | class ModelsTest { 9 | 10 | @Test fun `CacheTag cacheKey should combine context and key with colon`() { 11 | val cacheTag = CacheTag(context = "itemDetails", key = "12345") 12 | 13 | assertEquals("itemDetails:12345", cacheTag.cacheKey()) 14 | } 15 | 16 | @Test fun `CacheTag cacheKey should handle empty context`() { 17 | val cacheTag = CacheTag(context = "", key = "12345") 18 | 19 | assertEquals(":12345", cacheTag.cacheKey()) 20 | } 21 | 22 | @Test fun `CacheTag cacheKey should handle empty key`() { 23 | val cacheTag = CacheTag(context = "itemDetails", key = "") 24 | 25 | assertEquals("itemDetails:", cacheTag.cacheKey()) 26 | } 27 | 28 | @Test fun `CacheTag cacheKey should handle special characters`() { 29 | val cacheTag = CacheTag(context = "item-details", key = "key_with_underscore") 30 | 31 | assertEquals("item-details:key_with_underscore", cacheTag.cacheKey()) 32 | } 33 | 34 | @Test fun `TelemetryTag should initialize with valid service and endpoint`() { 35 | val telemetryTag = TelemetryTag(service = "item", endpoint = "details") 36 | 37 | assertEquals("item", telemetryTag.service) 38 | assertEquals("details", telemetryTag.endpoint) 39 | assertEquals(emptyMap(), telemetryTag.logMap) 40 | assertEquals(mapOf("service" to "item", "endpoint" to "details"), telemetryTag.metricTags) 41 | } 42 | 43 | @Test fun `TelemetryTag should initialize with custom logMap and metricTags`() { 44 | val logMap = mapOf("userId" to "123", "action" to "fetch") 45 | val metricTags = mapOf("service" to "item", "endpoint" to "details", "region" to "us-east") 46 | 47 | val telemetryTag = TelemetryTag( 48 | service = "item", 49 | endpoint = "details", 50 | logMap = logMap, 51 | metricTags = metricTags, 52 | ) 53 | 54 | assertEquals(logMap, telemetryTag.logMap) 55 | assertEquals(metricTags, telemetryTag.metricTags) 56 | } 57 | 58 | @Test fun `TelemetryTag should clean service name with invalid characters`() { 59 | // This should not throw an exception because the service name is cleaned 60 | val telemetryTag = TelemetryTag(service = "item-service", endpoint = "details") 61 | 62 | assertEquals("item-service", telemetryTag.service) 63 | assertEquals(mapOf("service" to "item-service", "endpoint" to "details"), telemetryTag.metricTags) 64 | } 65 | 66 | @Test fun `TelemetryTag should clean endpoint name with invalid characters`() { 67 | val telemetryTag = TelemetryTag(service = "item", endpoint = "get_details") 68 | 69 | assertEquals("get_details", telemetryTag.endpoint) 70 | assertEquals(mapOf("service" to "item", "endpoint" to "get_details"), telemetryTag.metricTags) 71 | } 72 | 73 | @Test fun `TelemetryTag should throw exception for invalid service name`() { 74 | assertThrows { 75 | TelemetryTag(service = "item service", endpoint = "details") 76 | } 77 | } 78 | 79 | @Test fun `TelemetryTag should throw exception for invalid endpoint name`() { 80 | assertThrows { 81 | TelemetryTag(service = "item", endpoint = "get details") 82 | } 83 | } 84 | 85 | @Test fun `TelemetryTag should throw exception for invalid metric tag values`() { 86 | assertThrows { 87 | TelemetryTag( 88 | service = "item", 89 | endpoint = "details", 90 | metricTags = mapOf("service" to "item service", "endpoint" to "details"), 91 | ) 92 | } 93 | } 94 | 95 | @Test fun `TelemetryTag should allow valid metric characters`() { 96 | val telemetryTag = TelemetryTag( 97 | service = "item-service_v1.0", 98 | endpoint = "get-details/v2", 99 | metricTags = mapOf( 100 | "service" to "item-service_v1.0", 101 | "endpoint" to "get-details/v2", 102 | "version" to "1.0.0", 103 | "region" to "us-east-1", 104 | ), 105 | ) 106 | 107 | assertTrue(telemetryTag.metricTags.containsKey("version")) 108 | assertTrue(telemetryTag.metricTags.containsKey("region")) 109 | } 110 | 111 | @Test fun `RequestContext should initialize with telemetryTag only`() { 112 | val telemetryTag = TelemetryTag(service = "item", endpoint = "details") 113 | val requestContext = RequestContext(telemetryTag = telemetryTag) 114 | 115 | assertEquals(telemetryTag, requestContext.telemetryTag) 116 | assertEquals(null, requestContext.cacheTag) 117 | } 118 | 119 | @Test fun `RequestContext should initialize with both telemetryTag and cacheTag`() { 120 | val telemetryTag = TelemetryTag(service = "item", endpoint = "details") 121 | val cacheTag = CacheTag(context = "itemDetails", key = "12345") 122 | val requestContext = RequestContext(telemetryTag = telemetryTag, cacheTag = cacheTag) 123 | 124 | assertEquals(telemetryTag, requestContext.telemetryTag) 125 | assertEquals(cacheTag, requestContext.cacheTag) 126 | } 127 | 128 | @Test fun `RequestContext should handle null cacheTag explicitly`() { 129 | val telemetryTag = TelemetryTag(service = "item", endpoint = "details") 130 | val requestContext = RequestContext(telemetryTag = telemetryTag, cacheTag = null) 131 | 132 | assertEquals(telemetryTag, requestContext.telemetryTag) 133 | assertEquals(null, requestContext.cacheTag) 134 | } 135 | 136 | @Test fun `String cleanedForMetrics should replace invalid characters with underscores`() { 137 | // This tests the private extension function indirectly through TelemetryTag validation 138 | assertThrows { 139 | TelemetryTag(service = "item@service#", endpoint = "details") 140 | } 141 | } 142 | 143 | @Test fun `String cleanedForMetrics should preserve valid characters`() { 144 | // Valid characters: alphanumeric, '.', '/', '-', '_' 145 | val telemetryTag = TelemetryTag( 146 | service = "item_service-v1.0", 147 | endpoint = "get/details", 148 | ) 149 | 150 | assertEquals("item_service-v1.0", telemetryTag.service) 151 | assertEquals("get/details", telemetryTag.endpoint) 152 | } 153 | } 154 | --------------------------------------------------------------------------------