├── .gitignore ├── .scalafmt.conf ├── README.md ├── avro └── src │ └── main │ └── scala │ └── car │ └── avro │ ├── RegisterAvroSchemas.scala │ └── avro.scala ├── build.sbt ├── car-data-consumer └── src │ └── main │ └── scala │ └── car │ └── consumer │ └── CarDataConsumer.scala ├── car-data-producer └── src │ └── main │ └── scala │ └── car │ └── producer │ ├── CarDataProducer.scala │ └── RandomData.scala ├── docker-compose.yml ├── domain └── src │ └── main │ └── scala │ └── car │ └── domain │ └── domain.scala ├── driver-notifier └── src │ └── main │ └── scala │ └── car │ └── drivernotifier │ ├── AvroSerdes.scala │ ├── DriverNotifications.scala │ ├── DriverNotifier.scala │ └── DriverNotifierData.scala ├── kafka-cli-scripts ├── create-topics.sh ├── delete-topics.sh ├── driver-notifications-consumer.sh ├── list-topics.sh └── schema-registry.sh ├── project ├── Dependencies.scala └── build.properties ├── start-kafka.sh └── stop-kafka.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # bloop and metals 2 | .bloop 3 | .bsp 4 | 5 | # metals 6 | project/metals.sbt 7 | .metals 8 | 9 | # vs code 10 | .vscode 11 | 12 | # scala 3 13 | .tasty 14 | 15 | # sbt 16 | project/project/ 17 | project/target/ 18 | target/ 19 | 20 | # eclipse 21 | build/ 22 | .classpath 23 | .project 24 | .settings 25 | .worksheet 26 | bin/ 27 | .cache 28 | 29 | # intellij idea 30 | *.log 31 | *.iml 32 | *.ipr 33 | *.iws 34 | .idea 35 | 36 | # mac 37 | .DS_Store 38 | 39 | # other? 40 | .history 41 | .scala_dependencies 42 | .cache-main 43 | 44 | #general 45 | *.class -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | version = 2.5.0 2 | maxColumn = 140 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### Read the blog 2 | 3 | This code was created for the purpose of [this blog post](https://softwaremill.com/hands-on-kafka-streams-in-scala/), I encourage you to read it first :) 4 | 5 | ### Running the example 6 | 7 | I assume that `sbt` is installed directly on your OS. 8 | 9 | Defined `docker-compose` starts up a _zookeeper_, _kafka_ and _schema-registry_ instance as well as 10 | _tools_ container with [confluent-cli-tools](https://docs.confluent.io/platform/current/installation/cli-reference.html) installed. 11 | Scripts from `kafka-cli-scripts` directory are mounted to the _tools_ container. 12 | 13 | #### Running kafka 14 | 15 | Run `start-kafka.sh` script. It will start up docker network and create topics as defined in `kafka-cli-scripts/create-topics.sh`. 16 | 17 | #### Running apps 18 | 19 | Directory contains 4 sbt projects each being independent app: 20 | * avro - generates and registers _Avro_ schemas for data used as keys/values in created topics 21 | * car-data-producer - produces random data to kafka topics 22 | * driver-notifier - kafka streams application which aggregates data from several topics, processes and produces to `driver-notifications` 23 | * car-data-consumer - can consume data from any of created kafka topics (`driver-notifications` by default) 24 | 25 | Please also add this two entries to your `etc/hosts`, since apps above reach kafka using docker hostnames: 26 | ``` 27 | # streams-app host entries 28 | 127.0.0.1 kafka 29 | 127.0.0.1 schema-registry 30 | ``` 31 | 32 | With `sbt` each app can be started in separate shell instance with the following command: 33 | `sbt "project " "run"`, for an ex. `sbt "project carDataProducer" "run"`. 34 | 35 | Please run `avro` and `carDataProducer` first. 36 | 37 | If you are _Intellij_ user then most convenient option will be to use IDE for running apps, since you can easily tweak, debug and experiment with the code. 38 | 39 | #### Cleanup 40 | 41 | Run `stop-kafka.sh` script. 42 | -------------------------------------------------------------------------------- /avro/src/main/scala/car/avro/RegisterAvroSchemas.scala: -------------------------------------------------------------------------------- 1 | package car.avro 2 | 3 | import io.circe.generic.auto._ 4 | import sttp.client3.circe._ 5 | import sttp.client3.{HttpURLConnectionBackend, _} 6 | 7 | object RegisterAvroSchemas extends App { 8 | 9 | case class RegisterSchemaRequest(schema: String) 10 | 11 | val backend = HttpURLConnectionBackend() 12 | 13 | Seq( 14 | ("car-speed-key", RegisterSchemaRequest(carIdSchema.toString())), 15 | ("car-speed-value", RegisterSchemaRequest(carSpeedSchema.toString())), 16 | // 17 | ("car-engine-key", RegisterSchemaRequest(carIdSchema.toString())), 18 | ("car-engine-value", RegisterSchemaRequest(carEngineSchema.toString())), 19 | // 20 | ("car-location-key", RegisterSchemaRequest(carIdSchema.toString())), 21 | ("car-location-value", RegisterSchemaRequest(carLocationSchema.toString())), 22 | // 23 | ("location-data-key", RegisterSchemaRequest(locationIdSchema.toString())), 24 | ("location-data-value", RegisterSchemaRequest(locationDataSchema.toString())), 25 | // 26 | ("driver-notification-key", RegisterSchemaRequest(carIdSchema.toString())), 27 | ("driver-notification-value", RegisterSchemaRequest(driverNotificationSchema.toString())) 28 | ).map { 29 | case (subject, schema) => 30 | subject -> basicRequest 31 | .post(uri"http://schema-registry:8081/subjects/$subject/versions") 32 | .contentType("application/vnd.schemaregistry.v1+json") 33 | .body(schema) 34 | .send(backend) 35 | .code 36 | } foreach { case (subject, statusCode) => println(s"Register schema $subject, response code: $statusCode") } 37 | } 38 | -------------------------------------------------------------------------------- /avro/src/main/scala/car/avro/avro.scala: -------------------------------------------------------------------------------- 1 | package car 2 | 3 | import car.domain._ 4 | import com.sksamuel.avro4s.{AvroSchema, RecordFormat} 5 | import com.softwaremill.tagging._ 6 | import org.apache.avro.Schema 7 | 8 | package object avro { 9 | type KeyRFTag 10 | type KeyRecordFormat[K] = RecordFormat[K] @@ KeyRFTag 11 | 12 | type ValueRFTag 13 | type ValueRecordFormat[V] = RecordFormat[V] @@ ValueRFTag 14 | 15 | val carIdSchema: Schema = AvroSchema[CarId] 16 | val carSpeedSchema: Schema = AvroSchema[CarSpeed] 17 | val carEngineSchema: Schema = AvroSchema[CarEngine] 18 | val carLocationSchema: Schema = AvroSchema[CarLocation] 19 | val driverNotificationSchema: Schema = AvroSchema[DriverNotification] 20 | 21 | implicit val carIdRF: KeyRecordFormat[CarId] = RecordFormat[CarId].taggedWith[KeyRFTag] 22 | implicit val carSpeedRF: ValueRecordFormat[CarSpeed] = RecordFormat[CarSpeed].taggedWith[ValueRFTag] 23 | implicit val carEngineRF: ValueRecordFormat[CarEngine] = RecordFormat[CarEngine].taggedWith[ValueRFTag] 24 | implicit val carLocationRF: ValueRecordFormat[CarLocation] = RecordFormat[CarLocation].taggedWith[ValueRFTag] 25 | implicit val driverNotificationRF: ValueRecordFormat[DriverNotification] = RecordFormat[DriverNotification].taggedWith[ValueRFTag] 26 | 27 | val locationIdSchema: Schema = AvroSchema[LocationId] 28 | val locationDataSchema: Schema = AvroSchema[LocationData] 29 | 30 | implicit val locationIdRF: KeyRecordFormat[LocationId] = RecordFormat[LocationId].taggedWith[KeyRFTag] 31 | implicit val locationDataRF: ValueRecordFormat[LocationData] = RecordFormat[LocationData].taggedWith[ValueRFTag] 32 | } 33 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | import Dependencies._ 2 | 3 | name := "streams-app" 4 | version := "0.1" 5 | 6 | lazy val commonSettings = Seq( 7 | scalaVersion := "2.13.6", 8 | resolvers += "Confluent Maven Repository" at "https://packages.confluent.io/maven/" 9 | ) 10 | 11 | lazy val root = (project in file(".")).aggregate(carDataProducer, carDataConsumer, driverNotifier, avro, domain) 12 | 13 | lazy val carDataProducer = (project in file("car-data-producer")) 14 | .settings(commonSettings) 15 | .settings( 16 | name := "car-data-producer", 17 | libraryDependencies ++= Seq(Libs.kafkaClient, Libs.kafkaAvro, Libs.catsEffect) 18 | ) 19 | .dependsOn(domain, avro) 20 | 21 | lazy val carDataConsumer = (project in file("car-data-consumer")) 22 | .settings(commonSettings) 23 | .settings( 24 | name := "car-data-consumer", 25 | libraryDependencies ++= Seq(Libs.kafkaClient, Libs.kafkaAvro, Libs.catsEffect) 26 | ) 27 | .dependsOn(domain, avro) 28 | 29 | lazy val driverNotifier = (project in file("driver-notifier")) 30 | .settings(commonSettings) 31 | .settings( 32 | name := "driver-notifier", 33 | libraryDependencies ++= Seq(Libs.kafkaStreamsScala, Libs.kafkaStreamsAvro, Libs.avro4sKafka) 34 | ) 35 | .dependsOn(domain, avro) 36 | 37 | lazy val avro = (project in file("avro")) 38 | .settings(commonSettings) 39 | .settings( 40 | name := "avro", 41 | libraryDependencies ++= Seq(Libs.avro4sCore, Libs.sttp3Core, Libs.sttp3Circe, Libs.circeGeneric, Libs.smlTagging) 42 | ) 43 | .dependsOn(domain) 44 | 45 | lazy val domain = (project in file("domain")).settings(commonSettings) 46 | -------------------------------------------------------------------------------- /car-data-consumer/src/main/scala/car/consumer/CarDataConsumer.scala: -------------------------------------------------------------------------------- 1 | package car.consumer 2 | 3 | import car.avro._ 4 | import car.domain._ 5 | import cats.effect.{ExitCode, IO, IOApp, Resource} 6 | import cats.implicits._ 7 | import com.sksamuel.avro4s.RecordFormat 8 | import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG 9 | import io.confluent.kafka.serializers.KafkaAvroDeserializer 10 | import org.apache.avro.generic.IndexedRecord 11 | import org.apache.kafka.clients.consumer.ConsumerConfig._ 12 | import org.apache.kafka.clients.consumer.KafkaConsumer 13 | 14 | import java.time.Duration 15 | import scala.jdk.CollectionConverters._ 16 | 17 | object CarDataConsumer extends IOApp { 18 | 19 | val props: Map[String, Object] = Map( 20 | GROUP_ID_CONFIG -> "car-metrics-consumer", 21 | BOOTSTRAP_SERVERS_CONFIG -> "kafka:9092", 22 | KEY_DESERIALIZER_CLASS_CONFIG -> classOf[KafkaAvroDeserializer], 23 | VALUE_DESERIALIZER_CLASS_CONFIG -> classOf[KafkaAvroDeserializer], 24 | SCHEMA_REGISTRY_URL_CONFIG -> "http://schema-registry:8081" 25 | ) 26 | 27 | override def run(args: List[String]): IO[ExitCode] = 28 | pollForever[CarId, DriverNotification]("driver-notification").as(ExitCode.Success) 29 | 30 | private def pollForever[K, V](topic: String)(implicit krf: RecordFormat[K], vrf: RecordFormat[V]): IO[Nothing] = 31 | Resource 32 | .make(IO { 33 | val consumer = new KafkaConsumer[IndexedRecord, IndexedRecord](CarDataConsumer.props.asJava) 34 | consumer.subscribe(Seq(topic).asJava) 35 | consumer 36 | })(c => IO(println(s"[$topic] closing consumer...")) *> IO(c.close())) 37 | .use { consumer => 38 | val consume: IO[Unit] = for { 39 | records <- IO(consumer.poll(Duration.ofSeconds(5)).asScala.toSeq) 40 | keyValue = records.map { r => (krf.from(r.key()), vrf.from(r.value())) } 41 | _ <- keyValue.traverse { case (k, v) => IO(println(s"[$topic] $k => $v")) } 42 | } yield () 43 | consume.foreverM 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /car-data-producer/src/main/scala/car/producer/CarDataProducer.scala: -------------------------------------------------------------------------------- 1 | package car.producer 2 | 3 | import car.avro._ 4 | import cats.effect.{ExitCode, IO, IOApp, Resource} 5 | import cats.implicits._ 6 | import com.sksamuel.avro4s.RecordFormat 7 | import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG 8 | import io.confluent.kafka.serializers.KafkaAvroSerializer 9 | import org.apache.avro.generic.IndexedRecord 10 | import org.apache.kafka.clients.producer.ProducerConfig.{ 11 | BOOTSTRAP_SERVERS_CONFIG, 12 | CLIENT_ID_CONFIG, 13 | KEY_SERIALIZER_CLASS_CONFIG, 14 | VALUE_SERIALIZER_CLASS_CONFIG 15 | } 16 | import org.apache.kafka.clients.producer.{Callback, KafkaProducer, ProducerRecord, RecordMetadata} 17 | 18 | import scala.concurrent.Promise 19 | import scala.concurrent.duration.DurationInt 20 | import scala.jdk.CollectionConverters._ 21 | 22 | object CarDataProducer extends IOApp { 23 | 24 | private val props: Map[String, Object] = Map( 25 | CLIENT_ID_CONFIG -> "car-metrics-producer", 26 | BOOTSTRAP_SERVERS_CONFIG -> "kafka:9092", 27 | KEY_SERIALIZER_CLASS_CONFIG -> classOf[KafkaAvroSerializer], 28 | VALUE_SERIALIZER_CLASS_CONFIG -> classOf[KafkaAvroSerializer], 29 | SCHEMA_REGISTRY_URL_CONFIG -> "http://schema-registry:8081" 30 | ) 31 | 32 | override def run(args: List[String]): IO[ExitCode] = 33 | Resource 34 | .make(IO(new KafkaProducer[IndexedRecord, IndexedRecord](props.asJava)))(p => IO(println("closing producer...")) *> IO(p.close())) 35 | .use { producer => 36 | Seq( 37 | IO(RandomData.carSpeed).flatMap(send(producer)("car-speed", _)).foreverM, 38 | IO(RandomData.carEngine).flatMap(send(producer)("car-engine", _)).foreverM, 39 | IO(RandomData.carLocation).flatMap(send(producer)("car-location", _)).foreverM, 40 | IO(RandomData.locationData).flatMap(send(producer)("location-data", _)).foreverM 41 | ).parSequence_.as(ExitCode.Success) 42 | } 43 | 44 | private def send[K, V]( 45 | producer: KafkaProducer[IndexedRecord, IndexedRecord] 46 | )(topic: String, records: Seq[(K, V)])(implicit krf: RecordFormat[K], vrf: RecordFormat[V]): IO[Unit] = 47 | records.traverse { 48 | case (k, v) => 49 | val p = Promise[Unit]() 50 | producer.send( 51 | new ProducerRecord[IndexedRecord, IndexedRecord](topic, krf.to(k), vrf.to(v)), 52 | new Callback { 53 | override def onCompletion(metadata: RecordMetadata, exception: Exception): Unit = 54 | Option(exception).map(p.failure).getOrElse(p.success(())) 55 | } 56 | ) 57 | IO.fromFuture(IO(p.future)) *> IO(println(s"produced data to [$topic]")) *> IO.sleep(2.seconds) 58 | }.void 59 | } 60 | -------------------------------------------------------------------------------- /car-data-producer/src/main/scala/car/producer/RandomData.scala: -------------------------------------------------------------------------------- 1 | package car.producer 2 | 3 | import car.domain._ 4 | 5 | import scala.util.Random 6 | 7 | object RandomData { 8 | private val carIds = Seq(1, 2) 9 | private val cities = Seq("Wroclaw", "Cracow") 10 | private val streets = Seq("Sezamowa", "Tunelowa") 11 | 12 | def carSpeed: Seq[(CarId, CarSpeed)] = 13 | for { 14 | carId <- carIds 15 | speed = Random.between(5, 10) * 10 16 | } yield CarId(carId) -> CarSpeed(speed) 17 | 18 | def carEngine: Seq[(CarId, CarEngine)] = 19 | for { 20 | carId <- carIds 21 | rpm = Random.between(25, 35) * 100 22 | fuelLevel = (math floor Random.between(0d, 1d) * 100) / 100 23 | } yield CarId(carId) -> CarEngine(rpm, fuelLevel) 24 | 25 | def carLocation: Seq[(CarId, CarLocation)] = 26 | for { 27 | carId <- carIds 28 | city = cities(Random.nextInt(cities.size)) 29 | street = streets(Random.nextInt(streets.size)) 30 | } yield CarId(carId) -> CarLocation(LocationId(city, street)) 31 | 32 | def locationData: Seq[(LocationId, LocationData)] = 33 | for { 34 | city <- cities 35 | street <- streets 36 | speedLimit = Random.between(3, 7) * 10 37 | trafficVolume = TrafficVolume(Random.nextInt(TrafficVolume.maxId)) 38 | gasStationNearby = Random.nextBoolean() 39 | } yield LocationId(city, street) -> LocationData(speedLimit, trafficVolume, gasStationNearby) 40 | } 41 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.5' 2 | services: 3 | zookeeper: 4 | hostname: zookeeper 5 | image: "confluentinc/cp-zookeeper:5.3.0" 6 | networks: 7 | - streams-net 8 | environment: 9 | ZOOKEEPER_CLIENT_PORT: 2181 10 | ZOOKEEPER_TICK_TIME: 2000 11 | 12 | kafka: 13 | image: "confluentinc/cp-enterprise-kafka:5.3.0" 14 | hostname: kafka 15 | networks: 16 | - streams-net 17 | ports: 18 | - 9092:9092 19 | environment: 20 | KAFKA_BROKER_ID: 1 21 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 22 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 23 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" 24 | KAFKA_DELETE_TOPIC_ENABLE: "true" 25 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 26 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100 27 | 28 | schema-registry: 29 | hostname: schema-registry 30 | image: "confluentinc/cp-schema-registry:5.3.0" 31 | networks: 32 | - streams-net 33 | ports: 34 | - 8081:8081 35 | environment: 36 | SCHEMA_REGISTRY_HOST_NAME: schema-registry 37 | SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 38 | SCHEMA_REGISTRY_LISTENERS: http://schema-registry:8081 39 | 40 | tools: 41 | depends_on: 42 | - "kafka" 43 | - "zookeeper" 44 | - "schema-registry" 45 | image: cnfltraining/training-tools:19.06 46 | hostname: tools 47 | # default container names seems to differ based on version, this name is used in start-kafka.sh 48 | container_name: streams-app-tools-1 49 | networks: 50 | - streams-net 51 | volumes: 52 | - ./kafka-cli-scripts:/root/streams-app 53 | working_dir: /root/streams-app 54 | command: /bin/bash 55 | tty: true 56 | 57 | 58 | networks: 59 | streams-net: -------------------------------------------------------------------------------- /domain/src/main/scala/car/domain/domain.scala: -------------------------------------------------------------------------------- 1 | package car 2 | 3 | package object domain { 4 | 5 | case class CarId(value: Int) 6 | 7 | case class CarSpeed(value: Int) 8 | case class CarEngine(rpm: Int, fuelLevel: Double) 9 | case class CarLocation(locationId: LocationId) 10 | 11 | case class DriverNotification(msg: String) 12 | 13 | case class LocationId(city: String, street: String) 14 | case class LocationData(speedLimit: Int, trafficVolume: TrafficVolume.Value, gasStationNearby: Boolean) 15 | 16 | object TrafficVolume extends Enumeration { 17 | val Low, Medium, High = Value 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /driver-notifier/src/main/scala/car/drivernotifier/AvroSerdes.scala: -------------------------------------------------------------------------------- 1 | package car.drivernotifier 2 | 3 | import car.avro.{KeyRecordFormat, ValueRecordFormat} 4 | import com.sksamuel.avro4s.RecordFormat 5 | import io.confluent.kafka.streams.serdes.avro.GenericAvroSerde 6 | import org.apache.avro.generic.GenericRecord 7 | import org.apache.kafka.common.serialization.Serde 8 | import org.apache.kafka.streams.scala.serialization.Serdes 9 | 10 | import scala.jdk.CollectionConverters._ 11 | 12 | object AvroSerdes { 13 | 14 | private val props = Map("schema.registry.url" -> "http://schema-registry:8081") 15 | 16 | implicit def keySerde[K >: Null](implicit krf: KeyRecordFormat[K]): Serde[K] = { 17 | val avroKeySerde = new GenericAvroSerde 18 | avroKeySerde.configure(props.asJava, true) 19 | avroKeySerde.forCaseClass[K] 20 | } 21 | 22 | implicit def valueSerde[V >: Null](implicit vrf: ValueRecordFormat[V]): Serde[V] = { 23 | val avroValueSerde = new GenericAvroSerde 24 | avroValueSerde.configure(props.asJava, false) 25 | avroValueSerde.forCaseClass[V] 26 | } 27 | 28 | implicit class CaseClassSerde(inner: Serde[GenericRecord]) { 29 | def forCaseClass[T >: Null](implicit rf: RecordFormat[T]): Serde[T] = { 30 | Serdes.fromFn( 31 | (topic, data) => inner.serializer().serialize(topic, rf.to(data)), 32 | (topic, bytes) => Option(rf.from(inner.deserializer().deserialize(topic, bytes))) 33 | ) 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /driver-notifier/src/main/scala/car/drivernotifier/DriverNotifications.scala: -------------------------------------------------------------------------------- 1 | package car.drivernotifier 2 | 3 | import car.domain.{DriverNotification, LocationData, TrafficVolume} 4 | import car.drivernotifier.DriverNotifierData.{CarAndLocationData, CarData} 5 | 6 | object DriverNotifications { 7 | 8 | def apply(data: CarAndLocationData): List[DriverNotification] = 9 | List(checkSpeed, checkTrafficVolume, checkEngineRPM, checkFuelLevel).flatten(_.lift(data)) 10 | 11 | private val checkSpeed: PartialFunction[CarAndLocationData, DriverNotification] = { 12 | case CarAndLocationData(CarData(Some(speed), _, _), LocationData(speedLimit, _, _)) if speed.value > speedLimit => 13 | DriverNotification(s"Slow down, speed limit $speedLimit") 14 | } 15 | 16 | private val checkTrafficVolume: PartialFunction[CarAndLocationData, DriverNotification] = { 17 | case CarAndLocationData(CarData(_, _, Some(location)), LocationData(_, TrafficVolume.High, _)) => 18 | DriverNotification(s"High traffic ahead on ${location.locationId.street} street") 19 | } 20 | 21 | private val checkEngineRPM: PartialFunction[CarAndLocationData, DriverNotification] = { 22 | case CarAndLocationData(CarData(_, Some(engine), _), _) if engine.rpm > HighRPM => DriverNotification("Shift up a gear") 23 | } 24 | 25 | private val checkFuelLevel: PartialFunction[CarAndLocationData, DriverNotification] = { 26 | case CarAndLocationData(CarData(_, Some(engine), _), LocationData(_, _, gasStationNearby)) 27 | if engine.fuelLevel <= FuelReserve && gasStationNearby => 28 | DriverNotification("Low fuel level, navigate to nearest gas station?") 29 | } 30 | 31 | private val HighRPM = 3000 32 | private val FuelReserve = 0.2 33 | } 34 | -------------------------------------------------------------------------------- /driver-notifier/src/main/scala/car/drivernotifier/DriverNotifier.scala: -------------------------------------------------------------------------------- 1 | package car.drivernotifier 2 | 3 | import car.avro._ 4 | import car.domain._ 5 | import car.drivernotifier.AvroSerdes._ 6 | import cats.implicits._ 7 | import com.sksamuel.avro4s.BinaryFormat 8 | import com.sksamuel.avro4s.kafka.GenericSerde 9 | import org.apache.kafka.common.utils.Bytes 10 | import org.apache.kafka.streams.KafkaStreams 11 | import org.apache.kafka.streams.StreamsConfig.{APPLICATION_ID_CONFIG, BOOTSTRAP_SERVERS_CONFIG} 12 | import org.apache.kafka.streams.scala.ImplicitConversions._ 13 | import org.apache.kafka.streams.scala._ 14 | import org.apache.kafka.streams.scala.kstream.{KGroupedStream, KTable, Materialized} 15 | import org.apache.kafka.streams.state.KeyValueStore 16 | 17 | import java.util.Properties 18 | 19 | object DriverNotifier extends App { 20 | 21 | import DriverNotifierData._ 22 | 23 | val props = new Properties() 24 | props.put(APPLICATION_ID_CONFIG, "driver-notifier") 25 | props.put(BOOTSTRAP_SERVERS_CONFIG, "kafka:9092") 26 | 27 | val builder: StreamsBuilder = new StreamsBuilder 28 | 29 | val carSpeed: KGroupedStream[CarId, CarSpeed] = builder.stream[CarId, CarSpeed]("car-speed").groupByKey 30 | val carEngine: KGroupedStream[CarId, CarEngine] = builder.stream[CarId, CarEngine]("car-engine").groupByKey 31 | val carLocation: KGroupedStream[CarId, CarLocation] = builder.stream[CarId, CarLocation]("car-location").groupByKey 32 | val locationData: KTable[LocationId, LocationData] = builder.table[LocationId, LocationData]("location-data") 33 | 34 | implicit val carDataSerde: GenericSerde[CarData] = new GenericSerde[CarData](BinaryFormat) 35 | 36 | val carData: KTable[CarId, CarData] = carSpeed 37 | .cogroup[CarData]({ case (_, speed, agg) => agg.copy(speed = speed.some) }) 38 | .cogroup[CarEngine](carEngine, { case (_, engine, agg) => agg.copy(engine = engine.some) }) 39 | .cogroup[CarLocation](carLocation, { case (_, location, agg) => agg.copy(location = location.some) }) 40 | .aggregate(CarData.empty) 41 | 42 | implicit val carAndLocationDataSerde: GenericSerde[CarAndLocationData] = new GenericSerde[CarAndLocationData](BinaryFormat) 43 | 44 | val carAndLocationData: KTable[CarId, CarAndLocationData] = carData 45 | .filter({ case (_, carData) => carData.location.isDefined }) 46 | .join[CarAndLocationData, LocationId, LocationData]( 47 | locationData, 48 | keyExtractor = (carData: CarData) => carData.location.get.locationId, 49 | joiner = (carData: CarData, locationData: LocationData) => CarAndLocationData(carData, locationData), 50 | materialized = implicitly[Materialized[CarId, CarAndLocationData, KeyValueStore[Bytes, Array[Byte]]]] 51 | ) 52 | 53 | def print[K, V](k: K, v: V): Unit = println(s"$k -> $v") 54 | 55 | carAndLocationData.toStream.flatMapValues(DriverNotifications(_)).to("driver-notification") 56 | 57 | val topology = builder.build() 58 | val streams = new KafkaStreams(topology, props) 59 | 60 | streams.start() 61 | 62 | Runtime.getRuntime.addShutdownHook(new Thread(() => streams.close())) 63 | } 64 | -------------------------------------------------------------------------------- /driver-notifier/src/main/scala/car/drivernotifier/DriverNotifierData.scala: -------------------------------------------------------------------------------- 1 | package car.drivernotifier 2 | 3 | import car.domain._ 4 | 5 | object DriverNotifierData { 6 | case class CarData(speed: Option[CarSpeed], engine: Option[CarEngine], location: Option[CarLocation]) 7 | 8 | object CarData { 9 | val empty: CarData = CarData(None, None, None) 10 | } 11 | 12 | case class CarAndLocationData(carData: CarData, locationData: LocationData) 13 | } 14 | -------------------------------------------------------------------------------- /kafka-cli-scripts/create-topics.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kafka-topics --create --bootstrap-server kafka:9092 --partitions 2 --replication-factor 1 --topic car-speed 4 | kafka-topics --create --bootstrap-server kafka:9092 --partitions 2 --replication-factor 1 --topic car-engine 5 | kafka-topics --create --bootstrap-server kafka:9092 --partitions 2 --replication-factor 1 --topic car-location 6 | kafka-topics --create --bootstrap-server kafka:9092 --partitions 1 --replication-factor 1 --topic location-data --config "cleanup.policy=compact" 7 | kafka-topics --create --bootstrap-server kafka:9092 --partitions 2 --replication-factor 1 --topic driver-notification -------------------------------------------------------------------------------- /kafka-cli-scripts/delete-topics.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kafka-topics --delete --bootstrap-server kafka:9092 --topic car-speed 4 | kafka-topics --delete --bootstrap-server kafka:9092 --topic car-engine 5 | kafka-topics --delete --bootstrap-server kafka:9092 --topic car-location 6 | kafka-topics --delete --bootstrap-server kafka:9092 --topic location-data 7 | kafka-topics --delete --bootstrap-server kafka:9092 --topic driver-notification 8 | 9 | curl -X DELETE http://schema-registry:8081/subjects/car-speed-key?permanent=true 10 | curl -X DELETE http://schema-registry:8081/subjects/car-speed-value?permanent=true 11 | 12 | curl -X DELETE http://schema-registry:8081/subjects/car-engine-key?permanent=true 13 | curl -X DELETE http://schema-registry:8081/subjects/car-engine-value?permanent=true 14 | 15 | curl -X DELETE http://schema-registry:8081/subjects/car-location-key?permanent=true 16 | curl -X DELETE http://schema-registry:8081/subjects/car-location-value?permanent=true 17 | 18 | curl -X DELETE http://schema-registry:8081/subjects/location-data-key?permanent=true 19 | curl -X DELETE http://schema-registry:8081/subjects/location-data-value?permanent=true 20 | 21 | curl -X DELETE http://schema-registry:8081/subjects/driver-notification-key?permanent=true 22 | curl -X DELETE http://schema-registry:8081/subjects/driver-notification-value?permanent=true -------------------------------------------------------------------------------- /kafka-cli-scripts/driver-notifications-consumer.sh: -------------------------------------------------------------------------------- 1 | kafka-console-consumer --bootstrap-server kafka:9092 --from-beginning --topic driver-notifications -------------------------------------------------------------------------------- /kafka-cli-scripts/list-topics.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kafka-topics --bootstrap-server kafka:9092 --list -------------------------------------------------------------------------------- /kafka-cli-scripts/schema-registry.sh: -------------------------------------------------------------------------------- 1 | curl -X DELETE http://schema-registry:8081/subjects/car-metric-value/versions/1 2 | 3 | curl -X DELETE http://schema-registry:8081/subjects/car-metric-key 4 | curl -X DELETE http://schema-registry:8081/subjects/car-metric-value 5 | 6 | 7 | curl http://schema-registry:8081/subjects/car-metric-value/versions 8 | 9 | -------------------------------------------------------------------------------- /project/Dependencies.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | 3 | object Dependencies { 4 | 5 | object V { 6 | val kafka = "2.8.0" 7 | val kafkaAvro = "6.2.0" 8 | val avro4s = "4.0.10" 9 | val sttp3 = "3.3.11" 10 | val circe = "0.14.1" 11 | val cats = "3.2.5" 12 | val smlCommon = "2.3.1" 13 | } 14 | 15 | object Libs { 16 | val kafkaClient = "org.apache.kafka" % "kafka-clients" % V.kafka 17 | val kafkaStreams = "org.apache.kafka" % "kafka-streams" % V.kafka 18 | val kafkaStreamsScala = "org.apache.kafka" %% "kafka-streams-scala" % V.kafka 19 | val kafkaAvro = "io.confluent" % "kafka-avro-serializer" % V.kafkaAvro 20 | val kafkaStreamsAvro = "io.confluent" % "kafka-streams-avro-serde" % V.kafkaAvro 21 | 22 | val avro4sCore = "com.sksamuel.avro4s" % "avro4s-core_2.13" % V.avro4s 23 | val avro4sKafka = "com.sksamuel.avro4s" % "avro4s-kafka_2.13" % V.avro4s 24 | 25 | val sttp3Core = "com.softwaremill.sttp.client3" %% "core" % V.sttp3 26 | val sttp3Circe = "com.softwaremill.sttp.client3" %% "circe" % V.sttp3 27 | val circeGeneric = "io.circe" %% "circe-generic" % V.circe 28 | 29 | val catsEffect = "org.typelevel" %% "cats-effect" % V.cats 30 | 31 | val smlTagging = "com.softwaremill.common" %% "tagging" % V.smlCommon 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 1.5.4 -------------------------------------------------------------------------------- /start-kafka.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker compose up -d 4 | 5 | sleep 5 # wait until broker is ready to accept requests 6 | 7 | echo "Creating kafka topics..." 8 | docker exec -it streams-app-tools-1 ./create-topics.sh 9 | 10 | echo "Listing kafka topics..." 11 | docker exec -it streams-app-tools-1 ./list-topics.sh -------------------------------------------------------------------------------- /stop-kafka.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker-compose down 4 | --------------------------------------------------------------------------------