├── LICENSE ├── NOTICE ├── README.md ├── build.sbt └── src ├── main └── scala │ └── com │ └── github │ └── aseigneurin │ └── kafka │ ├── serialization │ └── scala │ │ ├── BaseSerde.scala │ │ ├── JsonSerde.scala │ │ ├── NumberAsBinarySerdes.scala │ │ ├── NumberAsStringSerdes.scala │ │ └── ScalaSerdes.scala │ └── streams │ └── scala │ ├── ImplicitConversions.scala │ ├── KGroupedStreamS.scala │ ├── KGroupedTableS.scala │ ├── KStreamBuilderS.scala │ ├── KStreamS.scala │ └── KTableS.scala └── test └── scala └── com └── github └── aseigneurin └── kafka └── streams └── scala ├── JsonSerdeDemo.scala └── WordCountDemo.scala /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Kafka Streams Scala 2 | Copyright 2017-2018 Alexis Seigneurin. 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kafka-streams-scala 2 | 3 | This is a thin Scala wrapper for the [Kafka Streams API](https://kafka.apache.org/documentation/streams). It does not intend to provide a Scala-idiomatic API, but rather intends to make the original API simpler to use from Scala. In particular, it provides the following adjustments: 4 | 5 | - Scala lambda expressions can be used directly 6 | - when aggregating and counting, counts are converted from Java `Long`s to Scala `Long`s 7 | - when using a `flatMap` operation, this lets you use a Scala `Iterable` 8 | - `Serde`s (Serializers/Deserializers) can be implicitly found in the scope 9 | 10 | This API also contains a few `Serde`s (Serializers/Deserializers): 11 | 12 | - to convert Scala `Int`/`Long`/`Double` to/from their binary representation 13 | - to convert Scala `Int`/`Long`/`Double` to/from string representation 14 | - to convert case classes to/from JSON 15 | 16 | Finally, the API provides the following extensions: 17 | 18 | - `KStreamS.split()` (see documentation below) 19 | 20 | ## Usage of the Kafka Streams API in Scala 21 | 22 | The main objects are: 23 | 24 | - `KStreamsBuilderS` as the entry point to build streams or tables 25 | - `KStreamS` as a wrapper around `KStream` 26 | - `KGroupedStreamS` as a wrapper around `KGroupedStream` 27 | - `KTableS` as a wrapper around `KTable` 28 | - `KGroupedTable` as a wrapper around `KGroupedTable` 29 | 30 | ### Using the builder 31 | 32 | With the original Java API, you would create an instance of `KStreamBuilder`, then use it to create streams or tables. Here, `KStreamsBuilderS` is an `object` that can be used directly: 33 | 34 | ```scala 35 | val stream: KStreamS[String, String] = KStreamBuilderS.stream[String, String]("my-stream") 36 | 37 | val table: KTableS[String, String] = KStreamBuilderS.table[String, String]("my-table") 38 | ``` 39 | 40 | When starting the application, you just need to unwrap the `KStreamBuilder` by calling `KStreamBuilderS.inner`: 41 | 42 | ```scala 43 | val streams = new KafkaStreams(KStreamBuilderS.inner, config) 44 | ``` 45 | 46 | ### Serdes (declare them as implicit) 47 | 48 | It is a common mistake to forget to specify `Serde`s when using the Java API, then resulting in class cast errors when objects are serialized or deserialized. 49 | 50 | To work around this issue, this API requires `Serde`s to be used. Most of the times, it is enough to declare your `Serde`s as `implicit` values, and they will be picked up automatically: 51 | 52 | ```scala 53 | implicit val stringSerde: Serde[String] = Serdes.String() 54 | implicit val userSerde: Serde[User] = new MyUserSerde 55 | 56 | val usersByIdStream = KStreamBuilderS.stream[String, User]("users-by-id") 57 | ``` 58 | 59 | Resolution is based on the type of the object to serialize/deserialize, so make sure you have a `Serde` of the appropriate type. If not, you should see an error such as: 60 | 61 | ``` 62 | Error:(87, 80) could not find implicit value for parameter keySerde: org.apache.kafka.common.serialization.Serde[String] 63 | ``` 64 | 65 | If, on the other hand, you have multiple `Serde`s for the same type, you might see the following error: 66 | 67 | ``` 68 | Error:(88, 80) ambiguous implicit values: 69 | both value stringSerde2 of type org.apache.kafka.common.serialization.Serde[String] 70 | and value stringSerde1 of type org.apache.kafka.common.serialization.Serde[String] 71 | match expected type org.apache.kafka.common.serialization.Serde[String] 72 | ``` 73 | 74 | In this case, just pass the `Serde` explicitly: 75 | 76 | ```scala 77 | val usersByIdStream = KStreamBuilderS.stream[String, User]("users-by-id")(stringSerde, userSerde) 78 | ``` 79 | 80 | ## Usage of the Serdes in Scala 81 | 82 | To convert Scala `Int`/`Long`/`Double` to/from their binary representation: 83 | 84 | ```scala 85 | import com.github.aseigneurin.kafka.serialization.scala._ 86 | 87 | implicit val intSerde = IntAsStringSerde 88 | implicit val longSerde = LongAsStringSerde 89 | implicit val doubleSerde = DoubleAsStringSerde 90 | ``` 91 | 92 | To convert Scala `Int`/`Long`/`Double` to/from string representation: 93 | 94 | ```scala 95 | import com.github.aseigneurin.kafka.serialization.scala._ 96 | 97 | implicit val intSerde = IntSerde 98 | implicit val longSerde = LongSerde 99 | implicit val doubleSerde = DoubleSerde 100 | ``` 101 | 102 | To convert case classes to/from JSON: 103 | 104 | - define a `case class` 105 | - create an instance of `JsonSerde` with the case class as the generic type 106 | 107 | Example: 108 | 109 | ```scala 110 | import com.github.aseigneurin.kafka.serialization.scala._ 111 | 112 | case class User(name: String) 113 | 114 | implicit val stringSerde = Serdes.String 115 | implicit val userSerde = new JsonSerde[User] 116 | 117 | // read JSON -> case class 118 | KStreamBuilderS.stream[String, User]("users") 119 | .mapValues { user => user.name } 120 | .to("names") 121 | 122 | // write case class -> JSON 123 | KStreamBuilderS.stream[String, String]("names") 124 | .mapValues { name => User(name) } 125 | .to("users") 126 | ``` 127 | 128 | ## Example 129 | 130 | This repository contains [a Scala version](https://github.com/aseigneurin/kafka-streams-scala/blob/master/src/test/scala/com/github/aseigneurin/kafka/streams/scala/WordCountDemo.scala) of [the Java Word Count Demo](https://github.com/apache/kafka/blob/trunk/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountDemo.java). 131 | 132 | Here is the code to implement a word count: 133 | 134 | ```scala 135 | val props = new Properties() 136 | // ... 137 | 138 | implicit val stringSerde = Serdes.String 139 | implicit val longSerde = LongAsStringSerde 140 | 141 | val source = KStreamBuilderS.stream[String, String]("streams-file-input") 142 | 143 | val counts: KTableS[String, Long] = source 144 | .flatMapValues { value => value.toLowerCase(Locale.getDefault).split(" ") } 145 | .map { (_, value) => (value, value) } 146 | .groupByKey 147 | .count("Counts") 148 | 149 | counts.to("streams-wordcount-output") 150 | 151 | val streams = new KafkaStreams(KStreamBuilderS.inner, props) 152 | streams.start() 153 | ``` 154 | 155 | ## Extensions 156 | 157 | ### KStreamS.split() 158 | 159 | This method applies a predicate and returns two `KStreamS`s, one with the messages that match the predicate, and another one with the messages that don't match. 160 | 161 | The two `KStreamS`s are returned in a tuple that can be easily _deconstructed_: 162 | 163 | ```scala 164 | def isValidMessage(v: ...): Boolean = ??? 165 | 166 | val (goodMessages, badMessages) = deserializedMessages.split((k, v) => isValidMessage(v)) 167 | ``` 168 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017-2018 Alexis Seigneurin. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at: 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | name := "kafka-streams-scala" 18 | 19 | organization := "com.github.aseigneurin" 20 | 21 | version := "0.0.1-SNAPSHOT" 22 | 23 | scalaVersion := "2.11.11" 24 | 25 | scalacOptions := Seq("-Xexperimental", "-unchecked", "-deprecation") 26 | 27 | // Kafka 28 | libraryDependencies += "org.apache.kafka" % "kafka-streams" % "0.10.2.0" 29 | // JSON 30 | libraryDependencies += "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.8.8" 31 | // logging 32 | libraryDependencies += "org.slf4j" % "slf4j-log4j12" % "1.7.25" 33 | libraryDependencies += "com.typesafe.scala-logging" %% "scala-logging" % "3.5.0" 34 | 35 | publishTo := Some(Resolver.file("file", new File(Path.userHome.absolutePath + "/.m2/repository"))) 36 | -------------------------------------------------------------------------------- /src/main/scala/com/github/aseigneurin/kafka/serialization/scala/BaseSerde.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017-2018 Alexis Seigneurin. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at: 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.github.aseigneurin.kafka.serialization.scala 18 | 19 | import java.util 20 | 21 | import org.apache.kafka.common.serialization.{Deserializer, Serde, Serializer} 22 | 23 | abstract class BaseSerde[T] extends Serde[T] { 24 | 25 | override def configure(configs: util.Map[String, _], isKey: Boolean): Unit = {} 26 | 27 | override def close(): Unit = {} 28 | 29 | override def serializer() = BaseSerializer(serialize) 30 | 31 | override def deserializer() = BaseDeserializer(deserialize) 32 | 33 | def serialize(topic: String, data: T): Array[Byte] 34 | 35 | def deserialize(topic: String, data: Array[Byte]): T 36 | 37 | } 38 | 39 | abstract class BaseSerializer[T] extends Serializer[T] { 40 | 41 | override def configure(configs: util.Map[String, _], isKey: Boolean): Unit = {} 42 | 43 | override def close(): Unit = {} 44 | 45 | } 46 | 47 | object BaseSerializer { 48 | 49 | def apply[T](func: (String, T) => Array[Byte]) = new BaseSerializer[T] { 50 | override def serialize(topic: String, data: T): Array[Byte] = func(topic, data) 51 | } 52 | 53 | } 54 | 55 | abstract class BaseDeserializer[T] extends Deserializer[T] { 56 | 57 | override def configure(configs: util.Map[String, _], isKey: Boolean): Unit = {} 58 | 59 | override def close(): Unit = {} 60 | 61 | } 62 | 63 | object BaseDeserializer { 64 | 65 | def apply[T](func: (String, Array[Byte]) => T) = new BaseDeserializer[T] { 66 | override def deserialize(topic: String, data: Array[Byte]): T = func(topic, data) 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /src/main/scala/com/github/aseigneurin/kafka/serialization/scala/JsonSerde.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017-2018 Alexis Seigneurin. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at: 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.github.aseigneurin.kafka.serialization.scala 18 | 19 | import com.fasterxml.jackson.databind.ObjectMapper 20 | import com.fasterxml.jackson.module.scala.DefaultScalaModule 21 | import com.typesafe.scalalogging.LazyLogging 22 | 23 | import scala.reflect.{ClassTag, classTag} 24 | 25 | class JsonSerde[T >: Null : ClassTag] extends BaseSerde[T] with LazyLogging { 26 | 27 | val mapper = new ObjectMapper 28 | mapper.registerModule(DefaultScalaModule) 29 | 30 | override def deserialize(topic: String, data: Array[Byte]): T = data match { 31 | case null => null 32 | case _ => 33 | try { 34 | mapper.readValue(data, classTag[T].runtimeClass.asInstanceOf[Class[T]]) 35 | } catch { 36 | case e: Exception => 37 | val jsonStr = new String(data, "UTF-8") 38 | logger.warn(s"Failed parsing ${jsonStr}", e) 39 | null 40 | } 41 | } 42 | 43 | override def serialize(topic: String, obj: T): Array[Byte] = { 44 | mapper.writeValueAsBytes(obj) 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/main/scala/com/github/aseigneurin/kafka/serialization/scala/NumberAsBinarySerdes.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017-2018 Alexis Seigneurin. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at: 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.github.aseigneurin.kafka.serialization.scala 18 | 19 | import org.apache.kafka.common.serialization._ 20 | 21 | object IntSerde extends BaseSerde[Int] { 22 | 23 | val innerSerializer = new IntegerSerializer 24 | val innerDeserializer = new IntegerDeserializer 25 | 26 | override def serialize(topic: String, data: Int): Array[Byte] = innerSerializer.serialize(topic, data) 27 | 28 | override def deserialize(topic: String, data: Array[Byte]): Int = innerDeserializer.deserialize(topic, data) 29 | 30 | } 31 | 32 | object LongSerde extends BaseSerde[Long] { 33 | 34 | val innerSerializer = new LongSerializer 35 | val innerDeserializer = new LongDeserializer 36 | 37 | override def serialize(topic: String, data: Long): Array[Byte] = innerSerializer.serialize(topic, data) 38 | 39 | override def deserialize(topic: String, data: Array[Byte]): Long = innerDeserializer.deserialize(topic, data) 40 | 41 | } 42 | 43 | object DoubleSerde extends BaseSerde[Double] { 44 | 45 | val innerSerializer = new DoubleSerializer 46 | val innerDeserializer = new DoubleDeserializer 47 | 48 | override def serialize(topic: String, data: Double): Array[Byte] = innerSerializer.serialize(topic, data) 49 | 50 | override def deserialize(topic: String, data: Array[Byte]): Double = innerDeserializer.deserialize(topic, data) 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/scala/com/github/aseigneurin/kafka/serialization/scala/NumberAsStringSerdes.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017-2018 Alexis Seigneurin. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at: 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.github.aseigneurin.kafka.serialization.scala 18 | 19 | object IntAsStringSerde extends BaseSerde[Int] { 20 | 21 | override def serialize(topic: String, data: Int): Array[Byte] = { 22 | data.toString.getBytes("UTF-8") 23 | } 24 | 25 | override def deserialize(topic: String, data: Array[Byte]): Int = data match { 26 | case null => 0 27 | case _ => 28 | val str = new String(data, "UTF-8") 29 | str.toInt 30 | } 31 | 32 | } 33 | 34 | object LongAsStringSerde extends BaseSerde[Long] { 35 | 36 | override def serialize(topic: String, data: Long): Array[Byte] = { 37 | data.toString.getBytes("UTF-8") 38 | } 39 | 40 | override def deserialize(topic: String, data: Array[Byte]): Long = data match { 41 | case null => 0L 42 | case _ => 43 | val str = new String(data, "UTF-8") 44 | str.toLong 45 | } 46 | 47 | } 48 | 49 | object DoubleAsStringSerde extends BaseSerde[Double] { 50 | 51 | override def serialize(topic: String, data: Double): Array[Byte] = { 52 | data.toString.getBytes("UTF-8") 53 | } 54 | 55 | override def deserialize(topic: String, data: Array[Byte]): Double = data match { 56 | case null => 0.0 57 | case _ => 58 | val str = new String(data, "UTF-8") 59 | str.toDouble 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /src/main/scala/com/github/aseigneurin/kafka/serialization/scala/ScalaSerdes.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017-2018 Alexis Seigneurin. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at: 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.github.aseigneurin.kafka.serialization.scala 18 | 19 | import org.apache.kafka.common.serialization.Serde 20 | 21 | object ScalaSerdes { 22 | 23 | def serdeFrom[T](serialize: (String, T) => Array[Byte], 24 | deserialize: (String, Array[Byte]) => T 25 | ): Serde[T] = { 26 | new BaseSerde[T] { 27 | override def serialize(topic: String, data: T): Array[Byte] = serialize(topic, data) 28 | 29 | override def deserialize(topic: String, data: Array[Byte]): T = deserialize(topic, data) 30 | } 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/scala/com/github/aseigneurin/kafka/streams/scala/ImplicitConversions.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017-2018 Alexis Seigneurin. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at: 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.github.aseigneurin.kafka.streams.scala 18 | 19 | import org.apache.kafka.streams.kstream.{KGroupedStream, KGroupedTable, KStream, KTable} 20 | 21 | import scala.language.implicitConversions 22 | 23 | object ImplicitConversions { 24 | 25 | implicit def wrapKStream[K, V](inner: KStream[K, V]): KStreamS[K, V] = 26 | new KStreamS[K, V](inner) 27 | 28 | implicit def wrapKGroupedStream[K, V](inner: KGroupedStream[K, V]): KGroupedStreamS[K, V] = 29 | new KGroupedStreamS[K, V](inner) 30 | 31 | implicit def wrapKTable[K, V](inner: KTable[K, V]): KTableS[K, V] = 32 | new KTableS[K, V](inner) 33 | 34 | implicit def wrapKGroupedTable[K, V](inner: KGroupedTable[K, V]): KGroupedTableS[K, V] = 35 | new KGroupedTableS[K, V](inner) 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/scala/com/github/aseigneurin/kafka/streams/scala/KGroupedStreamS.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017-2018 Alexis Seigneurin. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at: 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.github.aseigneurin.kafka.streams.scala 18 | 19 | import com.github.aseigneurin.kafka.streams.scala.ImplicitConversions._ 20 | import org.apache.kafka.common.serialization.Serde 21 | import org.apache.kafka.streams.kstream._ 22 | import org.apache.kafka.streams.processor.StateStoreSupplier 23 | import org.apache.kafka.streams.state.{KeyValueStore, SessionStore, WindowStore} 24 | 25 | class KGroupedStreamS[K, V](inner: KGroupedStream[K, V]) { 26 | 27 | def count(storeName: String): KTableS[K, Long] = { 28 | inner.count(storeName) 29 | .mapValues[Long](javaLong => Long.box(javaLong)) 30 | } 31 | 32 | def count(storeSupplier: StateStoreSupplier[KeyValueStore[_, _]]): KTableS[K, Long] = { 33 | inner.count(storeSupplier) 34 | .mapValues[Long](javaLong => Long.box(javaLong)) 35 | } 36 | 37 | def count[W <: Window](windows: Windows[W], 38 | storeName: String): KTableS[Windowed[K], Long] = { 39 | inner.count[W](windows, storeName) 40 | .mapValues[Long](javaLong => Long.box(javaLong)) 41 | } 42 | 43 | def count[W <: Window](windows: Windows[W], 44 | storeSupplier: StateStoreSupplier[WindowStore[_, _]]): KTableS[Windowed[K], Long] = { 45 | inner.count[W](windows, storeSupplier) 46 | .mapValues[Long](javaLong => Long.box(javaLong)) 47 | } 48 | 49 | def count(sessionWindows: SessionWindows, storeName: String): KTableS[Windowed[K], Long] = { 50 | inner.count(sessionWindows, storeName) 51 | .mapValues[Long](javaLong => Long.box(javaLong)) 52 | } 53 | 54 | def count(sessionWindows: SessionWindows, 55 | storeSupplier: StateStoreSupplier[SessionStore[_, _]]): KTableS[Windowed[K], Long] = { 56 | inner.count(sessionWindows, storeSupplier) 57 | .mapValues[Long](javaLong => Long.box(javaLong)) 58 | } 59 | 60 | def reduce(reducer: (V, V) => V, 61 | storeName: String): KTableS[K, V] = { 62 | val reducerJ: Reducer[V] = (v1: V, v2: V) => reducer(v1, v2) 63 | inner.reduce(reducerJ, storeName) 64 | } 65 | 66 | def reduce(reducer: (V, V) => V, 67 | storeSupplier: StateStoreSupplier[KeyValueStore[_, _]]): KTableS[K, V] = { 68 | val reducerJ: Reducer[V] = (v1: V, v2: V) => reducer(v1, v2) 69 | inner.reduce(reducerJ, storeSupplier) 70 | } 71 | 72 | def reduce[W <: Window](reducer: (V, V) => V, 73 | windows: Windows[W], 74 | storeName: String): KTableS[Windowed[K], V] = { 75 | val reducerJ: Reducer[V] = (v1: V, v2: V) => reducer(v1, v2) 76 | inner.reduce(reducerJ, windows, storeName) 77 | } 78 | 79 | def reduce[W <: Window](reducer: (V, V) => V, 80 | windows: Windows[W], 81 | storeSupplier: StateStoreSupplier[WindowStore[_, _]]): KTableS[Windowed[K], V] = { 82 | val reducerJ: Reducer[V] = (v1: V, v2: V) => reducer(v1, v2) 83 | inner.reduce(reducerJ, windows, storeSupplier) 84 | } 85 | 86 | def reduce(reducer: (V, V) => V, 87 | sessionWindows: SessionWindows, 88 | storeName: String): KTableS[Windowed[K], V] = { 89 | val reducerJ: Reducer[V] = (v1: V, v2: V) => reducer(v1, v2) 90 | inner.reduce(reducerJ, sessionWindows, storeName) 91 | } 92 | 93 | def reduce(reducer: (V, V) => V, 94 | sessionWindows: SessionWindows, 95 | storeSupplier: StateStoreSupplier[SessionStore[_, _]]): KTableS[Windowed[K], V] = { 96 | val reducerJ: Reducer[V] = (v1: V, v2: V) => reducer(v1, v2) 97 | inner.reduce(reducerJ, sessionWindows, storeSupplier) 98 | } 99 | 100 | def aggregate[VR](initializer: () => VR, 101 | aggregator: (K, V, VR) => VR, 102 | aggValueSerde: Serde[VR], 103 | storeName: String): KTableS[K, VR] = { 104 | val initializerJ: Initializer[VR] = () => initializer() 105 | val aggregatorJ: Aggregator[K, V, VR] = (k: K, v: V, va: VR) => aggregator(k, v, va) 106 | inner.aggregate(initializerJ, aggregatorJ, aggValueSerde, storeName) 107 | } 108 | 109 | def aggregate[VR](initializer: () => VR, 110 | aggregator: (K, V, VR) => VR, 111 | storeSupplier: StateStoreSupplier[KeyValueStore[_, _]]): KTableS[K, VR] = { 112 | val initializerJ: Initializer[VR] = () => initializer() 113 | val aggregatorJ: Aggregator[K, V, VR] = (k: K, v: V, va: VR) => aggregator(k, v, va) 114 | inner.aggregate(initializerJ, aggregatorJ, storeSupplier) 115 | } 116 | 117 | def aggregate[W <: Window, VR](initializer: () => VR, 118 | aggregator: (K, V, VR) => VR, 119 | windows: Windows[W], 120 | aggValueSerde: Serde[VR], 121 | storeName: String): KTableS[Windowed[K], VR] = { 122 | val initializerJ: Initializer[VR] = () => initializer() 123 | val aggregatorJ: Aggregator[K, V, VR] = (k: K, v: V, va: VR) => aggregator(k, v, va) 124 | inner.aggregate(initializerJ, aggregatorJ, windows, aggValueSerde, storeName) 125 | } 126 | 127 | def aggregate[W <: Window, VR](initializer: () => VR, 128 | aggregator: (K, V, VR) => VR, 129 | windows: Windows[W], 130 | storeSupplier: StateStoreSupplier[WindowStore[_, _]]): KTableS[Windowed[K], VR] = { 131 | val initializerJ: Initializer[VR] = () => initializer() 132 | val aggregatorJ: Aggregator[K, V, VR] = (k: K, v: V, va: VR) => aggregator(k, v, va) 133 | inner.aggregate(initializerJ, aggregatorJ, windows, storeSupplier) 134 | } 135 | 136 | def aggregate[T](initializer: Initializer[T], 137 | aggregator: (K, V, T) => T, 138 | sessionMerger: (K, T, T) => T, 139 | sessionWindows: SessionWindows, 140 | aggValueSerde: Serde[T], 141 | storeName: String): KTableS[Windowed[K], T] = { 142 | val initializerJ: Initializer[T] = () => initializer() 143 | val aggregatorJ: Aggregator[K, V, T] = (k: K, v: V, t: T) => aggregator(k, v, t) 144 | val sessionMergerJ: Merger[K, T] = (aggKey: K, aggOne: T, aggTwo: T) => sessionMerger(aggKey, aggOne, aggTwo) 145 | inner.aggregate(initializerJ, aggregatorJ, sessionMergerJ, sessionWindows, aggValueSerde, storeName) 146 | } 147 | 148 | def aggregate[T](initializer: Initializer[T], 149 | aggregator: (K, V, T) => T, 150 | sessionMerger: Merger[_ >: K, T], 151 | sessionWindows: SessionWindows, 152 | aggValueSerde: Serde[T], 153 | storeSupplier: StateStoreSupplier[SessionStore[_, _]]): KTableS[Windowed[K], T] = { 154 | val initializerJ: Initializer[T] = () => initializer() 155 | val aggregatorJ: Aggregator[K, V, T] = (k: K, v: V, t: T) => aggregator(k, v, t) 156 | val sessionMergerJ: Merger[K, T] = (aggKey: K, aggOne: T, aggTwo: T) => sessionMerger(aggKey, aggOne, aggTwo) 157 | inner.aggregate(initializerJ, aggregatorJ, sessionMergerJ, sessionWindows, aggValueSerde, storeSupplier) 158 | } 159 | 160 | } 161 | -------------------------------------------------------------------------------- /src/main/scala/com/github/aseigneurin/kafka/streams/scala/KGroupedTableS.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017-2018 Alexis Seigneurin. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at: 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.github.aseigneurin.kafka.streams.scala 18 | 19 | import com.github.aseigneurin.kafka.streams.scala.ImplicitConversions._ 20 | import org.apache.kafka.common.serialization.Serde 21 | import org.apache.kafka.streams.kstream._ 22 | import org.apache.kafka.streams.processor.StateStoreSupplier 23 | import org.apache.kafka.streams.state.KeyValueStore 24 | 25 | class KGroupedTableS[K, V](inner: KGroupedTable[K, V]) { 26 | 27 | def count(storeName: String): KTableS[K, Long] = { 28 | inner.count(storeName) 29 | .mapValues[Long](javaLong => Long.box(javaLong)) 30 | } 31 | 32 | def count(storeSupplier: StateStoreSupplier[KeyValueStore[_, _]]): KTableS[K, Long] = { 33 | inner.count(storeSupplier) 34 | .mapValues[Long](javaLong => Long.box(javaLong)) 35 | } 36 | 37 | def reduce(adder: (V, V) => V, 38 | subtractor: (V, V) => V, 39 | storeName: String): KTableS[K, V] = { 40 | val adderJ: Reducer[V] = (v1: V, v2: V) => adder(v1, v2) 41 | val subtractorJ: Reducer[V] = (v1: V, v2: V) => subtractor(v1, v2) 42 | inner.reduce(adderJ, subtractorJ, storeName) 43 | } 44 | 45 | def reduce(adder: Reducer[V], 46 | subtractor: Reducer[V], 47 | storeSupplier: StateStoreSupplier[KeyValueStore[_, _]]): KTableS[K, V] = { 48 | val adderJ: Reducer[V] = (v1: V, v2: V) => adder(v1, v2) 49 | val subtractorJ: Reducer[V] = (v1: V, v2: V) => subtractor(v1, v2) 50 | inner.reduce(adderJ, subtractorJ, storeSupplier) 51 | } 52 | 53 | def aggregate[VR](initializer: () => VR, 54 | adder: (K, V, VR) => VR, 55 | subtractor: (K, V, VR) => VR, 56 | storeName: String): KTableS[K, VR] = { 57 | val initializerJ: Initializer[VR] = () => initializer() 58 | val adderJ: Aggregator[K, V, VR] = (k: K, v: V, va: VR) => adder(k, v, va) 59 | val subtractorJ: Aggregator[K, V, VR] = (k: K, v: V, va: VR) => subtractor(k, v, va) 60 | inner.aggregate(initializerJ, adderJ, subtractorJ, storeName) 61 | } 62 | 63 | def aggregate[VR](initializer: () => VR, 64 | adder: (K, V, VR) => VR, 65 | subtractor: (K, V, VR) => VR, 66 | aggValueSerde: Serde[VR], 67 | storeName: String): KTableS[K, VR] = { 68 | val initializerJ: Initializer[VR] = () => initializer() 69 | val adderJ: Aggregator[K, V, VR] = (k: K, v: V, va: VR) => adder(k, v, va) 70 | val subtractorJ: Aggregator[K, V, VR] = (k: K, v: V, va: VR) => subtractor(k, v, va) 71 | inner.aggregate(initializerJ, adderJ, subtractorJ, aggValueSerde, storeName) 72 | } 73 | 74 | def aggregate[VR](initializer: () => VR, 75 | adder: (K, V, VR) => VR, 76 | subtractor: (K, V, VR) => VR, 77 | storeSupplier: StateStoreSupplier[KeyValueStore[_, _]]): KTableS[K, VR] = { 78 | val initializerJ: Initializer[VR] = () => initializer() 79 | val adderJ: Aggregator[K, V, VR] = (k: K, v: V, va: VR) => adder(k, v, va) 80 | val subtractorJ: Aggregator[K, V, VR] = (k: K, v: V, va: VR) => subtractor(k, v, va) 81 | inner.aggregate(initializerJ, adderJ, subtractorJ, storeSupplier) 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /src/main/scala/com/github/aseigneurin/kafka/streams/scala/KStreamBuilderS.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017-2018 Alexis Seigneurin. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at: 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.github.aseigneurin.kafka.streams.scala 18 | 19 | import java.util.regex.Pattern 20 | 21 | import com.github.aseigneurin.kafka.streams.scala.ImplicitConversions._ 22 | import org.apache.kafka.common.serialization.Serde 23 | import org.apache.kafka.streams.kstream.{GlobalKTable, KStreamBuilder} 24 | import org.apache.kafka.streams.processor.TopologyBuilder 25 | 26 | object KStreamBuilderS { 27 | 28 | val inner = new KStreamBuilder 29 | 30 | def stream[K, V](topics: String*) 31 | (implicit keySerde: Serde[K], valSerde: Serde[V]): KStreamS[K, V] = 32 | inner.stream[K, V](keySerde, valSerde, topics: _*) 33 | 34 | def stream[K, V](offsetReset: TopologyBuilder.AutoOffsetReset, 35 | topics: String*) 36 | (implicit keySerde: Serde[K], valSerde: Serde[V]): KStreamS[K, V] = 37 | inner.stream[K, V](offsetReset, keySerde, valSerde, topics: _*) 38 | 39 | def stream[K, V](topicPattern: Pattern) 40 | (implicit keySerde: Serde[K], valSerde: Serde[V]): KStreamS[K, V] = 41 | inner.stream[K, V](keySerde, valSerde, topicPattern) 42 | 43 | def stream[K, V](offsetReset: TopologyBuilder.AutoOffsetReset, 44 | topicPattern: Pattern) 45 | (implicit keySerde: Serde[K], valSerde: Serde[V]): KStreamS[K, V] = 46 | inner.stream[K, V](offsetReset, keySerde, valSerde, topicPattern) 47 | 48 | def table[K, V](topic: String, 49 | storeName: String) 50 | (implicit keySerde: Serde[K], valSerde: Serde[V]): KTableS[K, V] = 51 | inner.table[K, V](keySerde, valSerde, topic, storeName) 52 | 53 | def table[K, V](offsetReset: TopologyBuilder.AutoOffsetReset, 54 | topic: String, 55 | storeName: String) 56 | (implicit keySerde: Serde[K], valSerde: Serde[V]): KTableS[K, V] = 57 | inner.table[K, V](offsetReset, keySerde, valSerde, topic, storeName) 58 | 59 | 60 | def globalTable[K, V](topic: String, 61 | storeName: String) 62 | (implicit keySerde: Serde[K], valSerde: Serde[V]): GlobalKTable[K, V] = 63 | inner.globalTable(keySerde, valSerde, topic, storeName) 64 | 65 | def merge[K, V](streams: KStreamS[K, V]*): KStreamS[K, V] = { 66 | val streamsJ = streams.map { streamS => streamS.inner } 67 | inner.merge(streamsJ: _*) 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/main/scala/com/github/aseigneurin/kafka/streams/scala/KStreamS.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017-2018 Alexis Seigneurin. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at: 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.github.aseigneurin.kafka.streams.scala 18 | 19 | import com.github.aseigneurin.kafka.streams.scala.ImplicitConversions._ 20 | import org.apache.kafka.common.serialization.Serde 21 | import org.apache.kafka.streams.KeyValue 22 | import org.apache.kafka.streams.kstream._ 23 | import org.apache.kafka.streams.processor.{Processor, ProcessorContext, ProcessorSupplier, StreamPartitioner} 24 | 25 | import scala.collection.JavaConversions._ 26 | 27 | class KStreamS[K, V](val inner: KStream[K, V]) { 28 | 29 | def filter(predicate: (K, V) => Boolean): KStreamS[K, V] = { 30 | val predicateJ: Predicate[K, V] = (k, v) => predicate(k, v) 31 | inner.filter(predicateJ) 32 | } 33 | 34 | def filterNot(predicate: (K, V) => Boolean): KStreamS[K, V] = { 35 | val predicateJ: Predicate[K, V] = (k, v) => predicate(k, v) 36 | inner.filterNot(predicateJ) 37 | } 38 | 39 | def selectKey[KR](mapper: (K, V) => KR): KStreamS[KR, V] = { 40 | val mapperJ: KeyValueMapper[K, V, KR] = (k: K, v: V) => mapper(k, v) 41 | inner.selectKey[KR](mapperJ) 42 | } 43 | 44 | def map[KR, VR](mapper: (K, V) => (KR, VR)): KStreamS[KR, VR] = { 45 | val mapperJ: KeyValueMapper[K, V, KeyValue[KR, VR]] = (k: K, v: V) => { 46 | val res = mapper(k, v) 47 | new KeyValue[KR, VR](res._1, res._2) 48 | } 49 | inner.map[KR, VR](mapperJ) 50 | } 51 | 52 | def mapValues[VR](mapper: (V => VR)): KStreamS[K, VR] = { 53 | val mapperJ: ValueMapper[V, VR] = (v: V) => mapper(v) 54 | inner.mapValues[VR](mapperJ) 55 | } 56 | 57 | def flatMap[KR, VR](mapper: (K, V) => Iterable[(KR, VR)]): KStreamS[KR, VR] = { 58 | val mapperJ: KeyValueMapper[K, V, java.lang.Iterable[KeyValue[KR, VR]]] = (k: K, v: V) => { 59 | val resTuples: Iterable[(KR, VR)] = mapper(k, v) 60 | val res: Iterable[KeyValue[KR, VR]] = resTuples.map(t => new KeyValue[KR, VR](t._1, t._2)) 61 | asJavaIterable(res) 62 | } 63 | inner.flatMap[KR, VR](mapperJ) 64 | } 65 | 66 | def flatMapValues[VR](processor: V => Iterable[VR]): KStreamS[K, VR] = { 67 | val processorJ: ValueMapper[V, java.lang.Iterable[VR]] = (v: V) => { 68 | val res: Iterable[VR] = processor(v) 69 | asJavaIterable(res) 70 | } 71 | inner.flatMapValues[VR](processorJ) 72 | } 73 | 74 | def print() = 75 | inner.print() 76 | 77 | def print(streamName: String) = 78 | inner.print(streamName) 79 | 80 | def print(keySerde: Serde[K], valSerde: Serde[V]) = 81 | inner.print(keySerde, valSerde) 82 | 83 | def print(keySerde: Serde[K], valSerde: Serde[V], streamName: String) = 84 | inner.print(keySerde, valSerde, streamName) 85 | 86 | def writeAsText(filePath: String) 87 | (implicit keySerde: Serde[K], valSerde: Serde[V]) = { 88 | inner.writeAsText(filePath, keySerde, valSerde) 89 | } 90 | 91 | def writeAsText(filePath: String, 92 | streamName: String) 93 | (implicit keySerde: Serde[K], valSerde: Serde[V]) = { 94 | inner.writeAsText(filePath, streamName, keySerde, valSerde) 95 | } 96 | 97 | def foreach(action: (K, V) => Unit): Unit = { 98 | val actionJ: ForeachAction[_ >: K, _ >: V] = (k: K, v: V) => action(k, v) 99 | inner.foreach(actionJ) 100 | } 101 | 102 | def branch(predicates: ((K, V) => Boolean)*): Array[KStreamS[K, V]] = { 103 | val predicatesJ = predicates.map(predicate => { 104 | val predicateJ: Predicate[K, V] = (k, v) => predicate(k, v) 105 | predicateJ 106 | }) 107 | inner.branch(predicatesJ: _*) 108 | .map(kstream => wrapKStream(kstream)) 109 | } 110 | 111 | def through(topic: String) 112 | (implicit keySerde: Serde[K], valSerde: Serde[V]): KStreamS[K, V] = { 113 | inner.through(keySerde, valSerde, topic) 114 | } 115 | 116 | def through(partitioner: (K, V, Int) => Int, 117 | topic: String) 118 | (implicit keySerde: Serde[K], valSerde: Serde[V]): KStreamS[K, V] = { 119 | val partitionerJ: StreamPartitioner[K, V] = 120 | (key: K, value: V, numPartitions: Int) => partitioner(key, value, numPartitions) 121 | inner.through(keySerde, valSerde, partitionerJ, topic) 122 | } 123 | 124 | def to(topic: String) 125 | (implicit keySerde: Serde[K], valSerde: Serde[V]) = { 126 | inner.to(keySerde, valSerde, topic) 127 | } 128 | 129 | def to(partitioner: (K, V, Int) => Int, 130 | topic: String) 131 | (implicit keySerde: Serde[K], valSerde: Serde[V]) = { 132 | val partitionerJ: StreamPartitioner[K, V] = 133 | (key: K, value: V, numPartitions: Int) => partitioner(key, value, numPartitions) 134 | inner.to(keySerde, valSerde, partitionerJ, topic) 135 | } 136 | 137 | def transform[K1, V1](transformerSupplier: () => Transformer[K, V, (K1, V1)], 138 | stateStoreNames: String*): KStreamS[K1, V1] = { 139 | val transformerSupplierJ: TransformerSupplier[K, V, KeyValue[K1, V1]] = () => { 140 | val transformerS: Transformer[K, V, (K1, V1)] = transformerSupplier() 141 | new Transformer[K, V, KeyValue[K1, V1]] { 142 | override def transform(key: K, value: V): KeyValue[K1, V1] = { 143 | val res = transformerS.transform(key, value) 144 | new KeyValue[K1, V1](res._1, res._2) 145 | } 146 | 147 | override def init(context: ProcessorContext): Unit = transformerS.init(context) 148 | 149 | override def punctuate(timestamp: Long): KeyValue[K1, V1] = { 150 | val res = transformerS.punctuate(timestamp) 151 | new KeyValue[K1, V1](res._1, res._2) 152 | } 153 | 154 | override def close(): Unit = transformerS.close() 155 | } 156 | } 157 | inner.transform(transformerSupplierJ, stateStoreNames: _*) 158 | } 159 | 160 | def transformValues[VR](valueTransformerSupplier: () => ValueTransformer[V, VR], 161 | stateStoreNames: String*): KStreamS[K, VR] = { 162 | val valueTransformerSupplierJ: ValueTransformerSupplier[V, VR] = () => valueTransformerSupplier() 163 | inner.transformValues[VR](valueTransformerSupplierJ, stateStoreNames: _*) 164 | } 165 | 166 | def process(processorSupplier: () => Processor[K, V], 167 | stateStoreNames: String*) = { 168 | val processorSupplierJ: ProcessorSupplier[K, V] = () => processorSupplier() 169 | inner.process(processorSupplierJ, stateStoreNames: _*) 170 | } 171 | 172 | def groupByKey(implicit keySerde: Serde[K], valSerde: Serde[V]): KGroupedStreamS[K, V] = 173 | inner.groupByKey(keySerde, valSerde) 174 | 175 | def groupBy[KR](selector: (K, V) => KR): KGroupedStreamS[KR, V] = { 176 | val selectorJ: KeyValueMapper[K, V, KR] = (k: K, v: V) => selector(k, v) 177 | inner.groupBy(selectorJ) 178 | } 179 | 180 | def groupBy[KR](selector: (K, V) => KR, 181 | keySerde: Serde[KR], 182 | valueSerde: Serde[V]): KGroupedStreamS[KR, V] = { 183 | val selectorJ: KeyValueMapper[K, V, KR] = (k: K, v: V) => selector(k, v) 184 | inner.groupBy(selectorJ, keySerde, valueSerde) 185 | } 186 | 187 | def join[VO, VR](otherStream: KStreamS[K, VO], 188 | joiner: (V, VO) => VR, 189 | windows: JoinWindows): KStreamS[K, VR] = { 190 | val joinerJ: ValueJoiner[V, VO, VR] = (v1, v2) => joiner(v1, v2) 191 | inner.join[VO, VR](otherStream.inner, joinerJ, windows) 192 | } 193 | 194 | def join[VO, VR](otherStream: KStreamS[K, VO], 195 | joiner: (V, VO) => VR, 196 | windows: JoinWindows, 197 | keySerde: Serde[K], 198 | thisValueSerde: Serde[V], 199 | otherValueSerde: Serde[VO]): KStreamS[K, VR] = { 200 | val joinerJ: ValueJoiner[V, VO, VR] = (v1, v2) => joiner(v1, v2) 201 | inner.join[VO, VR](otherStream.inner, joinerJ, windows, keySerde, thisValueSerde, otherValueSerde) 202 | } 203 | 204 | def leftJoin[VO, VR](otherStream: KStreamS[K, VO], 205 | joiner: ValueJoiner[_ >: V, _ >: VO, _ <: VR], 206 | windows: JoinWindows): KStreamS[K, VR] = { 207 | val joinerJ: ValueJoiner[V, VO, VR] = (v1, v2) => joiner(v1, v2) 208 | inner.leftJoin[VO, VR](otherStream.inner, joinerJ, windows) 209 | } 210 | 211 | def leftJoin[VO, VR](otherStream: KStreamS[K, VO], 212 | joiner: ValueJoiner[_ >: V, _ >: VO, _ <: VR], 213 | windows: JoinWindows, 214 | keySerde: Serde[K], 215 | thisValSerde: Serde[V], 216 | otherValueSerde: Serde[VO]): KStreamS[K, VR] = { 217 | val joinerJ: ValueJoiner[V, VO, VR] = (v1, v2) => joiner(v1, v2) 218 | inner.leftJoin[VO, VR](otherStream.inner, joinerJ, windows, keySerde, thisValSerde, otherValueSerde) 219 | } 220 | 221 | def outerJoin[VO, VR](otherStream: KStreamS[K, VO], 222 | joiner: ValueJoiner[_ >: V, _ >: VO, _ <: VR], 223 | windows: JoinWindows): KStreamS[K, VR] = { 224 | val joinerJ: ValueJoiner[V, VO, VR] = (v1, v2) => joiner(v1, v2) 225 | inner.outerJoin[VO, VR](otherStream.inner, joinerJ, windows) 226 | } 227 | 228 | def outerJoin[VO, VR](otherStream: KStreamS[K, VO], 229 | joiner: ValueJoiner[_ >: V, _ >: VO, _ <: VR], 230 | windows: JoinWindows, 231 | keySerde: Serde[K], 232 | thisValueSerde: Serde[V], 233 | otherValueSerde: Serde[VO]): KStreamS[K, VR] = { 234 | val joinerJ: ValueJoiner[V, VO, VR] = (v1, v2) => joiner(v1, v2) 235 | inner.outerJoin[VO, VR](otherStream.inner, joinerJ, windows, keySerde, thisValueSerde, otherValueSerde) 236 | } 237 | 238 | def join[VT, VR](table: KTableS[K, VT], 239 | joiner: (V, VT) => VR): KStreamS[K, VR] = { 240 | val joinerJ: ValueJoiner[V, VT, VR] = (v1, v2) => joiner(v1, v2) 241 | inner.join[VT, VR](table.inner, joinerJ) 242 | } 243 | 244 | def join[VT, VR](table: KTableS[K, VT], 245 | joiner: (V, VT) => VR, 246 | keySerde: Serde[K], 247 | valSerde: Serde[V]): KStreamS[K, VR] = { 248 | val joinerJ: ValueJoiner[V, VT, VR] = (v1, v2) => joiner(v1, v2) 249 | inner.join[VT, VR](table.inner, joinerJ, keySerde, valSerde) 250 | } 251 | 252 | def leftJoin[VT, VR](table: KTableS[K, VT], 253 | joiner: (V, VT) => VR): KStreamS[K, VR] = { 254 | val joinerJ: ValueJoiner[V, VT, VR] = (v1, v2) => joiner(v1, v2) 255 | inner.leftJoin[VT, VR](table.inner, joinerJ) 256 | } 257 | 258 | def leftJoin[VT, VR](table: KTableS[K, VT], 259 | joiner: (V, VT) => VR, 260 | keySerde: Serde[K], 261 | valSerde: Serde[V]): KStreamS[K, VR] = { 262 | val joinerJ: ValueJoiner[V, VT, VR] = (v1, v2) => joiner(v1, v2) 263 | inner.leftJoin[VT, VR](table.inner, joinerJ, keySerde, valSerde) 264 | } 265 | 266 | def join[GK, GV, RV](globalKTable: GlobalKTable[GK, GV], 267 | keyValueMapper: (K, V) => GK, 268 | joiner: ValueJoiner[_ >: V, _ >: GV, _ <: RV]): KStreamS[K, RV] = { 269 | val keyValueMapperJ: KeyValueMapper[K, V, GK] = (k: K, v: V) => keyValueMapper(k, v) 270 | val joinerJ: ValueJoiner[V, GV, RV] = (v1, v2) => joiner(v1, v2) 271 | inner.join[GK, GV, RV](globalKTable, keyValueMapperJ, joinerJ) 272 | } 273 | 274 | def leftJoin[GK, GV, RV](globalKTable: GlobalKTable[GK, GV], 275 | keyValueMapper: KeyValueMapper[_ >: K, _ >: V, _ <: GK], 276 | valueJoiner: ValueJoiner[_ >: V, _ >: GV, _ <: RV]): KStreamS[K, RV] = { 277 | val keyValueMapperJ: KeyValueMapper[K, V, GK] = (k: K, v: V) => keyValueMapper(k, v) 278 | val valueJoinerJ: ValueJoiner[V, GV, RV] = (v1, v2) => valueJoiner(v1, v2) 279 | inner.leftJoin[GK, GV, RV](globalKTable, keyValueMapperJ, valueJoinerJ) 280 | } 281 | 282 | // -- EXTENSIONS TO KAFKA STREAMS -- 283 | 284 | // applies the predicate to know what messages shuold go to the left stream (predicate == true) 285 | // or to the right stream (predicate == false) 286 | def split(predicate: (K, V) => Boolean): (KStreamS[K, V], KStreamS[K, V]) = { 287 | (this.filter(predicate), this.filterNot(predicate)) 288 | } 289 | 290 | } 291 | -------------------------------------------------------------------------------- /src/main/scala/com/github/aseigneurin/kafka/streams/scala/KTableS.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017-2018 Alexis Seigneurin. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at: 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.github.aseigneurin.kafka.streams.scala 18 | 19 | import com.github.aseigneurin.kafka.streams.scala.ImplicitConversions._ 20 | import org.apache.kafka.common.serialization.Serde 21 | import org.apache.kafka.streams.KeyValue 22 | import org.apache.kafka.streams.kstream._ 23 | import org.apache.kafka.streams.processor.StreamPartitioner 24 | 25 | class KTableS[K, V](val inner: KTable[K, V]) { 26 | 27 | def filter(predicate: (K, V) => Boolean): KTableS[K, V] = { 28 | val predicateJ: Predicate[K, V] = (k, v) => predicate(k, v) 29 | inner.filter(predicateJ) 30 | } 31 | 32 | def filterNot(predicate: (K, V) => Boolean): KTableS[K, V] = { 33 | val predicateJ: Predicate[K, V] = (k, v) => predicate(k, v) 34 | inner.filterNot(predicateJ) 35 | } 36 | 37 | def mapValues[VR](mapper: (V) => VR): KTable[K, VR] = { 38 | def mapperJ: ValueMapper[V, VR] = (v) => mapper(v) 39 | 40 | inner.mapValues(mapperJ) 41 | } 42 | 43 | def print() = 44 | inner.print() 45 | 46 | def print(streamName: String) = 47 | inner.print(streamName) 48 | 49 | def print(keySerde: Serde[K], valSerde: Serde[V]) = 50 | inner.print(keySerde, valSerde) 51 | 52 | def print(keySerde: Serde[K], valSerde: Serde[V], streamName: String) = 53 | inner.print(keySerde, valSerde, streamName) 54 | 55 | def writeAsText(filePath: String) 56 | (implicit keySerde: Serde[K], valSerde: Serde[V]) = { 57 | inner.writeAsText(filePath, keySerde, valSerde) 58 | } 59 | 60 | def writeAsText(filePath: String, 61 | streamName: String) 62 | (implicit keySerde: Serde[K], valSerde: Serde[V]) = { 63 | inner.writeAsText(filePath, streamName, keySerde, valSerde) 64 | } 65 | 66 | def foreach(action: (K, V) => Unit): Unit = { 67 | val actionJ: ForeachAction[_ >: K, _ >: V] = (k: K, v: V) => action(k, v) 68 | inner.foreach(actionJ) 69 | } 70 | 71 | def toStream: KStreamS[K, V] = 72 | inner.toStream 73 | 74 | def toStream[KR](mapper: (K, V) => KR): KStreamS[KR, V] = { 75 | val mapperJ: KeyValueMapper[K, V, KR] = (k: K, v: V) => mapper(k, v) 76 | inner.toStream[KR](mapperJ) 77 | } 78 | 79 | def through(topic: String, 80 | storeName: String) 81 | (implicit keySerde: Serde[K], valSerde: Serde[V]): KTableS[K, V] = { 82 | inner.through(keySerde, valSerde, topic, storeName) 83 | } 84 | 85 | def through(partitioner: (K, V, Int) => Int, 86 | topic: String, 87 | storeName: String) 88 | (implicit keySerde: Serde[K], valSerde: Serde[V]): KTableS[K, V] = { 89 | val partitionerJ: StreamPartitioner[K, V] = 90 | (key: K, value: V, numPartitions: Int) => partitioner(key, value, numPartitions) 91 | inner.through(keySerde, valSerde, partitionerJ, topic, storeName) 92 | } 93 | 94 | def to(topic: String) 95 | (implicit keySerde: Serde[K], valSerde: Serde[V]) = { 96 | inner.to(keySerde, valSerde, topic) 97 | } 98 | 99 | def to(partitioner: (K, V, Int) => Int, 100 | topic: String) 101 | (implicit keySerde: Serde[K], valSerde: Serde[V]) = { 102 | val partitionerJ: StreamPartitioner[K, V] = 103 | (key: K, value: V, numPartitions: Int) => partitioner(key, value, numPartitions) 104 | inner.to(keySerde, valSerde, partitionerJ, topic) 105 | } 106 | 107 | def groupBy[KR, VR](selector: (K, V) => (KR, VR)): KGroupedTableS[KR, VR] = { 108 | val selectorJ: KeyValueMapper[K, V, KeyValue[KR, VR]] = (k: K, v: V) => { 109 | val res = selector(k, v) 110 | new KeyValue[KR, VR](res._1, res._2) 111 | } 112 | inner.groupBy(selectorJ) 113 | } 114 | 115 | def groupBy[KR, VR](selector: (K, V) => (KR, VR), 116 | keySerde: Serde[KR], 117 | valueSerde: Serde[VR]): KGroupedTableS[KR, VR] = { 118 | val selectorJ: KeyValueMapper[K, V, KeyValue[KR, VR]] = (k: K, v: V) => { 119 | val res = selector(k, v) 120 | new KeyValue[KR, VR](res._1, res._2) 121 | } 122 | inner.groupBy(selectorJ, keySerde, valueSerde) 123 | } 124 | 125 | def join[VO, VR](other: KTableS[K, VO], 126 | joiner: (V, VO) => VR): KTableS[K, VR] = { 127 | val joinerJ: ValueJoiner[V, VO, VR] = (v1, v2) => joiner(v1, v2) 128 | inner.join[VO, VR](other.inner, joinerJ) 129 | } 130 | 131 | def leftJoin[VO, VR](other: KTableS[K, VO], 132 | joiner: (V, VO) => VR): KTableS[K, VR] = { 133 | val joinerJ: ValueJoiner[V, VO, VR] = (v1, v2) => joiner(v1, v2) 134 | inner.leftJoin[VO, VR](other.inner, joinerJ) 135 | } 136 | 137 | def outerJoin[VO, VR](other: KTableS[K, VO], 138 | joiner: (V, VO) => VR): KTableS[K, VR] = { 139 | val joinerJ: ValueJoiner[V, VO, VR] = (v1, v2) => joiner(v1, v2) 140 | inner.outerJoin[VO, VR](other.inner, joinerJ) 141 | } 142 | 143 | def getStoreName: String = 144 | inner.getStoreName 145 | 146 | } 147 | -------------------------------------------------------------------------------- /src/test/scala/com/github/aseigneurin/kafka/streams/scala/JsonSerdeDemo.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017-2018 Alexis Seigneurin. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at: 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.github.aseigneurin.kafka.streams.scala 18 | 19 | import java.util.Properties 20 | 21 | import com.github.aseigneurin.kafka.serialization.scala.JsonSerde 22 | import org.apache.kafka.clients.consumer.ConsumerConfig 23 | import org.apache.kafka.common.serialization.Serdes 24 | import org.apache.kafka.streams.{KafkaStreams, StreamsConfig} 25 | 26 | /** 27 | * Prerequesites: 28 | * $ kafka-topics --zookeeper localhost:2181 --create --partitions 4 --replication-factor 1 --topic names 29 | * $ kafka-topics --zookeeper localhost:2181 --create --partitions 4 --replication-factor 1 --topic users 30 | * $ kafka-topics --zookeeper localhost:2181 --create --partitions 4 --replication-factor 1 --topic names-from-users 31 | * 32 | * Launch this code. 33 | * 34 | * Launch consumers: 35 | * $ kafka-console-consumer --bootstrap-server localhost:9092 --topic users 36 | * $ kafka-console-consumer --bootstrap-server localhost:9092 --topic names-from-users 37 | * 38 | * Launch a producer and type a few things: 39 | * $ kafka-console-producer --broker-list localhost:9092 --topic names 40 | */ 41 | 42 | object JsonSerdeDemo { 43 | 44 | case class User(name: String) 45 | 46 | def main(args: Array[String]): Unit = { 47 | 48 | val props = new Properties() 49 | props.put(StreamsConfig.APPLICATION_ID_CONFIG, "names") 50 | props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092") 51 | props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest") 52 | 53 | implicit val stringSerde = Serdes.String 54 | implicit val userSerde = new JsonSerde[User] 55 | 56 | KStreamBuilderS.stream[String, String]("names") 57 | .mapValues { name => User(name) } 58 | .to("users") 59 | 60 | KStreamBuilderS.stream[String, User]("users") 61 | .mapValues { user => user.name } 62 | .to("names-from-users") 63 | 64 | val streams = new KafkaStreams(KStreamBuilderS.inner, props) 65 | streams.start() 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /src/test/scala/com/github/aseigneurin/kafka/streams/scala/WordCountDemo.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017-2018 Alexis Seigneurin. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at: 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.github.aseigneurin.kafka.streams.scala 18 | 19 | import java.util.{Locale, Properties} 20 | 21 | import com.github.aseigneurin.kafka.serialization.scala.LongAsStringSerde 22 | import org.apache.kafka.clients.consumer.ConsumerConfig 23 | import org.apache.kafka.common.serialization.Serdes 24 | import org.apache.kafka.streams.{KafkaStreams, StreamsConfig} 25 | 26 | // copied and adapted from https://github.com/apache/kafka/blob/trunk/streams/examples/src/main/java/org/apache/kafka/streams/examples/wordcount/WordCountDemo.java 27 | object WordCountDemo { 28 | 29 | def main(args: Array[String]): Unit = { 30 | 31 | val props = new Properties() 32 | props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-wordcount") 33 | props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092") 34 | 35 | // setting offset reset to earliest so that we can re-run the demo code with the same pre-loaded data 36 | // Note: To re-run the demo, you need to use the offset reset tool: 37 | // https://cwiki.apache.org/confluence/display/KAFKA/Kafka+Streams+Application+Reset+Tool 38 | props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") 39 | 40 | implicit val stringSerde = Serdes.String 41 | implicit val longSerde = LongAsStringSerde 42 | 43 | val source = KStreamBuilderS.stream[String, String]("streams-file-input") 44 | 45 | val counts: KTableS[String, Long] = source 46 | .flatMapValues { value => value.toLowerCase(Locale.getDefault).split(" ") } 47 | .map { (_, value) => (value, value) } 48 | .groupByKey 49 | .count("Counts") 50 | 51 | counts.to("streams-wordcount-output") 52 | 53 | val streams = new KafkaStreams(KStreamBuilderS.inner, props) 54 | streams.start() 55 | 56 | // usually the stream application would be running forever, 57 | // in this example we just let it run for some time and stop since the input data is finite. 58 | Thread.sleep(5000L) 59 | 60 | streams.close() 61 | } 62 | 63 | } 64 | --------------------------------------------------------------------------------