├── .circleci └── config.yml ├── .gitignore ├── .scala-steward.conf ├── .scalafmt.conf ├── LICENSE ├── README.md ├── avro └── src │ ├── main │ └── scala │ │ └── com │ │ └── ovoenergy │ │ └── kafka │ │ └── serialization │ │ └── avro │ │ └── SchemaRegistryClientSettings.scala │ └── test │ └── scala │ └── com │ └── ovoenergy │ └── kafka │ └── serialization │ └── avro │ └── SchemaRegistryFixture.scala ├── avro4s └── src │ ├── main │ └── scala │ │ └── com │ │ └── ovoenergy │ │ └── kafka │ │ └── serialization │ │ └── avro4s │ │ ├── Avro4sSerialization.scala │ │ └── package.scala │ └── test │ └── scala │ └── com │ └── ovoenergy │ └── kafka │ └── serialization │ └── avro4s │ └── Avro4sSerializationSpec.scala ├── avro4s2 └── src │ ├── main │ └── scala │ │ └── com │ │ └── ovoenergy │ │ └── kafka │ │ └── serialization │ │ └── avro4s2 │ │ ├── Avro4s2Serialization.scala │ │ └── package.scala │ └── test │ └── scala │ └── com │ └── ovoenergy │ └── kafka │ └── serialization │ └── avro4s │ └── Avro4s2SerializationSpec.scala ├── build.sbt ├── build └── tag.sh ├── cats └── src │ ├── main │ └── scala │ │ └── com │ │ └── ovoenergy │ │ └── kafka │ │ └── serialization │ │ └── cats │ │ ├── DeserializerInstances.scala │ │ ├── SerializerInstances.scala │ │ └── package.scala │ └── test │ └── scala │ └── com │ └── ovoenergy │ └── kafka │ └── serialization │ └── cats │ ├── DeserializerInstancesSpec.scala │ └── SerializerInstancesSpec.scala ├── circe └── src │ ├── main │ └── scala │ │ └── com │ │ └── ovoenergy │ │ └── kafka │ │ └── serialization │ │ └── circe │ │ ├── CirceSerialization.scala │ │ └── package.scala │ └── test │ └── scala │ └── com │ └── ovoenergy │ └── kafka │ └── serialization │ └── circe │ └── CirceSerializationSpec.scala ├── core └── src │ ├── main │ └── scala │ │ └── com │ │ └── ovoenergy │ │ └── kafka │ │ └── serialization │ │ └── core │ │ ├── Deserialization.scala │ │ ├── Format.scala │ │ ├── Implicits.scala │ │ ├── Serialization.scala │ │ ├── UnsupportedFormatException.scala │ │ ├── package.scala │ │ └── syntax │ │ └── package.scala │ └── test │ └── scala │ └── com │ └── ovoenergy │ └── kafka │ └── serialization │ └── core │ ├── DeserializationSpec.scala │ └── SerializationSpec.scala ├── doc └── src │ └── README.md ├── json4s └── src │ ├── main │ └── scala │ │ └── com │ │ └── ovoenergy │ │ └── kafka │ │ └── serialization │ │ └── json4s │ │ ├── Json4sSerialization.scala │ │ └── package.scala │ └── test │ └── scala │ └── com │ └── ovoenergy │ └── kafka │ └── serialization │ └── json4s │ └── Json4sSerializationSpec.scala ├── jsoniter-scala └── src │ ├── main │ └── scala │ │ └── com │ │ └── ovoenergy │ │ └── kafka │ │ └── serialization │ │ └── jsoniter_scala │ │ ├── JsoniterScalaSerialization.scala │ │ └── package.scala │ └── test │ └── scala │ └── com │ └── ovoenergy │ └── kafka │ └── serialization │ └── jsoniter_scala │ └── JsoniterScalaSerializationSpec.scala ├── project ├── build.properties └── plugins.sbt ├── spray └── src │ ├── main │ └── scala │ │ └── com │ │ └── ovoenergy │ │ └── kafka │ │ └── serialization │ │ └── spray │ │ ├── SpraySerialization.scala │ │ └── package.scala │ └── test │ └── scala │ └── com │ └── ovoenergy │ └── kafka │ └── serialization │ └── spray │ └── SpraySerializationSpec.scala └── testkit └── src └── main ├── resources ├── application.conf └── logback-test.xml └── scala └── com └── ovoenergy └── kafka └── serialization └── testkit ├── UnitSpec.scala └── WireMockFixture.scala /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | orbs: 4 | snyk: snyk/snyk@1.1.2 5 | 6 | defaults: 7 | - &defaults 8 | docker: 9 | - image: cimg/openjdk:11.0 10 | - &save_dependencies_cache 11 | save_cache: 12 | paths: 13 | - ~/.ivy2 14 | - ~/.sbt 15 | key: dependencies-v2-{{ .Branch }}-{{ checksum "build.sbt" }} 16 | - &restore_dependencies_cache 17 | restore_cache: 18 | keys: 19 | - dependencies-v2-{{ .Branch }}-{{ checksum "build.sbt" }} 20 | - dependencies-v2-{{ .Branch }} 21 | - dependencies-v2 22 | 23 | - &configure_git_credetials 24 | run: 25 | name: Configure git credentials 26 | command: | 27 | echo 'Adding the github host SSH key...' 28 | mkdir -p -m 0700 ~/.ssh/ 29 | ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts 30 | git config user.name ovo-comms-circleci 31 | git config user.email "hello.comms@ovoenergy.com" 32 | 33 | 34 | jobs: 35 | snyk_test: 36 | docker: 37 | - image: cimg/openjdk:11.0 38 | steps: 39 | - checkout 40 | - snyk/scan: 41 | project: '${CIRCLE_PROJECT_REPONAME}' 42 | severity-threshold: high 43 | fail-on-issues: false 44 | monitor-on-build: true 45 | organization: 'oep-comms' 46 | 47 | build: 48 | <<: *defaults 49 | 50 | steps: 51 | 52 | - checkout 53 | 54 | - *restore_dependencies_cache 55 | 56 | - run: sbt update 57 | 58 | - *save_dependencies_cache 59 | 60 | - run: 61 | name: Scalafmt Check 62 | command: sbt scalafmtCheck test:scalafmtCheck scalafmtSbtCheck 63 | 64 | - run: 65 | name: Compile 66 | command: sbt test:compile 67 | 68 | - persist_to_workspace: 69 | root: . 70 | paths: # TODO is there a better way to do this? So that the publish step doesn't have to recompile everything. 71 | - target 72 | - project/target 73 | - project/project/target 74 | - avro/target 75 | - avro4s/target 76 | - cats/target 77 | - circe/target 78 | - core/target 79 | - doc/target 80 | - json4s/target 81 | - jsoniter-scala/target 82 | - spray/target 83 | - testkit/target 84 | 85 | unit_test: 86 | 87 | <<: *defaults 88 | 89 | environment: 90 | JAVA_OPTS: "-XX:+CMSClassUnloadingEnabled -XX:MaxMetaspaceSize=512M -XX:MetaspaceSize=512M -Xms1G -Xmx1G -XX:+PrintGCDetails -Xloggc:target/gc.log" 91 | TEST_TIME_FACTOR: 5.0 92 | 93 | steps: 94 | - checkout 95 | 96 | - attach_workspace: 97 | at: . 98 | 99 | - *restore_dependencies_cache 100 | 101 | - run: 102 | name: Test 103 | command: sbt test:test 104 | 105 | - store_test_results: 106 | path: target/test-reports 107 | 108 | - store_artifacts: 109 | path: target/gc.log 110 | 111 | tag: 112 | 113 | <<: *defaults 114 | 115 | steps: 116 | 117 | - checkout 118 | 119 | - attach_workspace: 120 | at: . 121 | 122 | - *restore_dependencies_cache 123 | 124 | - *configure_git_credetials 125 | 126 | - run: 127 | name: Tag Release 128 | command: build/tag.sh 129 | 130 | publish: 131 | 132 | <<: *defaults 133 | 134 | steps: 135 | 136 | - checkout 137 | 138 | - attach_workspace: 139 | at: . 140 | 141 | - *restore_dependencies_cache 142 | 143 | - *configure_git_credetials 144 | 145 | - run: 146 | name: Release 147 | command: sbt publish 148 | 149 | workflows: 150 | build-and-deploy: 151 | jobs: 152 | - build 153 | - snyk_test: 154 | context: 155 | - comms-internal-build 156 | filters: 157 | branches: 158 | only: 159 | - master 160 | - unit_test: 161 | requires: 162 | - build 163 | - tag: 164 | requires: 165 | - unit_test 166 | filters: 167 | branches: 168 | only: 169 | - master 170 | - publish: 171 | context: comms-internal-build 172 | requires: 173 | - tag 174 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Idea IDE related 2 | .idea/ 3 | .idea_modules/ 4 | *.iml 5 | .bsp/ 6 | 7 | # Ensime 8 | .ensime 9 | .ensime_cache/ 10 | 11 | # Metals 12 | .metals/ 13 | .bloop/ 14 | 15 | # Sbt 16 | target/ 17 | .history 18 | 19 | # Mac OSX 20 | .DS_Store 21 | 22 | # Project specific 23 | test.log -------------------------------------------------------------------------------- /.scala-steward.conf: -------------------------------------------------------------------------------- 1 | updates.ignore = [{ groupId = "com.sksamuel.avro4s" }] 2 | -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | style = IntelliJ, 2 | maxColumn = 120 3 | 4 | project { 5 | git = true 6 | } 7 | 8 | align { 9 | openParenCallSite = false 10 | } 11 | 12 | includeCurlyBraceInSelectChains = true 13 | 14 | optIn { 15 | breakChainOnFirstMethodDot = true 16 | } 17 | 18 | rewrite { 19 | rules = [ 20 | SortImports, 21 | RedundantBraces, 22 | RedundantParens, 23 | PreferCurlyFors 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2016 OVO Energy 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kafka serialization/deserialization building blocks 2 | 3 | [![CircleCI Badge](https://circleci.com/gh/ovotech/kafka-serialization.svg?style=shield)](https://circleci.com/gh/ovotech/kafka-serialization) 4 | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/a2d814f22d4e4facae0f8a3eb1c841fd)](https://www.codacy.com/app/filippo-deluca/kafka-serialization?utm_source=github.com&utm_medium=referral&utm_content=ovotech/kafka-serialization&utm_campaign=Badge_Grade) 5 | [Download](https://kaluza.jfrog.io/artifactory/maven/com/ovoenergy/kafka-serialization-core_2.12/[RELEASE]/kafka-serialization-core_2.12-[RELEASE].jar) 6 | 7 | The aim of this library is to provide the Lego™ bricks to build a serializer/deserializer for kafka messages. 8 | 9 | The serializers/deserializers built by this library cannot be used in the Kafka configuration through properties, but 10 | need to be passed through the Kafka Producer/Consumer constructors (It is feature IMHO). 11 | 12 | For the Avro serialization this library uses Avro4s while for JSON it supports Json4s, Circe and Spray out of the box. 13 | It is quite easy to add support for other libraries as well. 14 | 15 | ## Modules 16 | 17 | The library is composed by these modules: 18 | 19 | - kafka-serialization-core: provides the serialization primitives to build serializers and deserializers. 20 | - kafka-serialization-cats: provides cats typeclasses instances for serializers and deserializers. 21 | - kafka-serialization-json4s: provides serializer and deserializer based on Json4s 22 | - kafka-serialization-jsoniter-scala: provides serializer and deserializer based on Jsoniter Scala 23 | - kafka-serialization-spray: provides serializer and deserializer based on Spray Json 24 | - kafka-serialization-circe: provides serializer and deserializer based on Circe 25 | - kafka-serialization-avro: provides an schema-registry client settings 26 | - kafka-serialization-avro4s: provides serializer and deserializer based on Avro4s 1.x 27 | - kafka-serialization-avro4s2: provides serializer and deserializer based on Avro4s 2.x 28 | 29 | The Avro4s serialization support the schema evolution through the schema registry. The consumer can provide its own schema 30 | and Avro will take care of the conversion. 31 | 32 | ## Getting Started 33 | 34 | - The library is available in the Kaluza artifactory repository. 35 | - See [here](https://kaluza.jfrog.io/artifactory/maven/com/ovoenergy/kafka-serialization-core_2.12/) for the latest version. 36 | - Add this snippet to your build.sbt to use it: 37 | 38 | ```sbtshell 39 | import sbt._ 40 | import sbt.Keys. 41 | 42 | resolvers += "Artifactory" at "https://kaluza.jfrog.io/artifactory/maven" 43 | 44 | libraryDependencies ++= { 45 | val kafkaSerializationV = "0.5.25" 46 | Seq( 47 | "com.ovoenergy" %% "kafka-serialization-core" % kafkaSerializationV, 48 | "com.ovoenergy" %% "kafka-serialization-circe" % kafkaSerializationV, // To provide Circe JSON support 49 | "com.ovoenergy" %% "kafka-serialization-json4s" % kafkaSerializationV, // To provide Json4s JSON support 50 | "com.ovoenergy" %% "kafka-serialization-jsoniter-scala" % kafkaSerializationV, // To provide Jsoniter Scala JSON support 51 | "com.ovoenergy" %% "kafka-serialization-spray" % kafkaSerializationV, // To provide Spray-json JSON support 52 | "com.ovoenergy" %% "kafka-serialization-avro4s" % kafkaSerializationV // To provide Avro4s Avro support 53 | ) 54 | } 55 | 56 | ``` 57 | 58 | ## Circe example 59 | 60 | Circe is a JSON library for Scala that provides support for generic programming trough Shapeless. You can find more 61 | information on the [Circe website](https://circe.github.io/circe). 62 | 63 | Simple serialization/deserialization example with Circe: 64 | 65 | ```scala 66 | import com.ovoenergy.kafka.serialization.core._ 67 | import com.ovoenergy.kafka.serialization.circe._ 68 | 69 | // Import the Circe generic support 70 | import io.circe.generic.auto._ 71 | import io.circe.syntax._ 72 | 73 | import org.apache.kafka.clients.producer.KafkaProducer 74 | import org.apache.kafka.clients.consumer.KafkaConsumer 75 | import org.apache.kafka.clients.CommonClientConfigs._ 76 | 77 | import scala.collection.JavaConverters._ 78 | 79 | case class UserCreated(id: String, name: String, age: Int) 80 | 81 | val producer = new KafkaProducer( 82 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 83 | nullSerializer[Unit], 84 | circeJsonSerializer[UserCreated] 85 | ) 86 | 87 | val consumer = new KafkaConsumer( 88 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 89 | nullDeserializer[Unit], 90 | circeJsonDeserializer[UserCreated] 91 | ) 92 | ``` 93 | 94 | 95 | ## Jsoniter Scala example 96 | 97 | [Jsoniter Scala](https://github.com/plokhotnyuk/jsoniter-scala). is a library that generates codecs for case classes, 98 | standard types and collections to get maximum performance of JSON parsing & serialization. 99 | 100 | Here is an example of serialization/deserialization with Jsoniter Scala: 101 | 102 | ```scala 103 | import com.ovoenergy.kafka.serialization.core._ 104 | import com.ovoenergy.kafka.serialization.jsoniter_scala._ 105 | 106 | // Import the Jsoniter Scala macros & core support 107 | import com.github.plokhotnyuk.jsoniter_scala.macros._ 108 | import com.github.plokhotnyuk.jsoniter_scala.core._ 109 | 110 | import org.apache.kafka.clients.producer.KafkaProducer 111 | import org.apache.kafka.clients.consumer.KafkaConsumer 112 | import org.apache.kafka.clients.CommonClientConfigs._ 113 | 114 | import scala.collection.JavaConverters._ 115 | 116 | case class UserCreated(id: String, name: String, age: Int) 117 | 118 | implicit val userCreatedCodec: JsonValueCodec[UserCreated] = JsonCodecMaker.make[UserCreated](CodecMakerConfig) 119 | 120 | val producer = new KafkaProducer( 121 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 122 | nullSerializer[Unit], 123 | jsoniterScalaSerializer[UserCreated]() 124 | ) 125 | 126 | val consumer = new KafkaConsumer( 127 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 128 | nullDeserializer[Unit], 129 | jsoniterScalaDeserializer[UserCreated]() 130 | ) 131 | ``` 132 | 133 | 134 | ## Avro example 135 | 136 | Apache Avro is a remote procedure call and data serialization framework developed within Apache's Hadoop project. It uses 137 | JSON for defining data types and protocols, and serializes data in a compact binary format. 138 | 139 | Apache Avro provide some support to evolve your messages across multiple version without breaking compatibility with 140 | older or newer consumers. It supports several encoding formats but two are the most used in Kafka: Binary and Json. 141 | 142 | The encoded data is always validated and parsed using a Schema (defined in JSON) and eventually evolved to the reader 143 | Schema version. 144 | 145 | This library provided the support to Avro by using the [Avro4s](https://github.com/sksamuel/avro4s) libray. It uses macro 146 | and shapeless to allowing effortless serialization and deserialization. In addition to Avro4s it need a Confluent schema 147 | registry in place, It will provide a way to control the format of the messages produced in kafka. You can find more 148 | information in the [Confluent Schema Registry Documentation ](http://docs.confluent.io/current/schema-registry/docs/). 149 | 150 | 151 | An example with Avro4s binary and Schema Registry: 152 | ```scala 153 | import com.ovoenergy.kafka.serialization.core._ 154 | import com.ovoenergy.kafka.serialization.avro4s._ 155 | 156 | import com.sksamuel.avro4s._ 157 | 158 | import org.apache.kafka.clients.producer.KafkaProducer 159 | import org.apache.kafka.clients.consumer.KafkaConsumer 160 | import org.apache.kafka.clients.CommonClientConfigs._ 161 | 162 | import scala.collection.JavaConverters._ 163 | 164 | val schemaRegistryEndpoint = "http://localhost:8081" 165 | 166 | case class UserCreated(id: String, name: String, age: Int) 167 | 168 | // This type class is need by the avroBinarySchemaIdSerializer 169 | implicit val UserCreatedToRecord = ToRecord[UserCreated] 170 | 171 | val producer = new KafkaProducer( 172 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 173 | nullSerializer[Unit], 174 | avroBinarySchemaIdSerializer[UserCreated](schemaRegistryEndpoint, isKey = false, includesFormatByte = true) 175 | ) 176 | 177 | // This type class is need by the avroBinarySchemaIdDeserializer 178 | implicit val UserCreatedFromRecord = FromRecord[UserCreated] 179 | 180 | val consumer = new KafkaConsumer( 181 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 182 | nullDeserializer[Unit], 183 | avroBinarySchemaIdDeserializer[UserCreated](schemaRegistryEndpoint, isKey = false, includesFormatByte = true) 184 | ) 185 | ``` 186 | 187 | 188 | This Avro serializer will try to register the schema every new message type it will serialize and will save the obtained 189 | schema id in cache. The deserializer will contact the schema registry each time it will encounter a message with a never 190 | seen before schema id. 191 | 192 | The schema id will encoded in the first 4 bytes of the payload. The deserializer will extract the schema id from the 193 | payload and fetch the schema from the schema registry. The deserializer is able to evolve the original message to the 194 | consumer schema. The use case is when the consumer is only interested in a part of the original message (schema projection) 195 | or when the original message is in a older or newer format of the cosumer schema (schema evolution). 196 | 197 | An example of the consumer schema: 198 | ```scala 199 | import com.ovoenergy.kafka.serialization.core._ 200 | import com.ovoenergy.kafka.serialization.avro4s._ 201 | 202 | import com.sksamuel.avro4s._ 203 | 204 | import org.apache.kafka.clients.producer.KafkaProducer 205 | import org.apache.kafka.clients.consumer.KafkaConsumer 206 | import org.apache.kafka.clients.CommonClientConfigs._ 207 | 208 | import scala.collection.JavaConverters._ 209 | 210 | val schemaRegistryEndpoint = "http://localhost:8081" 211 | 212 | /* Assuming the original message has been serialized using the 213 | * previously defined UserCreated class. We are going to project 214 | * it ignoring the value of the age 215 | */ 216 | case class UserCreated(id: String, name: String) 217 | 218 | // This type class is need by the avroBinarySchemaIdDeserializer 219 | implicit val UserCreatedFromRecord = FromRecord[UserCreated] 220 | 221 | 222 | /* This type class is need by the avroBinarySchemaIdDeserializer 223 | * to obtain the consumer schema 224 | */ 225 | implicit val UserCreatedSchemaFor = SchemaFor[UserCreated] 226 | 227 | val consumer = new KafkaConsumer( 228 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 229 | nullDeserializer[Unit], 230 | avroBinarySchemaIdWithReaderSchemaDeserializer[UserCreated](schemaRegistryEndpoint, isKey = false, includesFormatByte = false) 231 | ) 232 | ``` 233 | 234 | 235 | ## Format byte 236 | 237 | The Original Confluent Avro serializer/deserializer prefix the payload with a "magic" byte to identify that the message 238 | has been written with the Avro serializer. 239 | 240 | Similarly this library support the same mechanism by mean of a couple of function. It is even able to multiplex and 241 | demultiplex different serializers/deserializers based on that format byte. At the moment the supported formats are 242 | - JSON 243 | - Avro Binary with schema ID 244 | - Avro JSON with schema ID 245 | 246 | let's see this mechanism in action: 247 | ```scala 248 | import com.ovoenergy.kafka.serialization.core._ 249 | import com.ovoenergy.kafka.serialization.avro4s._ 250 | import com.ovoenergy.kafka.serialization.circe._ 251 | 252 | // Import the Circe generic support 253 | import io.circe.generic.auto._ 254 | import io.circe.syntax._ 255 | 256 | import org.apache.kafka.clients.producer.KafkaProducer 257 | import org.apache.kafka.clients.consumer.KafkaConsumer 258 | import org.apache.kafka.clients.CommonClientConfigs._ 259 | import scala.collection.JavaConverters._ 260 | 261 | 262 | sealed trait Event 263 | case class UserCreated(id: String, name: String, email: String) extends Event 264 | 265 | val schemaRegistryEndpoint = "http://localhost:8081" 266 | 267 | /* This producer will produce messages in Avro binary format */ 268 | val avroBinaryProducer = new KafkaProducer( 269 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 270 | nullSerializer[Unit], 271 | formatSerializer(Format.AvroBinarySchemaId, avroBinarySchemaIdSerializer[UserCreated](schemaRegistryEndpoint, isKey = false, includesFormatByte = false)) 272 | ) 273 | 274 | /* This producer will produce messages in Json format */ 275 | val circeProducer = new KafkaProducer( 276 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 277 | nullSerializer[Unit], 278 | formatSerializer(Format.Json, circeJsonSerializer[UserCreated]) 279 | ) 280 | 281 | /* This consumer will be able to consume messages from both producer */ 282 | val consumer = new KafkaConsumer( 283 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 284 | nullDeserializer[Unit], 285 | formatDemultiplexerDeserializer[UserCreated](unknownFormat => failingDeserializer(new RuntimeException("Unsupported format"))){ 286 | case Format.Json => circeJsonDeserializer[UserCreated] 287 | case Format.AvroBinarySchemaId => avroBinarySchemaIdDeserializer[UserCreated](schemaRegistryEndpoint, isKey = false, includesFormatByte = false) 288 | } 289 | ) 290 | 291 | /* This consumer will be able to consume messages in Avro binary format with the magic format byte at the start */ 292 | val avroBinaryConsumer = new KafkaConsumer( 293 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 294 | nullDeserializer[Unit], 295 | avroBinarySchemaIdDeserializer[UserCreated](schemaRegistryEndpoint, isKey = false, includesFormatByte = true) 296 | ) 297 | ``` 298 | 299 | 300 | You can notice that the `formatDemultiplexerDeserializer` is little bit nasty because it is invariant in the type `T` so 301 | all the demultiplexed `serialiazer` must be declared as `Deserializer[T]`. 302 | 303 | There are other support serializer and deserializer, you can discover them looking trough the code and the tests. 304 | 305 | ## Useful de-serializers 306 | 307 | In the core module there are pleanty of serializers and deserializers that handle generic cases. 308 | 309 | ### Optional deserializer 310 | 311 | To handle the case in which the data is null, you need to wrap the deserializer in the `optionalDeserializer`: 312 | 313 | ```scala 314 | import com.ovoenergy.kafka.serialization.core._ 315 | import com.ovoenergy.kafka.serialization.circe._ 316 | 317 | // Import the Circe generic support 318 | import io.circe.generic.auto._ 319 | import io.circe.syntax._ 320 | 321 | import org.apache.kafka.common.serialization.Deserializer 322 | 323 | case class UserCreated(id: String, name: String, age: Int) 324 | 325 | val userCreatedDeserializer: Deserializer[Option[UserCreated]] = optionalDeserializer(circeJsonDeserializer[UserCreated]) 326 | ``` 327 | 328 | ## Cats instances 329 | 330 | The `cats` module provides the `Functor` typeclass instance for the `Deserializer` and `Contravariant` instance for the 331 | `Serializer`. This allow to do: 332 | 333 | ```scala 334 | import cats.implicits._ 335 | import com.ovoenergy.kafka.serialization.core._ 336 | import com.ovoenergy.kafka.serialization.cats._ 337 | import org.apache.kafka.common.serialization.{Serializer, Deserializer, IntegerSerializer, IntegerDeserializer} 338 | 339 | val intDeserializer: Deserializer[Int] = (new IntegerDeserializer).asInstanceOf[Deserializer[Int]] 340 | val stringDeserializer: Deserializer[String] = intDeserializer.map(_.toString) 341 | 342 | val intSerializer: Serializer[Int] = (new IntegerSerializer).asInstanceOf[Serializer[Int]] 343 | val stringSerializer: Serializer[String] = intSerializer.contramap(_.toInt) 344 | ``` 345 | 346 | ## Complaints and other Feedback 347 | 348 | Feedback of any kind is always appreciated. 349 | 350 | Issues and PR's are welcome as well. 351 | 352 | ## About this README 353 | 354 | The code samples in this README file are checked using [mdoc](https://github.com/scalameta/mdoc). 355 | 356 | This means that the `README.md` file is generated from `docs/src/README.md`. If you want to make any changes to the README, you should: 357 | 358 | 1. Edit `docs/src/README.md` 359 | 2. Run `sbt mdoc` to regenerate `./README.md` 360 | 3. Commit both files to git -------------------------------------------------------------------------------- /avro/src/main/scala/com/ovoenergy/kafka/serialization/avro/SchemaRegistryClientSettings.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.avro 18 | 19 | case class SchemaRegistryClientSettings(endpoint: String, 20 | authentication: Authentication, 21 | maxCacheSize: Int, 22 | cacheParallelism: Int) 23 | 24 | object SchemaRegistryClientSettings { 25 | 26 | val DefaultCacheSize: Int = 12 27 | 28 | val DefaultCacheParallelism: Int = 4 29 | 30 | val DefaultAuthentication: Authentication = Authentication.None 31 | 32 | def apply(endpoint: String): SchemaRegistryClientSettings = 33 | SchemaRegistryClientSettings(endpoint, DefaultAuthentication, DefaultCacheSize, DefaultCacheParallelism) 34 | 35 | def apply(endpoint: String, username: String, password: String): SchemaRegistryClientSettings = 36 | SchemaRegistryClientSettings( 37 | endpoint, 38 | Authentication.Basic(username, password), 39 | DefaultCacheSize, 40 | DefaultCacheParallelism 41 | ) 42 | 43 | def apply(endpoint: String, maxCacheSize: Int): SchemaRegistryClientSettings = 44 | SchemaRegistryClientSettings(endpoint, DefaultAuthentication, maxCacheSize, DefaultCacheParallelism) 45 | 46 | } 47 | 48 | sealed trait Authentication 49 | 50 | object Authentication { 51 | 52 | case object None extends Authentication 53 | 54 | case class Basic(username: String, password: String) extends Authentication 55 | 56 | } 57 | -------------------------------------------------------------------------------- /avro/src/test/scala/com/ovoenergy/kafka/serialization/avro/SchemaRegistryFixture.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.avro 18 | 19 | import com.github.tomakehurst.wiremock.client.WireMock._ 20 | import com.ovoenergy.kafka.serialization.testkit.WireMockFixture 21 | import org.apache.avro.Schema 22 | import org.scalatest.{BeforeAndAfterEach, Suite} 23 | 24 | trait SchemaRegistryFixture extends BeforeAndAfterEach { _: Suite with WireMockFixture => 25 | 26 | def schemaRegistryEndpoint: String = wireMockEndpoint 27 | 28 | override protected def beforeEach(): Unit = { 29 | super.beforeEach() 30 | 31 | // Some default behaviours 32 | 33 | stubFor( 34 | get(urlMatching(s"/schemas/ids/.*")) 35 | .atPriority(Int.MaxValue) 36 | .willReturn( 37 | aResponse() 38 | .withStatus(404) 39 | .withBody("""{ 40 | | "error_code": 40403, 41 | | "message": "Schema not found" 42 | |} 43 | """.stripMargin) 44 | .withHeader("Content-Type", "application/vnd.schemaregistry.v1+json") 45 | ) 46 | ) 47 | 48 | stubFor( 49 | post(urlMatching("/subjects/.*/versions")) 50 | .atPriority(Int.MaxValue) 51 | .willReturn( 52 | aResponse() 53 | .withBody("{\"id\": 999}") 54 | .withHeader("Content-Type", "application/vnd.schemaregistry.v1+json") 55 | ) 56 | ) 57 | } 58 | 59 | def givenSchema(schemaId: Int, schema: Schema): Unit = { 60 | 61 | // The schema property is a string containing JSON. 62 | val schemaBody = "{\"schema\": \"" + schema.toString.replace(""""""", """\"""") + "\"}" 63 | 64 | stubFor( 65 | get(urlMatching(s"/schemas/ids/$schemaId")) 66 | .willReturn( 67 | aResponse() 68 | .withBody(schemaBody) 69 | .withHeader("Content-Type", "application/vnd.schemaregistry.v1+json") 70 | ) 71 | ) 72 | } 73 | 74 | def givenNonExistingSchema(schemaId: Int): Unit = 75 | stubFor( 76 | get(urlMatching(s"/schemas/ids/$schemaId")) 77 | .willReturn( 78 | aResponse() 79 | .withStatus(404) 80 | .withBody("""{ 81 | | "error_code": 40403, 82 | | "message": "Schema not found" 83 | |} 84 | """.stripMargin) 85 | .withHeader("Content-Type", "application/vnd.schemaregistry.v1+json") 86 | ) 87 | ) 88 | 89 | // TODO match on the schema 90 | def givenNextSchemaId(subject: String, schemaId: Int): Unit = 91 | stubFor( 92 | post(urlMatching(s"/subjects/$subject/versions")) 93 | .willReturn( 94 | aResponse() 95 | .withStatus(200) 96 | .withBody("{\"id\": " + schemaId + "}") 97 | .withHeader("Content-Type", "application/vnd.schemaregistry.v1+json") 98 | ) 99 | ) 100 | 101 | // TODO verify the schema is the same 102 | def verifySchemaHasBeenPosted(subject: String) = 103 | verify(postRequestedFor(urlEqualTo(s"/subjects/$subject/versions"))) 104 | 105 | def givenNextError(status: Int, errorCode: Int, errorMessage: String): Unit = 106 | stubFor( 107 | any(anyUrl()) 108 | .willReturn( 109 | aResponse() 110 | .withStatus(status) 111 | .withBody("{\"error_code\": " + errorCode + ", \"message\": \"" + errorMessage + "\"}") 112 | .withHeader("Content-Type", "application/vnd.schemaregistry.v1+json") 113 | ) 114 | ) 115 | 116 | def givenHtmlResponse(status: Int, body: String): Unit = 117 | stubFor( 118 | any(anyUrl()) 119 | .willReturn( 120 | aResponse() 121 | .withStatus(status) 122 | .withBody(body) 123 | .withHeader("Content-Type", "text/html") 124 | ) 125 | ) 126 | 127 | def givenJsonResponse(status: Int, body: String, url: String): Unit = 128 | stubFor( 129 | any(urlEqualTo(url)) 130 | .willReturn( 131 | aResponse() 132 | .withStatus(status) 133 | .withBody(body) 134 | .withHeader("Content-Type", "application/json") 135 | ) 136 | ) 137 | 138 | } 139 | -------------------------------------------------------------------------------- /avro4s/src/main/scala/com/ovoenergy/kafka/serialization/avro4s/Avro4sSerialization.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.avro4s 18 | 19 | import java.io.ByteArrayOutputStream 20 | import java.nio.ByteBuffer 21 | 22 | import com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream 23 | import com.ovoenergy.kafka.serialization.avro.{Authentication, SchemaRegistryClientSettings} 24 | import com.ovoenergy.kafka.serialization.core._ 25 | import com.sksamuel.avro4s._ 26 | import io.confluent.kafka.schemaregistry.client.{CachedSchemaRegistryClient, SchemaRegistryClient} 27 | import io.confluent.kafka.serializers.{KafkaAvroDeserializer, KafkaAvroSerializer} 28 | import org.apache.avro.Schema 29 | import org.apache.avro.generic.{GenericDatumReader, GenericDatumWriter, GenericRecord} 30 | import org.apache.avro.io.{DecoderFactory, EncoderFactory} 31 | import org.apache.kafka.common.serialization.{Deserializer => KafkaDeserializer, Serializer => KafkaSerializer} 32 | 33 | import scala.collection.JavaConverters._ 34 | 35 | private[avro4s] trait Avro4sSerialization { 36 | 37 | /** 38 | * Build a deserializer for binary-encoded messages prepended with a four byte Schema Registry schema ID. 39 | * 40 | * Assumes that the schema registry does not require authentication. 41 | * 42 | * @param isKey true if this is a deserializer for keys, false if it is for values 43 | * @param includesFormatByte whether the messages are also prepended with a magic byte to specify the Avro format 44 | */ 45 | def avroBinarySchemaIdDeserializer[T: FromRecord](schemaRegistryEndpoint: String, 46 | isKey: Boolean, 47 | includesFormatByte: Boolean): KafkaDeserializer[T] = 48 | avroBinarySchemaIdDeserializer(SchemaRegistryClientSettings(schemaRegistryEndpoint), isKey, includesFormatByte) 49 | 50 | /** 51 | * Build a deserializer for binary-encoded messages prepended with a four byte Schema Registry schema ID. 52 | * 53 | * @param isKey true if this is a deserializer for keys, false if it is for values 54 | * @param includesFormatByte whether the messages are also prepended with a magic byte to specify the Avro format 55 | */ 56 | def avroBinarySchemaIdDeserializer[T: FromRecord](schemaRegistryClientSettings: SchemaRegistryClientSettings, 57 | isKey: Boolean, 58 | includesFormatByte: Boolean): KafkaDeserializer[T] = { 59 | val schemaRegistryClient = initSchemaRegistryClient(schemaRegistryClientSettings) 60 | avroBinarySchemaIdDeserializerWithProps(schemaRegistryClient, isKey, () => (), includesFormatByte) 61 | } 62 | 63 | /** 64 | * Build a deserializer for binary-encoded messages prepended with a four byte Schema Registry schema ID. 65 | * 66 | * @param isKey true if this is a deserializer for keys, false if it is for values 67 | * @param includesFormatByte whether the messages are also prepended with a magic byte to specify the Avro format 68 | * @param props configuration of the underlying [[KafkaAvroDeserializer]] 69 | */ 70 | def avroBinarySchemaIdDeserializer[T: FromRecord](schemaRegistryClient: SchemaRegistryClient, 71 | isKey: Boolean, 72 | includesFormatByte: Boolean, 73 | props: Map[String, String] = Map()): KafkaDeserializer[T] = 74 | avroBinarySchemaIdDeserializerWithProps(schemaRegistryClient, isKey, () => Unit, includesFormatByte, props) 75 | 76 | private def avroBinarySchemaIdDeserializerWithProps[T: FromRecord]( 77 | schemaRegistryClient: SchemaRegistryClient, 78 | isKey: Boolean, 79 | close: () => Unit, 80 | includesFormatByte: Boolean, 81 | props: Map[String, String] = Map() 82 | ): KafkaDeserializer[T] = { 83 | 84 | val fromRecord = implicitly[FromRecord[T]] 85 | 86 | val d: KafkaAvroDeserializer = new KafkaAvroDeserializer(schemaRegistryClient) 87 | 88 | // The configure method needs the `schema.registry.url` even if the schema registry client has been provided 89 | val properties = props.get("schema.registry.url") match { 90 | case Some(_) => props.asJava 91 | case None => (props + ("schema.registry.url" -> "")).asJava 92 | } 93 | d.configure(properties, isKey) 94 | 95 | deserializer( 96 | { (topic, data) => 97 | val bytes = { 98 | if (includesFormatByte) data 99 | else Array(0: Byte) ++ data // prepend the magic byte before delegating to the KafkaAvroDeserializer 100 | } 101 | 102 | fromRecord(d.deserialize(topic, bytes).asInstanceOf[GenericRecord]) 103 | }, 104 | close 105 | ) 106 | } 107 | 108 | /** 109 | * Build a deserializer for binary-encoded messages prepended with a four byte Schema Registry schema ID, 110 | * allowing you to specify a reader schema. 111 | * 112 | * Assumes that the schema registry does not require authentication. 113 | * 114 | * @param isKey true if this is a deserializer for keys, false if it is for values 115 | * @param includesFormatByte whether the messages are also prepended with a magic byte to specify the Avro format 116 | */ 117 | def avroBinarySchemaIdWithReaderSchemaDeserializer[T: FromRecord: SchemaFor]( 118 | schemaRegistryEndpoint: String, 119 | isKey: Boolean, 120 | includesFormatByte: Boolean 121 | ): KafkaDeserializer[T] = 122 | avroBinarySchemaIdWithReaderSchemaDeserializer( 123 | SchemaRegistryClientSettings(schemaRegistryEndpoint), 124 | isKey, 125 | includesFormatByte 126 | ) 127 | 128 | /** 129 | * Build a deserializer for binary-encoded messages prepended with a four byte Schema Registry schema ID, 130 | * allowing you to specify a reader schema. 131 | * 132 | * @param isKey true if this is a deserializer for keys, false if it is for values 133 | * @param includesFormatByte whether the messages are also prepended with a magic byte to specify the Avro format 134 | */ 135 | def avroBinarySchemaIdWithReaderSchemaDeserializer[T: FromRecord: SchemaFor]( 136 | schemaRegistryClientSettings: SchemaRegistryClientSettings, 137 | isKey: Boolean, 138 | includesFormatByte: Boolean 139 | ): KafkaDeserializer[T] = { 140 | val schemaRegistryClient = initSchemaRegistryClient(schemaRegistryClientSettings) 141 | avroBinarySchemaIdWithReaderSchemaDeserializerWithProps(schemaRegistryClient, isKey, () => (), includesFormatByte) 142 | } 143 | 144 | /** 145 | * Build a deserializer for binary-encoded messages prepended with a four byte Schema Registry schema ID, 146 | * allowing you to specify a reader schema. 147 | * 148 | * @param isKey true if this is a deserializer for keys, false if it is for values 149 | * @param includesFormatByte whether the messages are also prepended with a magic byte to specify the Avro format 150 | * @param props configuration of the underlying [[KafkaAvroDeserializer]] 151 | */ 152 | def avroBinarySchemaIdWithReaderSchemaDeserializer[T: FromRecord: SchemaFor]( 153 | schemaRegistryClient: SchemaRegistryClient, 154 | isKey: Boolean, 155 | includesFormatByte: Boolean, 156 | props: Map[String, String] = Map() 157 | ): KafkaDeserializer[T] = 158 | avroBinarySchemaIdWithReaderSchemaDeserializerWithProps( 159 | schemaRegistryClient, 160 | isKey, 161 | () => (), 162 | includesFormatByte, 163 | props 164 | ) 165 | 166 | @deprecated("There is no need to close the client") 167 | private def avroBinarySchemaIdWithReaderSchemaDeserializerWithProps[T: FromRecord: SchemaFor]( 168 | schemaRegistryClient: SchemaRegistryClient, 169 | isKey: Boolean, 170 | close: () => Unit, 171 | includesFormatByte: Boolean, 172 | props: Map[String, String] = Map() 173 | ): KafkaDeserializer[T] = { 174 | 175 | val fromRecord = implicitly[FromRecord[T]] 176 | val schemaFor = implicitly[SchemaFor[T]] 177 | 178 | val d: KafkaAvroDeserializer = new KafkaAvroDeserializer(schemaRegistryClient) 179 | 180 | // The configure method needs the `schema.registry.url` even if the schema registry client has been provided 181 | val properties = props.get("schema.registry.url") match { 182 | case Some(_) => props.asJava 183 | case None => (props + ("schema.registry.url" -> "")).asJava 184 | } 185 | 186 | d.configure(properties, isKey) 187 | 188 | deserializer( 189 | { (topic, data) => 190 | val bytes = { 191 | if (includesFormatByte) data 192 | else Array(0: Byte) ++ data // prepend the magic byte before delegating to the KafkaAvroDeserializer 193 | } 194 | 195 | fromRecord(d.deserialize(topic, bytes, schemaFor()).asInstanceOf[GenericRecord]) 196 | }, 197 | close 198 | ) 199 | } 200 | 201 | /** 202 | * Build a serializer for binary-encoded messages prepended with a four byte Schema Registry schema ID. 203 | * 204 | * Assumes that the schema registry does not require authentication. 205 | * 206 | * @param isKey true if this is a serializer for keys, false if it is for values 207 | * @param includesFormatByte whether the messages should be prepended with a magic byte to specify the Avro format 208 | */ 209 | def avroBinarySchemaIdSerializer[T: ToRecord](schemaRegistryEndpoint: String, 210 | isKey: Boolean, 211 | includesFormatByte: Boolean): KafkaSerializer[T] = 212 | avroBinarySchemaIdSerializer(SchemaRegistryClientSettings(schemaRegistryEndpoint), isKey, includesFormatByte) 213 | 214 | /** 215 | * Build a serializer for binary-encoded messages prepended with a four byte Schema Registry schema ID. 216 | * 217 | * Assumes that the schema registry does not require authentication. 218 | * 219 | * @param isKey true if this is a serializer for keys, false if it is for values 220 | * @param includesFormatByte whether the messages should be prepended with a magic byte to specify the Avro format 221 | */ 222 | def avroBinarySchemaIdSerializer[T: ToRecord](schemaRegistryClientSettings: SchemaRegistryClientSettings, 223 | isKey: Boolean, 224 | includesFormatByte: Boolean): KafkaSerializer[T] = { 225 | val schemaRegistryClient = initSchemaRegistryClient(schemaRegistryClientSettings) 226 | avroBinarySchemaIdSerializerWithProps(schemaRegistryClient, isKey, () => (), includesFormatByte) 227 | } 228 | 229 | /** 230 | * Build a serializer for binary-encoded messages prepended with a four byte Schema Registry schema ID. 231 | * 232 | * Assumes that the schema registry does not require authentication. 233 | * 234 | * @param isKey true if this is a serializer for keys, false if it is for values 235 | * @param includesFormatByte whether the messages should be prepended with a magic byte to specify the Avro format 236 | * @param props configuration of the underlying [[KafkaAvroSerializer]] 237 | */ 238 | def avroBinarySchemaIdSerializer[T: ToRecord](schemaRegistryClient: SchemaRegistryClient, 239 | isKey: Boolean, 240 | includesFormatByte: Boolean, 241 | props: Map[String, String] = Map()): KafkaSerializer[T] = 242 | avroBinarySchemaIdSerializerWithProps(schemaRegistryClient, isKey, () => Unit, includesFormatByte, props) 243 | 244 | private def avroBinarySchemaIdSerializerWithProps[T: ToRecord]( 245 | schemaRegistryClient: SchemaRegistryClient, 246 | isKey: Boolean, 247 | close: () => Unit, 248 | includesFormatByte: Boolean, 249 | props: Map[String, String] = Map() 250 | ): KafkaSerializer[T] = { 251 | val toRecord = implicitly[ToRecord[T]] 252 | 253 | val kafkaAvroSerializer = new KafkaAvroSerializer(schemaRegistryClient) 254 | 255 | // The configure method needs the `schema.registry.url` even if the schema registry client has been provided 256 | val properties = props.get("schema.registry.url") match { 257 | case Some(_) => props.asJava 258 | case None => (props + ("schema.registry.url" -> "")).asJava 259 | } 260 | kafkaAvroSerializer.configure(properties, isKey) 261 | 262 | def dropMagicByte(bytes: Array[Byte]): Array[Byte] = 263 | if (bytes != null && bytes.nonEmpty) { 264 | bytes.drop(1) 265 | } else { 266 | bytes 267 | } 268 | 269 | serializer({ (topic, t) => 270 | val bytes = kafkaAvroSerializer.serialize(topic, toRecord(t)) 271 | if (includesFormatByte) 272 | bytes 273 | else 274 | dropMagicByte(bytes) 275 | }, close) 276 | } 277 | 278 | def avroJsonSchemaIdDeserializerWithReaderSchema[T: FromRecord](schemaRegistryEndpoint: String, 279 | isKey: Boolean): KafkaDeserializer[T] = 280 | avroJsonSchemaIdDeserializerWithReaderSchema(SchemaRegistryClientSettings(schemaRegistryEndpoint), isKey) 281 | 282 | def avroJsonSchemaIdDeserializerWithReaderSchema[T: FromRecord]( 283 | schemaRegistryClientSettings: SchemaRegistryClientSettings, 284 | isKey: Boolean 285 | ): KafkaDeserializer[T] = { 286 | val schemaRegistryClient = initSchemaRegistryClient(schemaRegistryClientSettings) 287 | avroJsonSchemaIdDeserializerWithReaderSchema(schemaRegistryClient, isKey, () => ()) 288 | } 289 | 290 | def avroJsonSchemaIdDeserializerWithReaderSchema[T: FromRecord](schemaRegistryClient: SchemaRegistryClient, 291 | isKey: Boolean): KafkaDeserializer[T] = 292 | avroJsonSchemaIdDeserializerWithReaderSchema(schemaRegistryClient, isKey, () => Unit) 293 | 294 | @deprecated("There is no need to close the client") 295 | private def avroJsonSchemaIdDeserializerWithReaderSchema[T: FromRecord](schemaRegistryClient: SchemaRegistryClient, 296 | isKey: Boolean, 297 | close: () => Unit): KafkaDeserializer[T] = 298 | formatCheckingDeserializer(Format.AvroJsonSchemaId, deserializer({ (topic, data) => 299 | val buffer = ByteBuffer.wrap(data) 300 | val schemaId = buffer.getInt 301 | val schema = schemaRegistryClient.getById(schemaId) 302 | 303 | implicit val schemaFor: SchemaFor[T] = new SchemaFor[T] { 304 | override def apply(): Schema = schema 305 | } 306 | 307 | val avroIn = AvroJsonInputStream[T](new ByteBufferBackedInputStream(buffer)) 308 | 309 | avroIn.singleEntity.get 310 | }, close)) 311 | 312 | def avroJsonSchemaIdDeserializer[T: FromRecord: SchemaFor](schemaRegistryEndpoint: String, 313 | isKey: Boolean): KafkaDeserializer[T] = 314 | avroJsonSchemaIdDeserializer(SchemaRegistryClientSettings(schemaRegistryEndpoint), isKey) 315 | 316 | def avroJsonSchemaIdDeserializer[T: FromRecord: SchemaFor](schemaRegistryClientSettings: SchemaRegistryClientSettings, 317 | isKey: Boolean): KafkaDeserializer[T] = { 318 | val schemaRegistryClient = initSchemaRegistryClient(schemaRegistryClientSettings) 319 | avroJsonSchemaIdDeserializer(schemaRegistryClient, isKey, () => ()) 320 | } 321 | 322 | def avroJsonSchemaIdDeserializer[T: FromRecord: SchemaFor](schemaRegistryClient: SchemaRegistryClient, 323 | isKey: Boolean): KafkaDeserializer[T] = 324 | avroJsonSchemaIdDeserializer(schemaRegistryClient, isKey, () => Unit) 325 | 326 | private def avroJsonSchemaIdDeserializer[T: FromRecord: SchemaFor](schemaRegistryClient: SchemaRegistryClient, 327 | isKey: Boolean, 328 | close: () => Unit): KafkaDeserializer[T] = 329 | deserializer({ (topic, data) => 330 | val buffer = ByteBuffer.wrap(data) 331 | val schemaId = buffer.getInt 332 | val writerSchema = schemaRegistryClient.getById(schemaId) 333 | val readerSchema = implicitly[SchemaFor[T]].apply() 334 | 335 | val reader = new GenericDatumReader[GenericRecord](writerSchema, readerSchema) 336 | val jsonDecoder = DecoderFactory.get.jsonDecoder(writerSchema, new ByteBufferBackedInputStream(buffer)) 337 | 338 | val readRecord = reader.read(null, jsonDecoder) 339 | 340 | implicitly[FromRecord[T]].apply(readRecord) 341 | }, close) 342 | 343 | def avroJsonSchemaIdSerializer[T: ToRecord](schemaRegistryEndpoint: String, isKey: Boolean): KafkaSerializer[T] = 344 | avroJsonSchemaIdSerializer(SchemaRegistryClientSettings(schemaRegistryEndpoint), isKey) 345 | 346 | def avroJsonSchemaIdSerializer[T: ToRecord](schemaRegistryClientSettings: SchemaRegistryClientSettings, 347 | isKey: Boolean): KafkaSerializer[T] = { 348 | val schemaRegistryClient = initSchemaRegistryClient(schemaRegistryClientSettings) 349 | avroJsonSchemaIdSerializer(schemaRegistryClient, isKey, () => ()) 350 | } 351 | 352 | def avroJsonSchemaIdSerializer[T: ToRecord](schemaRegistryClient: SchemaRegistryClient, 353 | isKey: Boolean): KafkaSerializer[T] = 354 | avroJsonSchemaIdSerializer(schemaRegistryClient, isKey, () => Unit) 355 | 356 | private def avroJsonSchemaIdSerializer[T: ToRecord](schemaRegistryClient: SchemaRegistryClient, 357 | isKey: Boolean, 358 | close: () => Unit): KafkaSerializer[T] = { 359 | 360 | val toRecord: ToRecord[T] = implicitly 361 | 362 | serializer({ (topic, t) => 363 | val record = toRecord(t) 364 | implicit val schemaFor: SchemaFor[T] = new SchemaFor[T] { 365 | override def apply(): Schema = record.getSchema 366 | } 367 | 368 | val schemaId = schemaRegistryClient.register(s"$topic-${if (isKey) "key" else "value"}", record.getSchema) 369 | 370 | // TODO size hint 371 | val bout = new ByteArrayOutputStream() 372 | val writer = new GenericDatumWriter[GenericRecord](record.getSchema) 373 | val jsonEncoder = EncoderFactory.get.jsonEncoder(record.getSchema, bout) 374 | 375 | writer.write(toRecord(t), jsonEncoder) 376 | 377 | bout.flush() 378 | bout.close() 379 | 380 | ByteBuffer.allocate(4).putInt(schemaId).array() ++ bout.toByteArray 381 | }, close) 382 | } 383 | 384 | private def initSchemaRegistryClient(settings: SchemaRegistryClientSettings): SchemaRegistryClient = { 385 | val config = settings.authentication match { 386 | case Authentication.Basic(username, password) => 387 | Map( 388 | "basic.auth.credentials.source" -> "USER_INFO", 389 | "schema.registry.basic.auth.user.info" -> s"$username:$password" 390 | ) 391 | case Authentication.None => 392 | Map.empty[String, String] 393 | } 394 | 395 | new CachedSchemaRegistryClient(settings.endpoint, settings.maxCacheSize, config.asJava) 396 | } 397 | } 398 | -------------------------------------------------------------------------------- /avro4s/src/main/scala/com/ovoenergy/kafka/serialization/avro4s/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization 18 | 19 | package object avro4s extends Avro4sSerialization 20 | -------------------------------------------------------------------------------- /avro4s/src/test/scala/com/ovoenergy/kafka/serialization/avro4s/Avro4sSerializationSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.avro4s 18 | 19 | import java.io.{ByteArrayOutputStream, OutputStream} 20 | import java.nio.ByteBuffer 21 | 22 | import com.github.tomakehurst.wiremock.client.WireMock._ 23 | import com.ovoenergy.kafka.serialization.avro4s.Avro4sSerializationSpec._ 24 | import com.ovoenergy.kafka.serialization.testkit.UnitSpec._ 25 | import com.ovoenergy.kafka.serialization.testkit.{UnitSpec, WireMockFixture} 26 | import com.sksamuel.avro4s.{AvroOutputStream, FromRecord, SchemaFor, ToRecord} 27 | import org.apache.avro.Schema 28 | 29 | object Avro4sSerializationSpec { 30 | 31 | implicit val EventToRecord: ToRecord[Event] = ToRecord[Event] 32 | 33 | implicit val EventFromRecord: FromRecord[Event] = FromRecord[Event] 34 | 35 | implicit val EventSchemaFor: SchemaFor[Event] = SchemaFor[Event] 36 | 37 | } 38 | 39 | class Avro4sSerializationSpec extends UnitSpec with WireMockFixture { 40 | 41 | "Avro4sSerialization" when { 42 | "serializing binary" when { 43 | "is value serializer" should { 44 | "register the schema to the schemaRegistry for value" in forAll { (topic: String, event: Event) => 45 | testWithPostSchemaExpected(s"$topic-value") { 46 | val serializer = avroBinarySchemaIdSerializer(wireMockEndpoint, isKey = false, includesFormatByte = false) 47 | serializer.serialize(topic, event) 48 | } 49 | } 50 | 51 | "register the schema to the schemaRegistry for value only once" in forAll { 52 | (topic: String, events: List[Event]) => 53 | whenever(events.length > 1) { 54 | // This verifies that the HTTP cal to the schema registry happen only once. 55 | testWithPostSchemaExpected(s"$topic-value") { 56 | val serializer = 57 | avroBinarySchemaIdSerializer(wireMockEndpoint, isKey = false, includesFormatByte = false) 58 | events.foreach(event => serializer.serialize(topic, event)) 59 | } 60 | } 61 | } 62 | 63 | } 64 | 65 | "is key serializer" should { 66 | "register the schema to the schemaRegistry for key" in forAll { (topic: String, event: Event) => 67 | testWithPostSchemaExpected(s"$topic-key") { 68 | val serializer = avroBinarySchemaIdSerializer(wireMockEndpoint, isKey = true, includesFormatByte = false) 69 | serializer.serialize(topic, event) 70 | } 71 | } 72 | } 73 | } 74 | 75 | "serializing json" when { 76 | "the value is serializer" should { 77 | "register the schema to the schemaRegistry for value" in forAll { (topic: String, event: Event) => 78 | testWithPostSchemaExpected(s"$topic-value") { 79 | val serializer = avroJsonSchemaIdSerializer(wireMockEndpoint, isKey = false) 80 | serializer.serialize(topic, event) 81 | } 82 | } 83 | 84 | "register the schema to the schemaRegistry for value only once" in forAll { 85 | (topic: String, events: List[Event]) => 86 | whenever(events.length > 1) { 87 | // This verifies that the HTTP cal to the schema registry happen only once. 88 | testWithPostSchemaExpected(s"$topic-value") { 89 | val serializer = avroJsonSchemaIdSerializer(wireMockEndpoint, isKey = false) 90 | events.foreach(event => serializer.serialize(topic, event)) 91 | } 92 | } 93 | } 94 | } 95 | 96 | "is key serializer" should { 97 | "register the schema to the schemaRegistry for key" in forAll { (topic: String, event: Event) => 98 | testWithPostSchemaExpected(s"$topic-key") { 99 | val serializer = avroJsonSchemaIdSerializer(wireMockEndpoint, isKey = true) 100 | serializer.serialize(topic, event) 101 | } 102 | } 103 | } 104 | } 105 | 106 | "deserializing binary" when { 107 | "the writer schema is used" should { 108 | "read the schema from the registry" in forAll { (topic: String, schemaId: Int, event: Event) => 109 | val deserializer = avroBinarySchemaIdDeserializer(wireMockEndpoint, isKey = false, includesFormatByte = false) 110 | 111 | val bytes = asAvroBinaryWithSchemaIdBytes(event, schemaId) 112 | 113 | givenSchema(schemaId, EventSchemaFor()) 114 | 115 | val deserialized = deserializer.deserialize(topic, bytes) 116 | deserialized shouldBe event 117 | } 118 | 119 | "read the schema from the registry only once" in forAll { (topic: String, schemaId: Int, event: Event) => 120 | // The looping nature of scalacheck causes to reuse the same wiremock configuration 121 | resetWireMock() 122 | 123 | val deserializer = avroBinarySchemaIdDeserializer(wireMockEndpoint, isKey = false, includesFormatByte = false) 124 | val bytes = asAvroBinaryWithSchemaIdBytes(event, schemaId) 125 | 126 | givenSchema(schemaId, EventSchemaFor()) 127 | 128 | deserializer.deserialize(topic, bytes) 129 | deserializer.deserialize(topic, bytes) 130 | 131 | verify(1, getRequestedFor(urlMatching(s"/schemas/ids/$schemaId"))) 132 | } 133 | 134 | "handle a format byte in the header" in forAll { (topic: String, schemaId: Int, event: Event) => 135 | val deserializer = avroBinarySchemaIdDeserializer(wireMockEndpoint, isKey = false, includesFormatByte = true) 136 | 137 | val bytes = Array(0: Byte) ++ asAvroBinaryWithSchemaIdBytes(event, schemaId) 138 | 139 | givenSchema(schemaId, EventSchemaFor()) 140 | 141 | val deserialized = deserializer.deserialize(topic, bytes) 142 | deserialized shouldBe event 143 | } 144 | } 145 | } 146 | 147 | "deserializing json" when { 148 | "the writer schema is used" should { 149 | "read the schema from the registry" in forAll { (topic: String, schemaId: Int, event: Event) => 150 | val deserializer = avroJsonSchemaIdDeserializer(wireMockEndpoint, isKey = false) 151 | val bytes = asAvroJsonWithSchemaIdBytes(event, schemaId) 152 | 153 | givenSchema(schemaId, EventSchemaFor()) 154 | 155 | val deserialized = deserializer.deserialize(topic, bytes) 156 | 157 | deserialized shouldBe event 158 | } 159 | 160 | "read the schema from the registry only once" in forAll { (topic: String, schemaId: Int, event: Event) => 161 | // The looping nature of scalacheck causes to reuse the same wiremock configuration 162 | resetWireMock() 163 | 164 | val deserializer = avroJsonSchemaIdDeserializer(wireMockEndpoint, isKey = false) 165 | val bytes = asAvroJsonWithSchemaIdBytes(event, schemaId) 166 | 167 | givenSchema(schemaId, EventSchemaFor()) 168 | 169 | deserializer.deserialize(topic, bytes) 170 | deserializer.deserialize(topic, bytes) 171 | 172 | verify(1, getRequestedFor(urlMatching(s"/schemas/ids/$schemaId"))) 173 | } 174 | } 175 | } 176 | } 177 | 178 | private def givenSchema[T: SchemaFor](schemaId: Int, schema: Schema) = { 179 | 180 | // The schema property is a string containing JSON. 181 | val schemaBody = "{\"schema\": \"" + schema.toString.replace(""""""", """\"""") + "\"}" 182 | 183 | stubFor( 184 | get(urlMatching(s"/schemas/ids/$schemaId")) 185 | .willReturn( 186 | aResponse() 187 | .withBody(schemaBody) 188 | .withHeader("Content-Type", "application/vnd.schemaregistry.v1+json") 189 | ) 190 | ) 191 | } 192 | 193 | private def asAvroBinaryWithSchemaIdBytes[T: SchemaFor: ToRecord](t: T, schemaId: Int): Array[Byte] = 194 | asAvroWithSchemaIdBytes(t, schemaId, AvroOutputStream.binary[T]) 195 | 196 | private def asAvroJsonWithSchemaIdBytes[T: SchemaFor: ToRecord](t: T, schemaId: Int): Array[Byte] = 197 | asAvroWithSchemaIdBytes(t, schemaId, AvroOutputStream.json[T]) 198 | 199 | private def asAvroWithSchemaIdBytes[T: SchemaFor: ToRecord]( 200 | t: T, 201 | schemaId: Int, 202 | mkAvroOut: OutputStream => AvroOutputStream[T] 203 | ): Array[Byte] = { 204 | val bout = new ByteArrayOutputStream() 205 | val avroOut = mkAvroOut(bout) 206 | avroOut.write(t) 207 | avroOut.flush() 208 | ByteBuffer.allocate(bout.size() + 4).putInt(schemaId).put(bout.toByteArray).array() 209 | } 210 | 211 | private def testWithPostSchemaExpected[T](subject: String)(f: => T) = { 212 | 213 | stubFor( 214 | post(urlMatching("/subjects/.*/versions")) 215 | .willReturn( 216 | aResponse() 217 | .withBody("{\"id\": 1}") 218 | .withHeader("Content-Type", "application/vnd.schemaregistry.v1+json") 219 | ) 220 | ) 221 | 222 | val result = f 223 | 224 | // TODO verify the schema is the same 225 | verify(postRequestedFor(urlEqualTo(s"/subjects/$subject/versions"))) 226 | 227 | result 228 | } 229 | } 230 | -------------------------------------------------------------------------------- /avro4s2/src/main/scala/com/ovoenergy/kafka/serialization/avro4s2/Avro4s2Serialization.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization 18 | package avro4s2 19 | 20 | import java.io.ByteArrayOutputStream 21 | import java.nio.ByteBuffer 22 | import scala.collection.JavaConverters._ 23 | 24 | import com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream 25 | 26 | import org.apache.kafka.common.serialization.{Deserializer => KafkaDeserializer, Serializer => KafkaSerializer} 27 | import io.confluent.kafka.schemaregistry.client.{CachedSchemaRegistryClient, SchemaRegistryClient} 28 | import io.confluent.kafka.serializers.{KafkaAvroDeserializer, KafkaAvroSerializer} 29 | 30 | import com.sksamuel.avro4s._ 31 | 32 | import org.apache.avro.Schema 33 | import org.apache.avro.generic.{GenericDatumReader, GenericDatumWriter, GenericRecord} 34 | import org.apache.avro.io.{DecoderFactory, EncoderFactory} 35 | 36 | import avro.{Authentication, SchemaRegistryClientSettings} 37 | import core._ 38 | 39 | private[avro4s2] trait Avro4s2Serialization { 40 | 41 | /** 42 | * Build a deserializer for binary-encoded messages prepended with a four byte Schema Registry schema ID. 43 | * 44 | * Assumes that the schema registry does not require authentication. 45 | * 46 | * @param isKey true if this is a deserializer for keys, false if it is for values 47 | * @param includesFormatByte whether the messages are also prepended with a magic byte to specify the Avro format 48 | */ 49 | def avroBinarySchemaIdDeserializer[T: Decoder](schemaRegistryEndpoint: String, 50 | isKey: Boolean, 51 | includesFormatByte: Boolean): KafkaDeserializer[T] = 52 | avroBinarySchemaIdDeserializer(SchemaRegistryClientSettings(schemaRegistryEndpoint), isKey, includesFormatByte) 53 | 54 | /** 55 | * Build a deserializer for binary-encoded messages prepended with a four byte Schema Registry schema ID. 56 | * 57 | * @param isKey true if this is a deserializer for keys, false if it is for values 58 | * @param includesFormatByte whether the messages are also prepended with a magic byte to specify the Avro format 59 | */ 60 | def avroBinarySchemaIdDeserializer[T: Decoder](schemaRegistryClientSettings: SchemaRegistryClientSettings, 61 | isKey: Boolean, 62 | includesFormatByte: Boolean): KafkaDeserializer[T] = { 63 | val schemaRegistryClient = initSchemaRegistryClient(schemaRegistryClientSettings) 64 | avroBinarySchemaIdDeserializer(schemaRegistryClient, isKey, includesFormatByte) 65 | } 66 | 67 | /** 68 | * Build a deserializer for binary-encoded messages prepended with a four byte Schema Registry schema ID. 69 | * 70 | * @param isKey true if this is a deserializer for keys, false if it is for values 71 | * @param includesFormatByte whether the messages are also prepended with a magic byte to specify the Avro format 72 | * @param props configuration of the underlying [[KafkaAvroDeserializer]] 73 | */ 74 | def avroBinarySchemaIdDeserializer[T]( 75 | schemaRegistryClient: SchemaRegistryClient, 76 | isKey: Boolean, 77 | includesFormatByte: Boolean, 78 | props: Map[String, String] = Map() 79 | )(implicit decoder: Decoder[T]): KafkaDeserializer[T] = { 80 | 81 | val d: KafkaAvroDeserializer = new KafkaAvroDeserializer(schemaRegistryClient) 82 | 83 | // The configure method needs the `schema.registry.url` even if the schema registry client has been provided 84 | val properties = props.get("schema.registry.url") match { 85 | case Some(_) => props.asJava 86 | case None => (props + ("schema.registry.url" -> "")).asJava 87 | } 88 | d.configure(properties, isKey) 89 | 90 | deserializer({ (topic, data) => 91 | val bytes = { 92 | if (includesFormatByte) data 93 | else Array(0: Byte) ++ data // prepend the magic byte before delegating to the KafkaAvroDeserializer 94 | } 95 | 96 | val writerSchema = { 97 | val buffer = ByteBuffer.wrap(bytes) 98 | buffer.get() // Skip the magic byte 99 | val schemaId = buffer.getInt 100 | schemaRegistryClient.getById(schemaId) 101 | } 102 | 103 | decoder.decode(d.deserialize(topic, bytes), writerSchema) 104 | }) 105 | } 106 | 107 | /** 108 | * Build a deserializer for binary-encoded messages prepended with a four byte Schema Registry schema ID, 109 | * allowing you to specify a reader schema. 110 | * 111 | * Assumes that the schema registry does not require authentication. 112 | * 113 | * @param isKey true if this is a deserializer for keys, false if it is for values 114 | * @param includesFormatByte whether the messages are also prepended with a magic byte to specify the Avro format 115 | */ 116 | def avroBinarySchemaIdWithReaderSchemaDeserializer[T: Decoder: SchemaFor]( 117 | schemaRegistryEndpoint: String, 118 | isKey: Boolean, 119 | includesFormatByte: Boolean 120 | ): KafkaDeserializer[T] = 121 | avroBinarySchemaIdWithReaderSchemaDeserializer( 122 | SchemaRegistryClientSettings(schemaRegistryEndpoint), 123 | isKey, 124 | includesFormatByte 125 | ) 126 | 127 | /** 128 | * Build a deserializer for binary-encoded messages prepended with a four byte Schema Registry schema ID, 129 | * allowing you to specify a reader schema. 130 | * 131 | * @param isKey true if this is a deserializer for keys, false if it is for values 132 | * @param includesFormatByte whether the messages are also prepended with a magic byte to specify the Avro format 133 | */ 134 | def avroBinarySchemaIdWithReaderSchemaDeserializer[T: Decoder: SchemaFor]( 135 | schemaRegistryClientSettings: SchemaRegistryClientSettings, 136 | isKey: Boolean, 137 | includesFormatByte: Boolean 138 | ): KafkaDeserializer[T] = { 139 | val schemaRegistryClient = initSchemaRegistryClient(schemaRegistryClientSettings) 140 | avroBinarySchemaIdWithReaderSchemaDeserializer(schemaRegistryClient, isKey, includesFormatByte) 141 | } 142 | 143 | /** 144 | * Build a deserializer for binary-encoded messages prepended with a four byte Schema Registry schema ID, 145 | * allowing you to specify a reader schema. 146 | * 147 | * @param isKey true if this is a deserializer for keys, false if it is for values 148 | * @param includesFormatByte whether the messages are also prepended with a magic byte to specify the Avro format 149 | * @param props configuration of the underlying [[KafkaAvroDeserializer]] 150 | */ 151 | def avroBinarySchemaIdWithReaderSchemaDeserializer[T]( 152 | schemaRegistryClient: SchemaRegistryClient, 153 | isKey: Boolean, 154 | includesFormatByte: Boolean, 155 | props: Map[String, String] = Map() 156 | )(implicit decoder: Decoder[T], schemaFor: SchemaFor[T]): KafkaDeserializer[T] = { 157 | 158 | val d: KafkaAvroDeserializer = new KafkaAvroDeserializer(schemaRegistryClient) 159 | 160 | // The configure method needs the `schema.registry.url` even if the schema registry client has been provided 161 | val properties = props.get("schema.registry.url") match { 162 | case Some(_) => props.asJava 163 | case None => (props + ("schema.registry.url" -> "")).asJava 164 | } 165 | 166 | d.configure(properties, isKey) 167 | 168 | deserializer({ (topic, data) => 169 | val bytes = { 170 | if (includesFormatByte) data 171 | else Array(0: Byte) ++ data // prepend the magic byte before delegating to the KafkaAvroDeserializer 172 | } 173 | 174 | val buffer = ByteBuffer.wrap(bytes) 175 | val schemaId = { 176 | buffer.get() // Skip the magic byte 177 | buffer.getInt 178 | } 179 | 180 | val writerSchema = schemaRegistryClient.getById(schemaId) 181 | 182 | decoder.decode(d.deserialize(topic, bytes, schemaFor.schema), schemaFor.schema) 183 | }) 184 | } 185 | 186 | /** 187 | * Build a serializer for binary-encoded messages prepended with a four byte Schema Registry schema ID. 188 | * 189 | * Assumes that the schema registry does not require authentication. 190 | * 191 | * @param isKey true if this is a serializer for keys, false if it is for values 192 | * @param includesFormatByte whether the messages should be prepended with a magic byte to specify the Avro format 193 | */ 194 | def avroBinarySchemaIdSerializer[T: Encoder: SchemaFor](schemaRegistryEndpoint: String, 195 | isKey: Boolean, 196 | includesFormatByte: Boolean): KafkaSerializer[T] = 197 | avroBinarySchemaIdSerializer(SchemaRegistryClientSettings(schemaRegistryEndpoint), isKey, includesFormatByte) 198 | 199 | /** 200 | * Build a serializer for binary-encoded messages prepended with a four byte Schema Registry schema ID. 201 | * 202 | * Assumes that the schema registry does not require authentication. 203 | * 204 | * @param isKey true if this is a serializer for keys, false if it is for values 205 | * @param includesFormatByte whether the messages should be prepended with a magic byte to specify the Avro format 206 | */ 207 | def avroBinarySchemaIdSerializer[T: Encoder: SchemaFor](schemaRegistryClientSettings: SchemaRegistryClientSettings, 208 | isKey: Boolean, 209 | includesFormatByte: Boolean): KafkaSerializer[T] = { 210 | val schemaRegistryClient = initSchemaRegistryClient(schemaRegistryClientSettings) 211 | avroBinarySchemaIdSerializer(schemaRegistryClient, isKey, includesFormatByte) 212 | } 213 | 214 | /** 215 | * Build a serializer for binary-encoded messages prepended with a four byte Schema Registry schema ID. 216 | * 217 | * Assumes that the schema registry does not require authentication. 218 | * 219 | * @param isKey true if this is a serializer for keys, false if it is for values 220 | * @param includesFormatByte whether the messages should be prepended with a magic byte to specify the Avro format 221 | * @param props configuration of the underlying [[KafkaAvroSerializer]] 222 | */ 223 | def avroBinarySchemaIdSerializer[T]( 224 | schemaRegistryClient: SchemaRegistryClient, 225 | isKey: Boolean, 226 | includesFormatByte: Boolean, 227 | props: Map[String, String] = Map.empty 228 | )(implicit encoder: Encoder[T], schemaFor: SchemaFor[T]): KafkaSerializer[T] = { 229 | 230 | val kafkaAvroSerializer = new KafkaAvroSerializer(schemaRegistryClient) 231 | 232 | // The configure method needs the `schema.registry.url` even if the schema registry client has been provided 233 | val properties = props.get("schema.registry.url") match { 234 | case Some(_) => props.asJava 235 | case None => (props + ("schema.registry.url" -> "")).asJava 236 | } 237 | 238 | kafkaAvroSerializer.configure(properties, isKey) 239 | 240 | def dropMagicByte(bytes: Array[Byte]): Array[Byte] = 241 | if (bytes != null && bytes.nonEmpty) { 242 | bytes.drop(1) 243 | } else { 244 | bytes 245 | } 246 | 247 | serializer({ (topic, t) => 248 | val bytes = kafkaAvroSerializer.serialize(topic, encoder.encode(t, schemaFor.schema)) 249 | if (includesFormatByte) 250 | bytes 251 | else 252 | dropMagicByte(bytes) 253 | }) 254 | } 255 | 256 | def avroJsonSchemaIdDeserializerWithReaderSchema[T: Decoder: SchemaFor]( 257 | schemaRegistryEndpoint: String, 258 | isKey: Boolean, 259 | includesFormatByte: Boolean 260 | ): KafkaDeserializer[T] = 261 | avroJsonSchemaIdDeserializerWithReaderSchema( 262 | SchemaRegistryClientSettings(schemaRegistryEndpoint), 263 | isKey, 264 | includesFormatByte 265 | ) 266 | 267 | def avroJsonSchemaIdDeserializerWithReaderSchema[T: Decoder: SchemaFor]( 268 | schemaRegistryClientSettings: SchemaRegistryClientSettings, 269 | isKey: Boolean, 270 | includesFormatByte: Boolean 271 | ): KafkaDeserializer[T] = { 272 | val schemaRegistryClient = initSchemaRegistryClient(schemaRegistryClientSettings) 273 | avroJsonSchemaIdDeserializerWithReaderSchema(schemaRegistryClient, isKey, includesFormatByte) 274 | } 275 | 276 | def avroJsonSchemaIdDeserializerWithReaderSchema[T]( 277 | schemaRegistryClient: SchemaRegistryClient, 278 | isKey: Boolean, 279 | includesFormatByte: Boolean 280 | )(implicit decoder: Decoder[T], schemaFor: SchemaFor[T]): KafkaDeserializer[T] = { 281 | val des = deserializer({ (topic, data) => 282 | val buffer = ByteBuffer.wrap(data) 283 | val schemaId = buffer.getInt 284 | val writerSchema = schemaRegistryClient.getById(schemaId) 285 | 286 | val reader = new GenericDatumReader[GenericRecord](writerSchema, schemaFor.schema) 287 | val jsonDecoder = DecoderFactory.get.jsonDecoder(writerSchema, new ByteBufferBackedInputStream(buffer)) 288 | 289 | val readRecord = reader.read(null, jsonDecoder) 290 | 291 | decoder.decode(readRecord, schemaFor.schema) 292 | }) 293 | 294 | if (includesFormatByte) { 295 | formatCheckingDeserializer(Format.AvroJsonSchemaId, des) 296 | } else { 297 | des 298 | } 299 | } 300 | 301 | def avroJsonSchemaIdDeserializer[T: Decoder](schemaRegistryEndpoint: String, 302 | isKey: Boolean, 303 | includesFormatByte: Boolean): KafkaDeserializer[T] = 304 | avroJsonSchemaIdDeserializer(SchemaRegistryClientSettings(schemaRegistryEndpoint), isKey, includesFormatByte) 305 | 306 | def avroJsonSchemaIdDeserializer[T: Decoder](schemaRegistryClientSettings: SchemaRegistryClientSettings, 307 | isKey: Boolean, 308 | includesFormatByte: Boolean): KafkaDeserializer[T] = { 309 | val schemaRegistryClient = initSchemaRegistryClient(schemaRegistryClientSettings) 310 | avroJsonSchemaIdDeserializer(schemaRegistryClient, isKey, includesFormatByte) 311 | } 312 | 313 | def avroJsonSchemaIdDeserializer[T]( 314 | schemaRegistryClient: SchemaRegistryClient, 315 | isKey: Boolean, 316 | includesFormatByte: Boolean 317 | )(implicit decoder: Decoder[T]): KafkaDeserializer[T] = { 318 | 319 | val des = deserializer({ (topic, data) => 320 | val buffer = ByteBuffer.wrap(data) 321 | val schemaId = buffer.getInt 322 | val writerSchema = schemaRegistryClient.getById(schemaId) 323 | 324 | val reader = new GenericDatumReader[GenericRecord](writerSchema, writerSchema) 325 | val jsonDecoder = DecoderFactory.get.jsonDecoder(writerSchema, new ByteBufferBackedInputStream(buffer)) 326 | 327 | val readRecord = reader.read(null, jsonDecoder) 328 | 329 | decoder.decode(readRecord, writerSchema) 330 | }) 331 | 332 | if (includesFormatByte) { 333 | formatCheckingDeserializer(Format.AvroJsonSchemaId, des) 334 | } else { 335 | des 336 | } 337 | 338 | } 339 | 340 | def avroJsonSchemaIdSerializer[T: Encoder: SchemaFor](schemaRegistryEndpoint: String, 341 | isKey: Boolean, 342 | includesFormatByte: Boolean): KafkaSerializer[T] = 343 | avroJsonSchemaIdSerializer(SchemaRegistryClientSettings(schemaRegistryEndpoint), isKey, includesFormatByte) 344 | 345 | def avroJsonSchemaIdSerializer[T: Encoder: SchemaFor](schemaRegistryClientSettings: SchemaRegistryClientSettings, 346 | isKey: Boolean, 347 | includesFormatByte: Boolean): KafkaSerializer[T] = { 348 | val schemaRegistryClient = initSchemaRegistryClient(schemaRegistryClientSettings) 349 | avroJsonSchemaIdSerializer(schemaRegistryClient, isKey, includesFormatByte) 350 | } 351 | 352 | def avroJsonSchemaIdSerializer[T]( 353 | schemaRegistryClient: SchemaRegistryClient, 354 | isKey: Boolean, 355 | includesFormatByte: Boolean 356 | )(implicit encoder: Encoder[T], schemaFor: SchemaFor[T]): KafkaSerializer[T] = { 357 | val ser = serializer[T] { (topic: String, t: T) => 358 | val record = encoder.encode(t, schemaFor.schema).asInstanceOf[GenericRecord] 359 | val schemaId = schemaRegistryClient.register(s"$topic-${if (isKey) "key" else "value"}", record.getSchema) 360 | 361 | // TODO size hint 362 | val bout = new ByteArrayOutputStream() 363 | val writer = new GenericDatumWriter[GenericRecord](record.getSchema) 364 | val jsonEncoder = EncoderFactory.get.jsonEncoder(record.getSchema, bout) 365 | 366 | writer.write(record, jsonEncoder) 367 | 368 | bout.flush() 369 | bout.close() 370 | 371 | ByteBuffer.allocate(4).putInt(schemaId).array() ++ bout.toByteArray 372 | } 373 | 374 | if (includesFormatByte) { 375 | formatSerializer(Format.AvroJsonSchemaId, ser) 376 | } else { 377 | ser 378 | } 379 | } 380 | 381 | private def initSchemaRegistryClient(settings: SchemaRegistryClientSettings): SchemaRegistryClient = { 382 | val config = settings.authentication match { 383 | case Authentication.Basic(username, password) => 384 | Map( 385 | "basic.auth.credentials.source" -> "USER_INFO", 386 | "schema.registry.basic.auth.user.info" -> s"$username:$password" 387 | ) 388 | case Authentication.None => 389 | Map.empty[String, String] 390 | } 391 | 392 | new CachedSchemaRegistryClient(settings.endpoint, settings.maxCacheSize, config.asJava) 393 | } 394 | } 395 | -------------------------------------------------------------------------------- /avro4s2/src/main/scala/com/ovoenergy/kafka/serialization/avro4s2/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization 18 | 19 | package object avro4s2 extends Avro4s2Serialization 20 | -------------------------------------------------------------------------------- /avro4s2/src/test/scala/com/ovoenergy/kafka/serialization/avro4s/Avro4s2SerializationSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization 18 | package avro4s2 19 | 20 | import java.io.{ByteArrayOutputStream, OutputStream} 21 | import java.nio.ByteBuffer 22 | 23 | import com.github.tomakehurst.wiremock.client.WireMock._ 24 | import com.ovoenergy.kafka.serialization.testkit.UnitSpec._ 25 | import com.ovoenergy.kafka.serialization.testkit.{UnitSpec, WireMockFixture} 26 | import org.apache.avro.Schema 27 | 28 | import com.sksamuel.avro4s._ 29 | 30 | class Avro4s2SerializationSpec extends UnitSpec with WireMockFixture { 31 | 32 | "Avro4sSerialization" when { 33 | "serializing binary" when { 34 | "is value serializer" should { 35 | "register the schema to the schemaRegistry for value" in forAll { (topic: String, event: Event) => 36 | testWithPostSchemaExpected(s"$topic-value") { 37 | val serializer = 38 | avroBinarySchemaIdSerializer[Event](wireMockEndpoint, isKey = false, includesFormatByte = false) 39 | serializer.serialize(topic, event) 40 | } 41 | } 42 | 43 | "register the schema to the schemaRegistry for value only once" in forAll { 44 | (topic: String, events: List[Event]) => 45 | whenever(events.length > 1) { 46 | // This verifies that the HTTP cal to the schema registry happen only once. 47 | testWithPostSchemaExpected(s"$topic-value") { 48 | val serializer = 49 | avroBinarySchemaIdSerializer[Event](wireMockEndpoint, isKey = false, includesFormatByte = false) 50 | events.foreach(event => serializer.serialize(topic, event)) 51 | } 52 | } 53 | } 54 | 55 | } 56 | 57 | "is key serializer" should { 58 | "register the schema to the schemaRegistry for key" in forAll { (topic: String, event: Event) => 59 | testWithPostSchemaExpected(s"$topic-key") { 60 | val serializer = 61 | avroBinarySchemaIdSerializer[Event](wireMockEndpoint, isKey = true, includesFormatByte = false) 62 | serializer.serialize(topic, event) 63 | } 64 | } 65 | } 66 | } 67 | 68 | "serializing json" when { 69 | "the value is serializer" should { 70 | "register the schema to the schemaRegistry for value" in forAll { (topic: String, event: Event) => 71 | testWithPostSchemaExpected(s"$topic-value") { 72 | val serializer = 73 | avroJsonSchemaIdSerializer[Event](wireMockEndpoint, isKey = false, includesFormatByte = false) 74 | serializer.serialize(topic, event) 75 | } 76 | } 77 | 78 | "register the schema to the schemaRegistry for value only once" in forAll { 79 | (topic: String, events: List[Event]) => 80 | whenever(events.length > 1) { 81 | // This verifies that the HTTP cal to the schema registry happen only once. 82 | testWithPostSchemaExpected(s"$topic-value") { 83 | val serializer = 84 | avroJsonSchemaIdSerializer[Event](wireMockEndpoint, isKey = false, includesFormatByte = false) 85 | events.foreach(event => serializer.serialize(topic, event)) 86 | } 87 | } 88 | } 89 | } 90 | 91 | "is key serializer" should { 92 | "register the schema to the schemaRegistry for key" in forAll { (topic: String, event: Event) => 93 | testWithPostSchemaExpected(s"$topic-key") { 94 | val serializer = 95 | avroJsonSchemaIdSerializer[Event](wireMockEndpoint, isKey = true, includesFormatByte = false) 96 | serializer.serialize(topic, event) 97 | } 98 | } 99 | } 100 | } 101 | 102 | "deserializing binary" when { 103 | "the writer schema is used" should { 104 | "read the schema from the registry" in forAll { (topic: String, schemaId: Int, event: Event) => 105 | val deserializer = 106 | avroBinarySchemaIdDeserializer[Event](wireMockEndpoint, isKey = false, includesFormatByte = false) 107 | 108 | val bytes = asAvroBinaryWithSchemaIdBytes(event, schemaId) 109 | 110 | givenSchema(schemaId, SchemaFor[Event].schema) 111 | 112 | val deserialized = deserializer.deserialize(topic, bytes) 113 | deserialized shouldBe event 114 | } 115 | 116 | "read the schema from the registry only once" in forAll { (topic: String, schemaId: Int, event: Event) => 117 | // The looping nature of scalacheck causes to reuse the same wiremock configuration 118 | resetWireMock() 119 | 120 | val deserializer = 121 | avroBinarySchemaIdDeserializer[Event](wireMockEndpoint, isKey = false, includesFormatByte = false) 122 | val bytes = asAvroBinaryWithSchemaIdBytes(event, schemaId) 123 | 124 | givenSchema(schemaId, SchemaFor[Event].schema) 125 | 126 | deserializer.deserialize(topic, bytes) 127 | deserializer.deserialize(topic, bytes) 128 | 129 | verify(1, getRequestedFor(urlMatching(s"/schemas/ids/$schemaId"))) 130 | } 131 | 132 | "handle a format byte in the header" in forAll { (topic: String, schemaId: Int, event: Event) => 133 | val deserializer = 134 | avroBinarySchemaIdDeserializer[Event](wireMockEndpoint, isKey = false, includesFormatByte = true) 135 | 136 | val bytes = Array(0: Byte) ++ asAvroBinaryWithSchemaIdBytes(event, schemaId) 137 | 138 | givenSchema(schemaId, SchemaFor[Event].schema) 139 | 140 | val deserialized = deserializer.deserialize(topic, bytes) 141 | deserialized shouldBe event 142 | } 143 | } 144 | } 145 | 146 | "deserializing json" when { 147 | "the writer schema is used" should { 148 | "read the schema from the registry" in forAll { (topic: String, schemaId: Int, event: Event) => 149 | val deserializer = 150 | avroJsonSchemaIdDeserializer[Event](wireMockEndpoint, isKey = false, includesFormatByte = false) 151 | val bytes = asAvroJsonWithSchemaIdBytes(event, schemaId) 152 | 153 | givenSchema(schemaId, SchemaFor[Event].schema) 154 | 155 | val deserialized = deserializer.deserialize(topic, bytes) 156 | 157 | deserialized shouldBe event 158 | } 159 | 160 | "read the schema from the registry only once" in forAll { (topic: String, schemaId: Int, event: Event) => 161 | // The looping nature of scalacheck causes to reuse the same wiremock configuration 162 | resetWireMock() 163 | 164 | val deserializer = 165 | avroJsonSchemaIdDeserializer[Event](wireMockEndpoint, isKey = false, includesFormatByte = false) 166 | val bytes = asAvroJsonWithSchemaIdBytes(event, schemaId) 167 | 168 | givenSchema(schemaId, SchemaFor[Event].schema) 169 | 170 | deserializer.deserialize(topic, bytes) 171 | deserializer.deserialize(topic, bytes) 172 | 173 | verify(1, getRequestedFor(urlMatching(s"/schemas/ids/$schemaId"))) 174 | } 175 | } 176 | } 177 | } 178 | 179 | private def givenSchema(schemaId: Int, schema: Schema) = { 180 | 181 | // The schema property is a string containing JSON. 182 | val schemaBody = "{\"schema\": \"" + schema.toString.replace(""""""", """\"""") + "\"}" 183 | 184 | stubFor( 185 | get(urlMatching(s"/schemas/ids/$schemaId")) 186 | .willReturn( 187 | aResponse() 188 | .withBody(schemaBody) 189 | .withHeader("Content-Type", "application/vnd.schemaregistry.v1+json") 190 | ) 191 | ) 192 | } 193 | 194 | private def asAvroBinaryWithSchemaIdBytes[T: Encoder](t: T, 195 | schemaId: Int)(implicit schemaFor: SchemaFor[T]): Array[Byte] = 196 | asAvroWithSchemaIdBytes(t, schemaId, out => AvroOutputStream.binary[T].to(out).build(schemaFor.schema)) 197 | 198 | private def asAvroJsonWithSchemaIdBytes[T: SchemaFor: Encoder](t: T, schemaId: Int)( 199 | implicit schemaFor: SchemaFor[T] 200 | ): Array[Byte] = 201 | asAvroWithSchemaIdBytes(t, schemaId, out => AvroOutputStream.json[T].to(out).build(schemaFor.schema)) 202 | 203 | private def asAvroWithSchemaIdBytes[T](t: T, 204 | schemaId: Int, 205 | mkAvroOut: OutputStream => AvroOutputStream[T]): Array[Byte] = { 206 | val bout = new ByteArrayOutputStream() 207 | val avroOut = mkAvroOut(bout) 208 | avroOut.write(t) 209 | avroOut.flush() 210 | ByteBuffer.allocate(bout.size() + 4).putInt(schemaId).put(bout.toByteArray).array() 211 | } 212 | 213 | private def testWithPostSchemaExpected[T](subject: String)(f: => T) = { 214 | 215 | stubFor( 216 | post(urlMatching("/subjects/.*/versions")) 217 | .willReturn( 218 | aResponse() 219 | .withBody("{\"id\": 1}") 220 | .withHeader("Content-Type", "application/vnd.schemaregistry.v1+json") 221 | ) 222 | ) 223 | 224 | val result = f 225 | 226 | // TODO verify the schema is the same 227 | verify(postRequestedFor(urlEqualTo(s"/subjects/$subject/versions"))) 228 | 229 | result 230 | } 231 | } 232 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | import sbtrelease.ExtraReleaseCommands 2 | import sbtrelease.ReleaseStateTransformations._ 3 | import sbtrelease.tagsonly.TagsOnly._ 4 | 5 | lazy val catsVersion = "2.10.0" 6 | lazy val circeVersion = "0.14.6" 7 | lazy val logbackVersion = "1.4.11" 8 | lazy val avro4sVersion = "1.9.0" 9 | lazy val avro4s2Version = "2.0.4" 10 | lazy val json4sVersion = "3.6.12" 11 | lazy val slf4jVersion = "1.7.36" 12 | lazy val sprayJsonVersion = "1.3.6" 13 | lazy val kafkaClientVersion = "3.5.1" 14 | lazy val jsoninterScalaVersion = "1.2.0" 15 | lazy val confluentPlatformVersion = "5.3.8" 16 | lazy val scalaTestVersion = "3.0.9" 17 | lazy val scalaCheckVersion = "1.17.0" 18 | lazy val scalaMockVersion = "3.6.0" 19 | lazy val wiremockVersion = "2.35.1" 20 | lazy val scalaArmVersion = "2.0" 21 | 22 | lazy val publicArtifactory = "Artifactory Realm" at "https://kaluza.jfrog.io/artifactory/maven" 23 | 24 | lazy val publishSettings = Seq( 25 | publishTo := Some(publicArtifactory), 26 | credentials += { 27 | for { 28 | usr <- sys.env.get("ARTIFACTORY_USER") 29 | password <- sys.env.get("ARTIFACTORY_PASS") 30 | } yield Credentials("Artifactory Realm", "kaluza.jfrog.io", usr, password) 31 | }.getOrElse(Credentials(Path.userHome / ".ivy2" / ".credentials")), 32 | releaseProcess := Seq[ReleaseStep]( 33 | checkSnapshotDependencies, 34 | releaseStepCommand(ExtraReleaseCommands.initialVcsChecksCommand), 35 | setVersionFromTags(releaseTagPrefix.value), 36 | runClean, 37 | tagRelease, 38 | publishArtifacts, 39 | pushTagsOnly 40 | ) 41 | ) 42 | 43 | lazy val `kafka-serialization` = project 44 | .in(file(".")) 45 | .aggregate(avro, avro4s, avro4s2, cats, circe, core, json4s, `jsoniter-scala`, spray, testkit, doc) 46 | .settings( 47 | inThisBuild( 48 | List( 49 | organization := "com.ovoenergy", 50 | organizationName := "OVO Energy", 51 | organizationHomepage := Some(url("https://www.ovoenergy.com/")), 52 | homepage := Some(url("https://github.com/ovotech/kafka-serialization")), 53 | startYear := Some(2017), 54 | licenses := Seq(("Apache-2.0", url("http://www.apache.org/licenses/LICENSE-2.0"))), 55 | scmInfo := Some( 56 | ScmInfo( 57 | url("https://github.com/ovotech/kafka-serialization"), 58 | "git@github.com:ovotech/kafka-serialization.git" 59 | ) 60 | ), 61 | // TODO Find a way to extract those from github (sbt plugin) 62 | developers := List( 63 | Developer( 64 | "filippo.deluca", 65 | "Filippo De Luca", 66 | "filippo.deluca@ovoenergy.com", 67 | url("https://github.com/filosganga") 68 | ) 69 | ), 70 | scalaVersion := "2.12.18", 71 | resolvers ++= Seq( 72 | Resolver.mavenLocal, 73 | Resolver.typesafeRepo("releases"), 74 | "confluent-release" at "https://packages.confluent.io/maven/", 75 | "redhat-ga" at "https://maven.repository.redhat.com/ga/" 76 | ) 77 | ) 78 | ) 79 | ) 80 | .settings(name := "kafka-serialization", publishArtifact := false, publish := {}) 81 | .settings(publishSettings) 82 | 83 | lazy val doc = project 84 | .in(file("doc")) 85 | .enablePlugins(MdocPlugin) 86 | .dependsOn(avro, avro4s, cats, circe, core, json4s, `jsoniter-scala`, spray) 87 | .settings( 88 | name := "kafka-serialization-doc", 89 | publishArtifact := false, 90 | publish := {}, 91 | mdocIn := baseDirectory.value / "src", 92 | mdocOut := (baseDirectory.value).getParentFile, 93 | libraryDependencies ++= Seq( 94 | "io.circe" %% "circe-generic" % circeVersion, 95 | "org.apache.kafka" % "kafka-clients" % kafkaClientVersion exclude ("org.slf4j", "slf4j-log4j12"), 96 | "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-core" % jsoninterScalaVersion, 97 | "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-macros" % jsoninterScalaVersion % Provided 98 | ) 99 | ) 100 | .settings(publishSettings) 101 | 102 | lazy val testkit = project 103 | .in(file("testkit")) 104 | .settings( 105 | name := "kafka-serialization-testkit", 106 | libraryDependencies ++= Seq( 107 | "org.scalatest" %% "scalatest" % scalaTestVersion, 108 | "org.scalacheck" %% "scalacheck" % scalaCheckVersion, 109 | "org.scalamock" %% "scalamock-scalatest-support" % scalaMockVersion, 110 | "com.github.tomakehurst" % "wiremock-jre8" % wiremockVersion, 111 | "com.jsuereth" %% "scala-arm" % scalaArmVersion, 112 | "org.slf4j" % "log4j-over-slf4j" % slf4jVersion, 113 | "org.slf4j" % "jcl-over-slf4j" % slf4jVersion, 114 | "ch.qos.logback" % "logback-core" % logbackVersion, 115 | "ch.qos.logback" % "logback-classic" % logbackVersion 116 | ) 117 | ) 118 | .settings(publishSettings) 119 | 120 | lazy val json4s = project 121 | .in(file("json4s")) 122 | .dependsOn(core, testkit % Test) 123 | .settings( 124 | name := "kafka-serialization-json4s", 125 | libraryDependencies ++= Seq( 126 | "org.scala-lang" % "scala-reflect" % scalaVersion.value, 127 | "org.json4s" %% "json4s-core" % json4sVersion, 128 | "org.json4s" %% "json4s-native" % json4sVersion 129 | ) 130 | ) 131 | .settings(publishSettings) 132 | 133 | lazy val avro = project 134 | .in(file("avro")) 135 | .dependsOn(core, testkit % Test) 136 | .settings( 137 | name := "kafka-serialization-avro", 138 | libraryDependencies ++= Seq( 139 | "io.confluent" % "kafka-avro-serializer" % confluentPlatformVersion exclude ("org.slf4j", "slf4j-log4j12") 140 | ) 141 | ) 142 | .settings(publishSettings) 143 | 144 | lazy val avro4s = project 145 | .in(file("avro4s")) 146 | .dependsOn(core, avro, testkit % Test) 147 | .settings( 148 | name := "kafka-serialization-avro4s", 149 | libraryDependencies ++= Seq( 150 | "com.sksamuel.avro4s" %% "avro4s-macros" % avro4sVersion, 151 | "com.sksamuel.avro4s" %% "avro4s-core" % avro4sVersion, 152 | "com.sksamuel.avro4s" %% "avro4s-json" % avro4sVersion 153 | ) 154 | ) 155 | .settings(publishSettings) 156 | 157 | lazy val avro4s2 = project 158 | .in(file("avro4s2")) 159 | .dependsOn(core, avro, testkit % Test) 160 | .settings( 161 | name := "kafka-serialization-avro4s2", 162 | libraryDependencies ++= Seq( 163 | "com.sksamuel.avro4s" %% "avro4s-macros" % avro4s2Version, 164 | "com.sksamuel.avro4s" %% "avro4s-core" % avro4s2Version, 165 | "com.sksamuel.avro4s" %% "avro4s-json" % avro4s2Version 166 | ) 167 | ) 168 | .settings(publishSettings) 169 | 170 | lazy val `jsoniter-scala` = project 171 | .in(file("jsoniter-scala")) 172 | .dependsOn(core, testkit % Test) 173 | .settings( 174 | name := "kafka-serialization-jsoniter-scala", 175 | libraryDependencies ++= Seq( 176 | "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-core" % jsoninterScalaVersion, 177 | "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-macros" % jsoninterScalaVersion % Provided 178 | ) 179 | ) 180 | .settings(publishSettings) 181 | 182 | lazy val circe = project 183 | .in(file("circe")) 184 | .dependsOn(core, testkit % Test) 185 | .settings( 186 | name := "kafka-serialization-circe", 187 | libraryDependencies ++= Seq( 188 | "io.circe" %% "circe-core" % circeVersion, 189 | "io.circe" %% "circe-parser" % circeVersion, 190 | "io.circe" %% "circe-generic" % circeVersion % Test 191 | ) 192 | ) 193 | .settings(publishSettings) 194 | 195 | lazy val spray = project 196 | .in(file("spray")) 197 | .dependsOn(core, testkit % Test) 198 | .settings( 199 | name := "kafka-serialization-spray", 200 | libraryDependencies ++= Seq("io.spray" %% "spray-json" % sprayJsonVersion) 201 | ) 202 | .settings(publishSettings) 203 | 204 | lazy val core = project 205 | .in(file("core")) 206 | .dependsOn(testkit % Test) 207 | .settings( 208 | name := "kafka-serialization-core", 209 | libraryDependencies ++= Seq( 210 | "org.apache.kafka" % "kafka-clients" % kafkaClientVersion exclude ("org.slf4j", "slf4j-log4j12"), 211 | "org.slf4j" % "slf4j-api" % slf4jVersion 212 | ) 213 | ) 214 | .settings(publishSettings) 215 | 216 | lazy val cats = project 217 | .in(file("cats")) 218 | .dependsOn(core, testkit % Test) 219 | .settings( 220 | name := "kafka-serialization-cats", 221 | libraryDependencies ++= Seq("org.typelevel" %% "cats-core" % catsVersion) 222 | ) 223 | .settings(publishSettings) 224 | -------------------------------------------------------------------------------- /build/tag.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | echo 'Fetching tag from remote...' 6 | git tag -l | xargs git tag -d 7 | git fetch --tags 8 | 9 | # TODO Find a way to store the output 10 | if ! git describe --exact-match 2>/dev/null; then 11 | echo 'Not tag found...' 12 | 13 | last_tag=`git describe --abbrev=0 --tags` 14 | current_version=${last_tag#'v'} 15 | 16 | echo "Current version ${current_version}" 17 | 18 | #replace . with space so can split into an array 19 | current_version_parts=(${current_version//./ }) 20 | 21 | #get number parts and increase last one by 1 22 | current_version_major=${current_version_parts[0]} 23 | current_version_minor=${current_version_parts[1]} 24 | current_version_build=${current_version_parts[2]} 25 | 26 | next_version_build=$((current_version_build+1)) 27 | next_version="$current_version_major.$current_version_minor.$next_version_build" 28 | next_tag="v${next_version}" 29 | 30 | echo "Tagging the current commit with ${next_tag}" 31 | 32 | git tag -a ${next_tag} -m "Release version "${next_version} 33 | 34 | echo "Pushing tag ${next_tag} to origin" 35 | git push origin ${next_tag} 36 | 37 | else 38 | echo 'Tag found, no tag will be add' 39 | fi 40 | -------------------------------------------------------------------------------- /cats/src/main/scala/com/ovoenergy/kafka/serialization/cats/DeserializerInstances.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.cats 18 | 19 | import cats.Functor 20 | import org.apache.kafka.common.serialization.Deserializer 21 | import com.ovoenergy.kafka.serialization.core._ 22 | 23 | trait DeserializerInstances { 24 | 25 | implicit lazy val catsInstancesForKafkaDeserializer: Functor[Deserializer] = new Functor[Deserializer] { 26 | override def map[A, B](fa: Deserializer[A])(f: A => B): Deserializer[B] = deserializer[B] { 27 | (topic: String, data: Array[Byte]) => 28 | f(fa.deserialize(topic, data)) 29 | } 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /cats/src/main/scala/com/ovoenergy/kafka/serialization/cats/SerializerInstances.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.cats 18 | 19 | import cats.Contravariant 20 | import com.ovoenergy.kafka.serialization.core._ 21 | import org.apache.kafka.common.serialization.Serializer 22 | 23 | trait SerializerInstances { 24 | 25 | implicit lazy val catsInstancesForKafkaSerializer: Contravariant[Serializer] = new Contravariant[Serializer] { 26 | override def contramap[A, B](fa: Serializer[A])(f: B => A): Serializer[B] = serializer[B] { (topic: String, b: B) => 27 | fa.serialize(topic, f(b)) 28 | } 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /cats/src/main/scala/com/ovoenergy/kafka/serialization/cats/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization 18 | 19 | package object cats extends SerializerInstances with DeserializerInstances 20 | -------------------------------------------------------------------------------- /cats/src/test/scala/com/ovoenergy/kafka/serialization/cats/DeserializerInstancesSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.cats 18 | 19 | import com.ovoenergy.kafka.serialization.core._ 20 | import com.ovoenergy.kafka.serialization.testkit.UnitSpec 21 | import cats.syntax.functor._ 22 | 23 | class DeserializerInstancesSpec extends UnitSpec with DeserializerInstances { 24 | 25 | "Deserializer" should { 26 | "be a functor" in forAll { (x: Int, f: Int => Int) => 27 | constDeserializer(x).map(f).deserialize("DoesNotMatter", Array.empty[Byte]) shouldBe f(x) 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /cats/src/test/scala/com/ovoenergy/kafka/serialization/cats/SerializerInstancesSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.cats 18 | 19 | import cats.syntax.contravariant._ 20 | import com.ovoenergy.kafka.serialization.core._ 21 | import com.ovoenergy.kafka.serialization.testkit.UnitSpec 22 | 23 | class SerializerInstancesSpec extends UnitSpec with SerializerInstances { 24 | 25 | "Serializer" should { 26 | "be a Contravariant" in forAll { (bytes: Array[Byte], data: Int) => 27 | identitySerializer.contramap[Int](_ => bytes).serialize("DoesNotMatter", data).deep shouldBe bytes 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /circe/src/main/scala/com/ovoenergy/kafka/serialization/circe/CirceSerialization.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.circe 18 | 19 | import java.nio.charset.StandardCharsets 20 | 21 | import cats.syntax.either._ 22 | import com.ovoenergy.kafka.serialization.core._ 23 | import io.circe.parser._ 24 | import io.circe.syntax._ 25 | import io.circe.{Decoder, Encoder, Error, Json} 26 | import org.apache.kafka.common.serialization.{Deserializer => KafkaDeserializer, Serializer => KafkaSerializer} 27 | 28 | private[circe] trait CirceSerialization { 29 | 30 | def circeJsonSerializer[T: Encoder]: KafkaSerializer[T] = serializer { (_, data) => 31 | data.asJson.noSpaces.getBytes(StandardCharsets.UTF_8) 32 | } 33 | 34 | def circeJsonDeserializer[T: Decoder]: KafkaDeserializer[T] = deserializer { (_, data) => 35 | (for { 36 | json <- parse(new String(data, StandardCharsets.UTF_8)): Either[Error, Json] 37 | t <- json.as[T]: Either[Error, T] 38 | } yield 39 | t).fold(error => throw new RuntimeException(s"Deserialization failure: ${error.getMessage}", error), identity _) 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /circe/src/main/scala/com/ovoenergy/kafka/serialization/circe/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization 18 | 19 | package object circe extends CirceSerialization 20 | -------------------------------------------------------------------------------- /circe/src/test/scala/com/ovoenergy/kafka/serialization/circe/CirceSerializationSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.circe 18 | 19 | import java.nio.charset.StandardCharsets.UTF_8 20 | 21 | import com.ovoenergy.kafka.serialization.testkit.UnitSpec 22 | import com.ovoenergy.kafka.serialization.testkit.UnitSpec._ 23 | import io.circe.generic.auto._ 24 | import io.circe.parser._ 25 | import io.circe.syntax._ 26 | 27 | class CirceSerializationSpec extends UnitSpec with CirceSerialization { 28 | 29 | "CirceSerialization" when { 30 | "serializing" should { 31 | "write the Json body" in forAll { event: Event => 32 | val serializer = circeJsonSerializer[Event] 33 | val bytes = serializer.serialize("Does not matter", event) 34 | 35 | parse(new String(bytes, UTF_8)) shouldBe Right(event.asJson) 36 | } 37 | } 38 | 39 | "deserializing" should { 40 | "parse the json" in forAll { event: Event => 41 | val jsonBytes = event.asJson.noSpaces.getBytes(UTF_8) 42 | val deserializer = circeJsonDeserializer[Event] 43 | 44 | val deserialized = deserializer.deserialize("does not matter", jsonBytes) 45 | 46 | deserialized shouldBe event 47 | } 48 | } 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /core/src/main/scala/com/ovoenergy/kafka/serialization/core/Deserialization.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.core 18 | 19 | import java.util 20 | 21 | import com.ovoenergy.kafka.serialization.core.syntax._ 22 | import org.apache.kafka.common.serialization.{Deserializer => KafkaDeserializer} 23 | 24 | /** 25 | * Provides all the basic function to build a Kafka Deserializer. 26 | */ 27 | private[core] trait Deserialization { 28 | 29 | /** 30 | * Builds a Kafka deserializer from a function `(String, Array[Byte]) => T` and a close function. 31 | * 32 | * The close function is useful for those serializer that need to allocate some resources. 33 | */ 34 | def deserializer[T](f: (String, Array[Byte]) => T, c: () => Unit): KafkaDeserializer[T] = new KafkaDeserializer[T] { 35 | 36 | override def configure(configs: util.Map[String, _], isKey: Boolean): Unit = {} 37 | 38 | override def close(): Unit = c() 39 | 40 | override def deserialize(topic: String, data: Array[Byte]): T = f(topic, data) 41 | } 42 | 43 | /** 44 | * Builds a Kafka deserializer from a function `(String, Array[Byte]) => T`. 45 | */ 46 | def deserializer[T](f: (String, Array[Byte]) => T): KafkaDeserializer[T] = deserializer(f, () => Unit) 47 | 48 | /** 49 | * Builds a Kafka deserializer from a function `String => T`.ignoring the topic as it is usually not taken in 50 | * account. 51 | */ 52 | def deserializer[T](f: Array[Byte] => T): KafkaDeserializer[T] = deserializer { (_, bytes) => 53 | f(bytes) 54 | } 55 | 56 | /** 57 | * Wraps a Kafka deserializer by dropping the first byte of the payload (assuming it is the format byte). 58 | */ 59 | def formatDroppingDeserializer[T](d: KafkaDeserializer[T]): KafkaDeserializer[T] = 60 | deserializer({ (topic, data) => 61 | d.deserialize(topic, data.drop(1)) 62 | }) 63 | 64 | /** 65 | * Wraps a Kafka deserializer by checking if the first byte is the desired format byte. 66 | * 67 | * By default the format byte will be also dropped. This behaviour can be controlled by the `dropFormat` parmaeter. 68 | */ 69 | def formatCheckingDeserializer[T](expectedFormat: Format, 70 | d: KafkaDeserializer[T], 71 | dropFormat: Boolean = true): KafkaDeserializer[T] = 72 | deserializer({ (topic, data) => 73 | (if (data.isEmpty) { 74 | nullDeserializer[T] 75 | } else if (data(0) == Format.toByte(expectedFormat) && dropFormat) { 76 | formatDroppingDeserializer(d) 77 | } else if (data(0) == Format.toByte(expectedFormat) && !dropFormat) { 78 | d 79 | } else { 80 | failingDeserializer(new UnsupportedFormatException(data(0).toFormat)) 81 | }).deserialize(topic, data) 82 | }) 83 | 84 | /** 85 | * This allow to build a Kafka deserializer that will be able to deserialize payloads encoded in different way. 86 | * 87 | * If the payload format does not match any [[Format]] handled by the `Format => serializer` partial function the 88 | * given default Kafka deserializer will be used to deserialize the payload. 89 | * 90 | * The `dropFormat` parameter controls whether passing the whole payload to the following deserializer or dropping it. 91 | * By default the format byte is dropped. 92 | */ 93 | def formatDemultiplexerDeserializer[T](unknownFormat: Format => KafkaDeserializer[T], dropFormat: Boolean = true)( 94 | pf: PartialFunction[Format, KafkaDeserializer[T]] 95 | ): KafkaDeserializer[T] = 96 | deserializer({ (topic, data) => 97 | pf.andThen(d => if (dropFormat) formatDroppingDeserializer(d) else d) 98 | .applyOrElse(Format.fromByte(data(0)), unknownFormat) 99 | .deserialize(topic, data) 100 | }) 101 | 102 | /** 103 | * This allow to build a Kafka deserializer that will be able to apply different Kafka deserializer depending 104 | * on the source topic. 105 | * 106 | * If the source topic does not match any topic handled by the `String => deserializer` partial function the given 107 | * default Kafka deserializer will be used to deserialize the payload. 108 | */ 109 | def topicDemultiplexerDeserializer[T]( 110 | noMatchingTopic: String => KafkaDeserializer[T] 111 | )(pf: PartialFunction[String, KafkaDeserializer[T]]): KafkaDeserializer[T] = 112 | deserializer({ (topic, data) => 113 | pf.applyOrElse[String, KafkaDeserializer[T]](topic, noMatchingTopic).deserialize(topic, data) 114 | }) 115 | 116 | /** 117 | * Wraps a Kafka deserializer to make it non-strict. The deserialization logic will be applied lazily. 118 | */ 119 | def nonStrictDeserializer[T](d: KafkaDeserializer[T]): KafkaDeserializer[() => T] = 120 | deserializer({ (topic, data) => () => 121 | d.deserialize(topic, data) 122 | }) 123 | 124 | /** 125 | * Wraps a Kafka deserializer to return None if the data is null. 126 | */ 127 | def optionalDeserializer[T](d: KafkaDeserializer[T]): KafkaDeserializer[Option[T]] = 128 | deserializer((topic, data) => Option(data).map(_ => d.deserialize(topic, data))) 129 | 130 | /** 131 | * Builds a Kafka deserializer that will return the payload as-is. 132 | */ 133 | def identityDeserializer: KafkaDeserializer[Array[Byte]] = identity[Array[Byte]] _ 134 | 135 | /** 136 | * Builds a Kafka deserializer that will return always the same value. 137 | */ 138 | def constDeserializer[T](t: T): KafkaDeserializer[T] = deserializer(_ => t) 139 | 140 | /** 141 | * Builds a Kafka deserializer that will return always null. 142 | */ 143 | def nullDeserializer[T]: KafkaDeserializer[T] = constDeserializer(null.asInstanceOf[T]) 144 | 145 | /** 146 | * Builds a Kafka deserializer that will always fail with the given exception. 147 | */ 148 | def failingDeserializer[T](e: Throwable): KafkaDeserializer[T] = deserializer(_ => throw e) 149 | 150 | } 151 | -------------------------------------------------------------------------------- /core/src/main/scala/com/ovoenergy/kafka/serialization/core/Format.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.core 18 | 19 | /** 20 | * Represent a payload format. 21 | * 22 | * It will be serialized as a single byte as payload prefix. Different formats have different byte value. 23 | */ 24 | sealed trait Format 25 | 26 | object Format { 27 | 28 | /** 29 | * This is an Avro binary message prefixed by 4 bytes containing a schema id obtained fro mthe schema registry. 30 | */ 31 | case object AvroBinarySchemaId extends Format 32 | 33 | /** 34 | * This is an Avro JSON message prefixed by 4 bytes containing a schema id obtained fro mthe schema registry. 35 | */ 36 | case object AvroJsonSchemaId extends Format 37 | 38 | /** 39 | * This is JSON message. 40 | */ 41 | case object Json extends Format 42 | 43 | /** 44 | * This is custom format message where the byte value is given in the constructor. 45 | */ 46 | case class Custom(b: Byte) extends Format 47 | 48 | def toByte(f: Format): Byte = f match { 49 | case AvroBinarySchemaId => 0 50 | case AvroJsonSchemaId => 1 51 | case Json => 2 52 | case Custom(b) => b 53 | } 54 | 55 | def fromByte(b: Byte): Format = b match { 56 | case 0 => AvroBinarySchemaId 57 | case 1 => AvroJsonSchemaId 58 | case 2 => Json 59 | case n => Custom(n) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /core/src/main/scala/com/ovoenergy/kafka/serialization/core/Implicits.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.core 18 | 19 | import org.apache.kafka.common.serialization.{Deserializer => KafkaDeserializer, Serializer => KafkaSerializer} 20 | import scala.language.implicitConversions 21 | 22 | /** 23 | * Provides implicit conversion between functions and Kafka serializer/deserializer. 24 | */ 25 | trait Implicits { 26 | 27 | implicit def function2Serializer[T](f: (String, T) => Array[Byte]): KafkaSerializer[T] = serializer(f) 28 | 29 | implicit def function2Serializer[T](f: T => Array[Byte]): KafkaSerializer[T] = serializer(f) 30 | 31 | implicit def function2Deserializer[T](f: (String, Array[Byte]) => T): KafkaDeserializer[T] = deserializer(f) 32 | 33 | implicit def function2Deserializer[T](f: Array[Byte] => T): KafkaDeserializer[T] = deserializer(f) 34 | } 35 | -------------------------------------------------------------------------------- /core/src/main/scala/com/ovoenergy/kafka/serialization/core/Serialization.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.core 18 | 19 | import com.ovoenergy.kafka.serialization.core.syntax._ 20 | import java.util 21 | 22 | import org.apache.kafka.common.serialization.{Serializer => KafkaSerializer} 23 | 24 | /** 25 | * Provides all the basic function to build a Kafka Serializer. 26 | */ 27 | private[core] trait Serialization { 28 | 29 | /** 30 | * Builds a Kafka serializer from a function `(String, T) => Array[Byte]` and close function. 31 | * 32 | * The close function is useful for those serializer that need to allocate some resources. 33 | */ 34 | def serializer[T](f: (String, T) => Array[Byte], c: () => Unit): KafkaSerializer[T] = new KafkaSerializer[T] { 35 | 36 | // Keep in mind that for serializer built with this library Kafka will never call `configure`. 37 | override def configure(configs: util.Map[String, _], isKey: Boolean): Unit = {} 38 | 39 | override def close(): Unit = c() 40 | 41 | override def serialize(topic: String, data: T): Array[Byte] = f(topic, data) 42 | } 43 | 44 | /** 45 | * Builds a Kafka serializer from a function `(String, T) => Array[Byte]` 46 | */ 47 | def serializer[T](f: (String, T) => Array[Byte]): KafkaSerializer[T] = serializer(f, () => Unit) 48 | 49 | /** 50 | * Builds a Kafka serializer from a function `String => Array[Byte]` the topic is ignored as most of the time 51 | * it is not taken in account. 52 | */ 53 | def serializer[T](f: T => Array[Byte]): KafkaSerializer[T] = serializer { (_, t) => 54 | f(t) 55 | } 56 | 57 | /** 58 | * Wraps a Kafka serializer prepending the format byte to the original payload. 59 | */ 60 | def formatSerializer[T](format: Format, delegate: KafkaSerializer[T]): KafkaSerializer[T] = 61 | serializer({ (topic, data) => 62 | Array(format.toByte) ++ delegate.serialize(topic, data) 63 | }) 64 | 65 | /** 66 | * Provides a way to use different serializer depending of the target topic. 67 | */ 68 | def topicMultiplexerSerializer[T]( 69 | noMatchingTopic: String => KafkaSerializer[T] 70 | )(pf: PartialFunction[String, KafkaSerializer[T]]): KafkaSerializer[T] = 71 | serializer({ (topic, data) => 72 | pf.applyOrElse[String, KafkaSerializer[T]](topic, noMatchingTopic).serialize(topic, data) 73 | }) 74 | 75 | /** 76 | * Builds a serializer that will serialize the message as-is. 77 | */ 78 | def identitySerializer: KafkaSerializer[Array[Byte]] = identity[Array[Byte]] _ 79 | 80 | /** 81 | * Builds a serializer that always serialize the same payload. 82 | */ 83 | def constSerializer[T](bytes: Array[Byte]): KafkaSerializer[T] = serializer(_ => bytes) 84 | 85 | /** 86 | * Builds a serializer that serialize null if the given . 87 | */ 88 | def optionalSerializer[T](ts: KafkaSerializer[T]): KafkaSerializer[Option[T]] = serializer { (topic, ot) => 89 | ot match { 90 | case Some(t) => ts.serialize(topic, t) 91 | case None => null.asInstanceOf[Array[Byte]] 92 | } 93 | } 94 | 95 | /** 96 | * Builds a serializer that always serialize null. 97 | */ 98 | def nullSerializer[T]: KafkaSerializer[T] = serializer(_ => null.asInstanceOf[Array[Byte]]) 99 | 100 | /** 101 | * Builds a serializer that always fails with the given exception. 102 | */ 103 | def failingSerializer[T](e: Throwable): KafkaSerializer[T] = serializer(_ => throw e) 104 | 105 | } 106 | -------------------------------------------------------------------------------- /core/src/main/scala/com/ovoenergy/kafka/serialization/core/UnsupportedFormatException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.core 18 | 19 | /** 20 | * This exception is raised when the payload format does not match the expected one. 21 | */ 22 | class UnsupportedFormatException(format: Format) extends RuntimeException(s"Unsupported format: $format") 23 | -------------------------------------------------------------------------------- /core/src/main/scala/com/ovoenergy/kafka/serialization/core/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization 18 | 19 | /** 20 | * Aggregates basic blocks to build kafka serializers and deserializers from the extended traits. 21 | */ 22 | package object core extends Serialization with Deserialization 23 | -------------------------------------------------------------------------------- /core/src/main/scala/com/ovoenergy/kafka/serialization/core/syntax/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.core 18 | 19 | import org.apache.kafka.common.serialization.{Deserializer => KafkaDeserializer, Serializer => KafkaSerializer} 20 | 21 | package object syntax extends Implicits { 22 | 23 | implicit class RichFormat(val f: Format) extends AnyVal { 24 | def toByte: Byte = Format.toByte(f) 25 | } 26 | 27 | implicit class RichByte(val b: Byte) extends AnyVal { 28 | def toFormat: Format = Format.fromByte(b) 29 | } 30 | 31 | implicit class RichBytesToTFunction[T](f: Array[Byte] => T) { 32 | def asDeserializer: KafkaDeserializer[T] = deserializer(f) 33 | } 34 | 35 | implicit class RichStringAndBytesToTFunction[T](f: (String, Array[Byte]) => T) { 36 | def asDeserializer: KafkaDeserializer[T] = deserializer(f) 37 | } 38 | 39 | implicit class RichTToBytesFunction[T](f: T => Array[Byte]) { 40 | def asSerializer: KafkaSerializer[T] = serializer(f) 41 | } 42 | 43 | implicit class RichStringAndTToBytesFunction[T](f: (String, T) => Array[Byte]) { 44 | def asSerializer: KafkaSerializer[T] = serializer(f) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /core/src/test/scala/com/ovoenergy/kafka/serialization/core/DeserializationSpec.scala: -------------------------------------------------------------------------------- 1 | package com.ovoenergy.kafka.serialization.core 2 | 3 | import java.nio.ByteBuffer 4 | import java.nio.charset.StandardCharsets.UTF_8 5 | 6 | import com.ovoenergy.kafka.serialization.core.syntax._ 7 | import com.ovoenergy.kafka.serialization.testkit.UnitSpec 8 | import org.apache.kafka.common.serialization.{Deserializer, Serializer} 9 | 10 | import scala.util.Random 11 | 12 | import DeserializationSpec._ 13 | 14 | object DeserializationSpec { 15 | 16 | val IgnoredTopic = "Does not matter" 17 | 18 | val StringTopic = "string-topic" 19 | 20 | val IntTopic = "int-topic" 21 | 22 | val StringSerializer: Serializer[String] = (s: String) => s.getBytes(UTF_8) 23 | val StringDeserializer: Deserializer[String] = (data: Array[Byte]) => new String(data, UTF_8) 24 | 25 | val IntSerializer: Serializer[Int] = { (i: Int) => 26 | ByteBuffer.allocate(4).putInt(i).array() 27 | } 28 | 29 | val IntDeserializer: Deserializer[Int] = { (data: Array[Byte]) => 30 | ByteBuffer.wrap(data).getInt 31 | } 32 | } 33 | 34 | class DeserializationSpec extends UnitSpec { 35 | 36 | "Deserialization" when { 37 | 38 | "demultiplexing the format" when { 39 | 40 | "the format does not match" should { 41 | "use the default deserializer" in { 42 | 43 | val expectedFormat = Format.Custom(19) 44 | val expectedString = "TestString" 45 | 46 | val d: Deserializer[String] = formatDemultiplexerDeserializer(_ => constDeserializer(expectedString)) { 47 | case Format.Custom(9) => StringDeserializer 48 | case Format.Custom(8) => constDeserializer("Bad String") 49 | } 50 | 51 | val deserialized = d.deserialize( 52 | "test-topic", 53 | formatSerializer(expectedFormat, StringSerializer).serialize("", expectedString) 54 | ) 55 | 56 | deserialized shouldBe expectedString 57 | } 58 | } 59 | 60 | "the format does not match" should { 61 | "not drop unknown format byte" in { 62 | 63 | val expectedString = "TestString" 64 | val topic = "test-topic" 65 | 66 | val d: Deserializer[String] = formatDemultiplexerDeserializer(_ => StringDeserializer) { 67 | case Format.Json => throw new RuntimeException("must not match") 68 | } 69 | 70 | val deserialized = d.deserialize(topic, StringSerializer.serialize(topic, expectedString)) 71 | deserialized shouldBe expectedString 72 | } 73 | } 74 | 75 | "dropping format is default" should { 76 | "drop the magic byte correctly" in { 77 | 78 | val expectedFormat = Format.Custom(9) 79 | val expectedString = "TestString" 80 | 81 | val d: Deserializer[String] = formatDemultiplexerDeserializer( 82 | _ => failingDeserializer[String](new RuntimeException("Wrong or unsupported serialization format byte")) 83 | ) { 84 | case `expectedFormat` => StringDeserializer 85 | case Format.Custom(8) => constDeserializer("Bad String") 86 | } 87 | 88 | val deserialized = d.deserialize( 89 | "test-topic", 90 | formatSerializer(expectedFormat, StringSerializer).serialize("", expectedString) 91 | ) 92 | 93 | deserialized shouldBe expectedString 94 | } 95 | } 96 | 97 | "dropping format is false" should { 98 | "not drop the magic byte" in { 99 | 100 | val expectedFormat = Format.Custom(9) 101 | val expectedString = "TestString" 102 | 103 | val d: Deserializer[String] = formatDemultiplexerDeserializer( 104 | _ => failingDeserializer[String](new RuntimeException("Wrong or unsupported serialization format byte")), 105 | dropFormat = false 106 | ) { 107 | case `expectedFormat` => formatCheckingDeserializer(expectedFormat, StringDeserializer) 108 | case Format.Custom(8) => constDeserializer("Bad String") 109 | } 110 | 111 | val deserialized = d.deserialize( 112 | "test-topic", 113 | formatSerializer(expectedFormat, StringSerializer).serialize("", expectedString) 114 | ) 115 | 116 | deserialized shouldBe expectedString 117 | } 118 | } 119 | } 120 | 121 | "dropping the magic byte" should { 122 | "drop the magic byte" in { 123 | val expectedBytes = "test string".getBytes(UTF_8) 124 | val deserializer = formatDroppingDeserializer { data: Array[Byte] => 125 | data 126 | } 127 | 128 | deserializer.deserialize("test-topic", Array(12: Byte) ++ expectedBytes).deep shouldBe expectedBytes.deep 129 | } 130 | } 131 | 132 | "checking the magic byte" when { 133 | "the format matches" should { 134 | "deserialize successfully" in { 135 | 136 | val expectedValue = "Foo" 137 | val expectedFormat = Format.Custom(9) 138 | 139 | val deserializer = formatCheckingDeserializer(Format.Custom(9), constDeserializer(expectedValue)) 140 | 141 | val deserialized = deserializer.deserialize( 142 | IgnoredTopic, 143 | Array(expectedFormat.toByte) ++ Array.fill(5)(Random.nextInt().toByte) 144 | ) 145 | 146 | deserialized shouldBe expectedValue 147 | } 148 | } 149 | 150 | "the format does not match" should { 151 | "fail to deserialize" in { 152 | 153 | val unexpectedFormat = Format.Custom(19) 154 | 155 | val deserializer = formatCheckingDeserializer(Format.Custom(9), constDeserializer("Foo")) 156 | 157 | a[UnsupportedFormatException] should be thrownBy deserializer.deserialize( 158 | IgnoredTopic, 159 | Array(unexpectedFormat.toByte) ++ Array.fill(5)(Random.nextInt().toByte) 160 | ) 161 | 162 | } 163 | } 164 | 165 | "dropFormat is default" should { 166 | "drop the format byte" in { 167 | 168 | val expectedFormat = Format.Custom(9) 169 | val expectedValue: Array[Byte] = Array.fill(5)(Random.nextInt().toByte) 170 | 171 | val deserializer = formatCheckingDeserializer(Format.Custom(9), identityDeserializer) 172 | 173 | val deserialized = deserializer.deserialize(IgnoredTopic, Array(expectedFormat.toByte) ++ expectedValue) 174 | 175 | deserialized.deep shouldBe expectedValue.deep 176 | 177 | } 178 | } 179 | 180 | "dropFormat is false" should { 181 | "not dropping the format byte" in { 182 | 183 | val expectedFormat = Format.Custom(9) 184 | val expectedValue: Array[Byte] = Array(expectedFormat.toByte) ++ Array.fill(5)(Random.nextInt().toByte) 185 | 186 | val deserializer = formatCheckingDeserializer(Format.Custom(9), identityDeserializer, dropFormat = false) 187 | 188 | val deserialized = deserializer.deserialize(IgnoredTopic, expectedValue) 189 | 190 | deserialized.deep shouldBe expectedValue.deep 191 | 192 | } 193 | } 194 | } 195 | 196 | "demultiplexing the topic" when { 197 | "the topic matches a branch" should { 198 | "use the matched deserializer" in { 199 | 200 | val expectedInt = 34 201 | 202 | // This code is nasty, but in production no one is going to have a consumer with two unrelated types. 203 | val deserializer = 204 | topicDemultiplexerDeserializer[Any](topic => failingDeserializer(new IllegalArgumentException(topic))) { 205 | case StringTopic => StringDeserializer.asInstanceOf[Deserializer[Any]] 206 | case IntTopic => IntDeserializer.asInstanceOf[Deserializer[Any]] 207 | } 208 | 209 | val deserialized = deserializer.deserialize(IntTopic, IntSerializer.serialize(IgnoredTopic, expectedInt)) 210 | 211 | deserialized shouldBe expectedInt 212 | } 213 | } 214 | 215 | "the topic does not match any branch" should { 216 | "use the non matching deserializer" in { 217 | 218 | val nonMatchingTopic = "test-topic" 219 | 220 | val expectedValue = "test-value" 221 | 222 | // This code is nasty, but in production no one is going to have a consumer with two unrelated types. 223 | val deserializer = topicDemultiplexerDeserializer[Any](_ => constDeserializer(expectedValue)) { 224 | case StringTopic => StringDeserializer.asInstanceOf[Deserializer[Any]] 225 | case IntTopic => IntDeserializer.asInstanceOf[Deserializer[Any]] 226 | } 227 | 228 | val deserialized = deserializer.deserialize(nonMatchingTopic, IntSerializer.serialize(IgnoredTopic, 45)) 229 | 230 | deserialized shouldBe expectedValue 231 | } 232 | } 233 | 234 | } 235 | 236 | "checking for null data" when { 237 | "data is not null" should { 238 | "return Some[T]" in forAll() { string: String => 239 | val result = optionalDeserializer[String](StringDeserializer) 240 | .deserialize(IgnoredTopic, StringSerializer.serialize(IgnoredTopic, string)) 241 | 242 | result shouldBe Some(string) 243 | } 244 | } 245 | 246 | "data is null" should { 247 | "return None" in { 248 | 249 | val result = optionalDeserializer[String](StringDeserializer) 250 | .deserialize(IgnoredTopic, null) 251 | 252 | result shouldBe None 253 | } 254 | } 255 | } 256 | 257 | } 258 | 259 | } 260 | -------------------------------------------------------------------------------- /core/src/test/scala/com/ovoenergy/kafka/serialization/core/SerializationSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.core 18 | 19 | import java.nio.ByteBuffer 20 | import java.nio.charset.StandardCharsets.UTF_8 21 | 22 | import com.ovoenergy.kafka.serialization.core.syntax._ 23 | import com.ovoenergy.kafka.serialization.testkit.UnitSpec 24 | import org.apache.kafka.common.serialization.{Deserializer, Serializer} 25 | 26 | import scala.util.Random 27 | 28 | import SerializationSpec._ 29 | 30 | object SerializationSpec { 31 | 32 | val IgnoredTopic = "Does not matter" 33 | 34 | val StringTopic = "string-topic" 35 | 36 | val IntTopic = "int-topic" 37 | 38 | val StringSerializer: Serializer[String] = (s: String) => s.getBytes(UTF_8) 39 | val StringDeserializer: Deserializer[String] = (data: Array[Byte]) => new String(data, UTF_8) 40 | 41 | val IntSerializer: Serializer[Int] = { (i: Int) => 42 | ByteBuffer.allocate(4).putInt(i).array() 43 | } 44 | 45 | val IntDeserializer: Deserializer[Int] = { (data: Array[Byte]) => 46 | ByteBuffer.wrap(data).getInt 47 | } 48 | } 49 | 50 | class SerializationSpec extends UnitSpec { 51 | 52 | "Serialization" when { 53 | 54 | "add the magic byte to the serialized data" in { 55 | formatSerializer(Format.Json, StringSerializer).serialize("test", "Test")(0) should be(Format.toByte(Format.Json)) 56 | } 57 | 58 | "multiplexing the topic" when { 59 | "the topic matches a branch" should { 60 | 61 | "use the matched serializer" in { 62 | 63 | // This code is nasty, but in production no one is going to have a consumer with two unrelated types. 64 | val serializer = topicMultiplexerSerializer[Any](_ => constSerializer("foo".getBytes(UTF_8))) { 65 | case StringTopic => StringSerializer.asInstanceOf[Serializer[Any]] 66 | case IntTopic => IntSerializer.asInstanceOf[Serializer[Any]] 67 | } 68 | 69 | val serialized = serializer.serialize(IntTopic, 56) 70 | 71 | serialized.deep shouldBe IntSerializer.serialize("Does not Matter", 56).deep 72 | } 73 | } 74 | 75 | "the topic does not match any branch" should { 76 | 77 | "use the non matched serializer" in { 78 | 79 | val expectedValue = "foo".getBytes(UTF_8) 80 | 81 | // This code is nasty, but in production no one is going to have a consumer with two unrelated types. 82 | val serializer = topicMultiplexerSerializer[Any](_ => constSerializer(expectedValue)) { 83 | case StringTopic => StringSerializer.asInstanceOf[Serializer[Any]] 84 | case IntTopic => IntSerializer.asInstanceOf[Serializer[Any]] 85 | } 86 | 87 | val serialized = serializer.serialize(IgnoredTopic, 56) 88 | 89 | serialized.deep shouldBe expectedValue.deep 90 | } 91 | } 92 | } 93 | 94 | "serialize Option[T]" when { 95 | "the given option is Some" should { 96 | "serialize the T" in forAll { s: String => 97 | optionalSerializer(StringSerializer).serialize("test", Some(s)) shouldBe StringSerializer.serialize("test", s) 98 | 99 | } 100 | } 101 | 102 | "the given option is None" should { 103 | "serialize null" in { 104 | Option(optionalSerializer(StringSerializer).serialize("test", Option.empty[String])) shouldBe None 105 | } 106 | } 107 | } 108 | } 109 | 110 | } 111 | -------------------------------------------------------------------------------- /doc/src/README.md: -------------------------------------------------------------------------------- 1 | # Kafka serialization/deserialization building blocks 2 | 3 | [![CircleCI Badge](https://circleci.com/gh/ovotech/kafka-serialization.svg?style=shield)](https://circleci.com/gh/ovotech/kafka-serialization) 4 | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/a2d814f22d4e4facae0f8a3eb1c841fd)](https://www.codacy.com/app/filippo-deluca/kafka-serialization?utm_source=github.com&utm_medium=referral&utm_content=ovotech/kafka-serialization&utm_campaign=Badge_Grade) 5 | [Download](https://kaluza.jfrog.io/artifactory/maven/com/ovoenergy/kafka-serialization-core_2.12/[RELEASE]/kafka-serialization-core_2.12-[RELEASE].jar) 6 | 7 | The aim of this library is to provide the Lego™ bricks to build a serializer/deserializer for kafka messages. 8 | 9 | The serializers/deserializers built by this library cannot be used in the Kafka configuration through properties, but 10 | need to be passed through the Kafka Producer/Consumer constructors (It is feature IMHO). 11 | 12 | For the Avro serialization this library uses Avro4s while for JSON it supports Json4s, Circe and Spray out of the box. 13 | It is quite easy to add support for other libraries as well. 14 | 15 | ## Modules 16 | 17 | The library is composed by these modules: 18 | 19 | - kafka-serialization-core: provides the serialization primitives to build serializers and deserializers. 20 | - kafka-serialization-cats: provides cats typeclasses instances for serializers and deserializers. 21 | - kafka-serialization-json4s: provides serializer and deserializer based on Json4s 22 | - kafka-serialization-jsoniter-scala: provides serializer and deserializer based on Jsoniter Scala 23 | - kafka-serialization-spray: provides serializer and deserializer based on Spray Json 24 | - kafka-serialization-circe: provides serializer and deserializer based on Circe 25 | - kafka-serialization-avro: provides an schema-registry client settings 26 | - kafka-serialization-avro4s: provides serializer and deserializer based on Avro4s 1.x 27 | - kafka-serialization-avro4s2: provides serializer and deserializer based on Avro4s 2.x 28 | 29 | The Avro4s serialization support the schema evolution through the schema registry. The consumer can provide its own schema 30 | and Avro will take care of the conversion. 31 | 32 | ## Getting Started 33 | 34 | - The library is available in the Kaluza artifactory repository. 35 | - See [here](https://kaluza.jfrog.io/artifactory/maven/com/ovoenergy/kafka-serialization-core_2.12/) for the latest version. 36 | - Add this snippet to your build.sbt to use it: 37 | 38 | ```sbtshell 39 | import sbt._ 40 | import sbt.Keys. 41 | 42 | resolvers += "Artifactory" at "https://kaluza.jfrog.io/artifactory/maven" 43 | 44 | libraryDependencies ++= { 45 | val kafkaSerializationV = "0.5.25" 46 | Seq( 47 | "com.ovoenergy" %% "kafka-serialization-core" % kafkaSerializationV, 48 | "com.ovoenergy" %% "kafka-serialization-circe" % kafkaSerializationV, // To provide Circe JSON support 49 | "com.ovoenergy" %% "kafka-serialization-json4s" % kafkaSerializationV, // To provide Json4s JSON support 50 | "com.ovoenergy" %% "kafka-serialization-jsoniter-scala" % kafkaSerializationV, // To provide Jsoniter Scala JSON support 51 | "com.ovoenergy" %% "kafka-serialization-spray" % kafkaSerializationV, // To provide Spray-json JSON support 52 | "com.ovoenergy" %% "kafka-serialization-avro4s" % kafkaSerializationV // To provide Avro4s Avro support 53 | ) 54 | } 55 | 56 | ``` 57 | 58 | ## Circe example 59 | 60 | Circe is a JSON library for Scala that provides support for generic programming trough Shapeless. You can find more 61 | information on the [Circe website](https://circe.github.io/circe). 62 | 63 | Simple serialization/deserialization example with Circe: 64 | 65 | ```scala mdoc:silent 66 | import com.ovoenergy.kafka.serialization.core._ 67 | import com.ovoenergy.kafka.serialization.circe._ 68 | 69 | // Import the Circe generic support 70 | import io.circe.generic.auto._ 71 | import io.circe.syntax._ 72 | 73 | import org.apache.kafka.clients.producer.KafkaProducer 74 | import org.apache.kafka.clients.consumer.KafkaConsumer 75 | import org.apache.kafka.clients.CommonClientConfigs._ 76 | 77 | import scala.collection.JavaConverters._ 78 | 79 | case class UserCreated(id: String, name: String, age: Int) 80 | 81 | val producer = new KafkaProducer( 82 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 83 | nullSerializer[Unit], 84 | circeJsonSerializer[UserCreated] 85 | ) 86 | 87 | val consumer = new KafkaConsumer( 88 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 89 | nullDeserializer[Unit], 90 | circeJsonDeserializer[UserCreated] 91 | ) 92 | ``` 93 | 94 | ```scala mdoc:invisible 95 | producer.close() 96 | consumer.close() 97 | ``` 98 | 99 | ## Jsoniter Scala example 100 | 101 | [Jsoniter Scala](https://github.com/plokhotnyuk/jsoniter-scala). is a library that generates codecs for case classes, 102 | standard types and collections to get maximum performance of JSON parsing & serialization. 103 | 104 | Here is an example of serialization/deserialization with Jsoniter Scala: 105 | 106 | ```scala mdoc:silent:reset 107 | import com.ovoenergy.kafka.serialization.core._ 108 | import com.ovoenergy.kafka.serialization.jsoniter_scala._ 109 | 110 | // Import the Jsoniter Scala macros & core support 111 | import com.github.plokhotnyuk.jsoniter_scala.macros._ 112 | import com.github.plokhotnyuk.jsoniter_scala.core._ 113 | 114 | import org.apache.kafka.clients.producer.KafkaProducer 115 | import org.apache.kafka.clients.consumer.KafkaConsumer 116 | import org.apache.kafka.clients.CommonClientConfigs._ 117 | 118 | import scala.collection.JavaConverters._ 119 | 120 | case class UserCreated(id: String, name: String, age: Int) 121 | 122 | implicit val userCreatedCodec: JsonValueCodec[UserCreated] = JsonCodecMaker.make[UserCreated](CodecMakerConfig) 123 | 124 | val producer = new KafkaProducer( 125 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 126 | nullSerializer[Unit], 127 | jsoniterScalaSerializer[UserCreated]() 128 | ) 129 | 130 | val consumer = new KafkaConsumer( 131 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 132 | nullDeserializer[Unit], 133 | jsoniterScalaDeserializer[UserCreated]() 134 | ) 135 | ``` 136 | 137 | ```scala mdoc:invisible 138 | producer.close() 139 | consumer.close() 140 | ``` 141 | 142 | ## Avro example 143 | 144 | Apache Avro is a remote procedure call and data serialization framework developed within Apache's Hadoop project. It uses 145 | JSON for defining data types and protocols, and serializes data in a compact binary format. 146 | 147 | Apache Avro provide some support to evolve your messages across multiple version without breaking compatibility with 148 | older or newer consumers. It supports several encoding formats but two are the most used in Kafka: Binary and Json. 149 | 150 | The encoded data is always validated and parsed using a Schema (defined in JSON) and eventually evolved to the reader 151 | Schema version. 152 | 153 | This library provided the support to Avro by using the [Avro4s](https://github.com/sksamuel/avro4s) libray. It uses macro 154 | and shapeless to allowing effortless serialization and deserialization. In addition to Avro4s it need a Confluent schema 155 | registry in place, It will provide a way to control the format of the messages produced in kafka. You can find more 156 | information in the [Confluent Schema Registry Documentation ](http://docs.confluent.io/current/schema-registry/docs/). 157 | 158 | 159 | An example with Avro4s binary and Schema Registry: 160 | ```scala mdoc:silent:reset 161 | import com.ovoenergy.kafka.serialization.core._ 162 | import com.ovoenergy.kafka.serialization.avro4s._ 163 | 164 | import com.sksamuel.avro4s._ 165 | 166 | import org.apache.kafka.clients.producer.KafkaProducer 167 | import org.apache.kafka.clients.consumer.KafkaConsumer 168 | import org.apache.kafka.clients.CommonClientConfigs._ 169 | 170 | import scala.collection.JavaConverters._ 171 | 172 | val schemaRegistryEndpoint = "http://localhost:8081" 173 | 174 | case class UserCreated(id: String, name: String, age: Int) 175 | 176 | // This type class is need by the avroBinarySchemaIdSerializer 177 | implicit val UserCreatedToRecord = ToRecord[UserCreated] 178 | 179 | val producer = new KafkaProducer( 180 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 181 | nullSerializer[Unit], 182 | avroBinarySchemaIdSerializer[UserCreated](schemaRegistryEndpoint, isKey = false, includesFormatByte = true) 183 | ) 184 | 185 | // This type class is need by the avroBinarySchemaIdDeserializer 186 | implicit val UserCreatedFromRecord = FromRecord[UserCreated] 187 | 188 | val consumer = new KafkaConsumer( 189 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 190 | nullDeserializer[Unit], 191 | avroBinarySchemaIdDeserializer[UserCreated](schemaRegistryEndpoint, isKey = false, includesFormatByte = true) 192 | ) 193 | ``` 194 | 195 | ```scala mdoc:invisible 196 | producer.close() 197 | consumer.close() 198 | ``` 199 | 200 | This Avro serializer will try to register the schema every new message type it will serialize and will save the obtained 201 | schema id in cache. The deserializer will contact the schema registry each time it will encounter a message with a never 202 | seen before schema id. 203 | 204 | The schema id will encoded in the first 4 bytes of the payload. The deserializer will extract the schema id from the 205 | payload and fetch the schema from the schema registry. The deserializer is able to evolve the original message to the 206 | consumer schema. The use case is when the consumer is only interested in a part of the original message (schema projection) 207 | or when the original message is in a older or newer format of the cosumer schema (schema evolution). 208 | 209 | An example of the consumer schema: 210 | ```scala mdoc:silent:reset 211 | import com.ovoenergy.kafka.serialization.core._ 212 | import com.ovoenergy.kafka.serialization.avro4s._ 213 | 214 | import com.sksamuel.avro4s._ 215 | 216 | import org.apache.kafka.clients.producer.KafkaProducer 217 | import org.apache.kafka.clients.consumer.KafkaConsumer 218 | import org.apache.kafka.clients.CommonClientConfigs._ 219 | 220 | import scala.collection.JavaConverters._ 221 | 222 | val schemaRegistryEndpoint = "http://localhost:8081" 223 | 224 | /* Assuming the original message has been serialized using the 225 | * previously defined UserCreated class. We are going to project 226 | * it ignoring the value of the age 227 | */ 228 | case class UserCreated(id: String, name: String) 229 | 230 | // This type class is need by the avroBinarySchemaIdDeserializer 231 | implicit val UserCreatedFromRecord = FromRecord[UserCreated] 232 | 233 | 234 | /* This type class is need by the avroBinarySchemaIdDeserializer 235 | * to obtain the consumer schema 236 | */ 237 | implicit val UserCreatedSchemaFor = SchemaFor[UserCreated] 238 | 239 | val consumer = new KafkaConsumer( 240 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 241 | nullDeserializer[Unit], 242 | avroBinarySchemaIdWithReaderSchemaDeserializer[UserCreated](schemaRegistryEndpoint, isKey = false, includesFormatByte = false) 243 | ) 244 | ``` 245 | 246 | ```scala mdoc:invisible 247 | consumer.close() 248 | ``` 249 | 250 | ## Format byte 251 | 252 | The Original Confluent Avro serializer/deserializer prefix the payload with a "magic" byte to identify that the message 253 | has been written with the Avro serializer. 254 | 255 | Similarly this library support the same mechanism by mean of a couple of function. It is even able to multiplex and 256 | demultiplex different serializers/deserializers based on that format byte. At the moment the supported formats are 257 | - JSON 258 | - Avro Binary with schema ID 259 | - Avro JSON with schema ID 260 | 261 | let's see this mechanism in action: 262 | ```scala mdoc:silent:reset 263 | import com.ovoenergy.kafka.serialization.core._ 264 | import com.ovoenergy.kafka.serialization.avro4s._ 265 | import com.ovoenergy.kafka.serialization.circe._ 266 | 267 | // Import the Circe generic support 268 | import io.circe.generic.auto._ 269 | import io.circe.syntax._ 270 | 271 | import org.apache.kafka.clients.producer.KafkaProducer 272 | import org.apache.kafka.clients.consumer.KafkaConsumer 273 | import org.apache.kafka.clients.CommonClientConfigs._ 274 | import scala.collection.JavaConverters._ 275 | 276 | 277 | sealed trait Event 278 | case class UserCreated(id: String, name: String, email: String) extends Event 279 | 280 | val schemaRegistryEndpoint = "http://localhost:8081" 281 | 282 | /* This producer will produce messages in Avro binary format */ 283 | val avroBinaryProducer = new KafkaProducer( 284 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 285 | nullSerializer[Unit], 286 | formatSerializer(Format.AvroBinarySchemaId, avroBinarySchemaIdSerializer[UserCreated](schemaRegistryEndpoint, isKey = false, includesFormatByte = false)) 287 | ) 288 | 289 | /* This producer will produce messages in Json format */ 290 | val circeProducer = new KafkaProducer( 291 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 292 | nullSerializer[Unit], 293 | formatSerializer(Format.Json, circeJsonSerializer[UserCreated]) 294 | ) 295 | 296 | /* This consumer will be able to consume messages from both producer */ 297 | val consumer = new KafkaConsumer( 298 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 299 | nullDeserializer[Unit], 300 | formatDemultiplexerDeserializer[UserCreated](unknownFormat => failingDeserializer(new RuntimeException("Unsupported format"))){ 301 | case Format.Json => circeJsonDeserializer[UserCreated] 302 | case Format.AvroBinarySchemaId => avroBinarySchemaIdDeserializer[UserCreated](schemaRegistryEndpoint, isKey = false, includesFormatByte = false) 303 | } 304 | ) 305 | 306 | /* This consumer will be able to consume messages in Avro binary format with the magic format byte at the start */ 307 | val avroBinaryConsumer = new KafkaConsumer( 308 | Map[String, AnyRef](BOOTSTRAP_SERVERS_CONFIG->"localhost:9092").asJava, 309 | nullDeserializer[Unit], 310 | avroBinarySchemaIdDeserializer[UserCreated](schemaRegistryEndpoint, isKey = false, includesFormatByte = true) 311 | ) 312 | ``` 313 | 314 | ```scala mdoc:invisible 315 | avroBinaryProducer.close() 316 | circeProducer.close() 317 | consumer.close() 318 | avroBinaryConsumer.close() 319 | ``` 320 | 321 | You can notice that the `formatDemultiplexerDeserializer` is little bit nasty because it is invariant in the type `T` so 322 | all the demultiplexed `serialiazer` must be declared as `Deserializer[T]`. 323 | 324 | There are other support serializer and deserializer, you can discover them looking trough the code and the tests. 325 | 326 | ## Useful de-serializers 327 | 328 | In the core module there are pleanty of serializers and deserializers that handle generic cases. 329 | 330 | ### Optional deserializer 331 | 332 | To handle the case in which the data is null, you need to wrap the deserializer in the `optionalDeserializer`: 333 | 334 | ```scala mdoc:silent:reset 335 | import com.ovoenergy.kafka.serialization.core._ 336 | import com.ovoenergy.kafka.serialization.circe._ 337 | 338 | // Import the Circe generic support 339 | import io.circe.generic.auto._ 340 | import io.circe.syntax._ 341 | 342 | import org.apache.kafka.common.serialization.Deserializer 343 | 344 | case class UserCreated(id: String, name: String, age: Int) 345 | 346 | val userCreatedDeserializer: Deserializer[Option[UserCreated]] = optionalDeserializer(circeJsonDeserializer[UserCreated]) 347 | ``` 348 | 349 | ## Cats instances 350 | 351 | The `cats` module provides the `Functor` typeclass instance for the `Deserializer` and `Contravariant` instance for the 352 | `Serializer`. This allow to do: 353 | 354 | ```scala mdoc:silent 355 | import cats.implicits._ 356 | import com.ovoenergy.kafka.serialization.core._ 357 | import com.ovoenergy.kafka.serialization.cats._ 358 | import org.apache.kafka.common.serialization.{Serializer, Deserializer, IntegerSerializer, IntegerDeserializer} 359 | 360 | val intDeserializer: Deserializer[Int] = (new IntegerDeserializer).asInstanceOf[Deserializer[Int]] 361 | val stringDeserializer: Deserializer[String] = intDeserializer.map(_.toString) 362 | 363 | val intSerializer: Serializer[Int] = (new IntegerSerializer).asInstanceOf[Serializer[Int]] 364 | val stringSerializer: Serializer[String] = intSerializer.contramap(_.toInt) 365 | ``` 366 | 367 | ## Complaints and other Feedback 368 | 369 | Feedback of any kind is always appreciated. 370 | 371 | Issues and PR's are welcome as well. 372 | 373 | ## About this README 374 | 375 | The code samples in this README file are checked using [mdoc](https://github.com/scalameta/mdoc). 376 | 377 | This means that the `README.md` file is generated from `docs/src/README.md`. If you want to make any changes to the README, you should: 378 | 379 | 1. Edit `docs/src/README.md` 380 | 2. Run `sbt mdoc` to regenerate `./README.md` 381 | 3. Commit both files to git -------------------------------------------------------------------------------- /json4s/src/main/scala/com/ovoenergy/kafka/serialization/json4s/Json4sSerialization.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.json4s 18 | 19 | import java.io.{ByteArrayInputStream, ByteArrayOutputStream, InputStreamReader, OutputStreamWriter} 20 | import java.nio.charset.StandardCharsets 21 | 22 | import com.ovoenergy.kafka.serialization.core._ 23 | import org.apache.kafka.common.serialization.{Deserializer => KafkaDeserializer, Serializer => KafkaSerializer} 24 | import org.json4s.Formats 25 | import org.json4s.native.Serialization.{read, write} 26 | 27 | import scala.reflect.ClassTag 28 | import scala.reflect.runtime.universe._ 29 | 30 | trait Json4sSerialization { 31 | 32 | def json4sSerializer[T <: AnyRef](implicit jsonFormats: Formats): KafkaSerializer[T] = serializer { (_, data) => 33 | val bout = new ByteArrayOutputStream() 34 | val writer = new OutputStreamWriter(bout, StandardCharsets.UTF_8) 35 | 36 | // TODO Use scala-arm 37 | try { 38 | write(data, writer) 39 | writer.flush() 40 | } finally { 41 | writer.close() 42 | } 43 | bout.toByteArray 44 | } 45 | 46 | def json4sDeserializer[T: TypeTag](implicit jsonFormats: Formats): KafkaDeserializer[T] = deserializer { (_, data) => 47 | val tt = implicitly[TypeTag[T]] 48 | implicit val cl = ClassTag[T](tt.mirror.runtimeClass(tt.tpe)) 49 | read[T](new InputStreamReader(new ByteArrayInputStream(data), StandardCharsets.UTF_8)) 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /json4s/src/main/scala/com/ovoenergy/kafka/serialization/json4s/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization 18 | 19 | package object json4s extends Json4sSerialization 20 | -------------------------------------------------------------------------------- /json4s/src/test/scala/com/ovoenergy/kafka/serialization/json4s/Json4sSerializationSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.json4s 18 | 19 | import java.nio.charset.StandardCharsets.UTF_8 20 | 21 | import com.ovoenergy.kafka.serialization.testkit.UnitSpec 22 | import com.ovoenergy.kafka.serialization.testkit.UnitSpec._ 23 | import org.json4s.DefaultFormats 24 | import org.json4s.native.Serialization._ 25 | 26 | class Json4sSerializationSpec extends UnitSpec with Json4sSerialization { 27 | 28 | implicit val formats = DefaultFormats 29 | 30 | "Json4sSerialization" when { 31 | "serializing" should { 32 | "write the Json json body" in forAll { event: Event => 33 | val serializer = json4sSerializer[Event] 34 | 35 | val bytes = serializer.serialize("Does not matter", event) 36 | 37 | read[Event](new String(bytes, UTF_8)) shouldBe event 38 | } 39 | } 40 | 41 | "deserializing" should { 42 | "parse the json" in forAll { event: Event => 43 | val deserializer = json4sDeserializer[Event] 44 | 45 | val bytes = write(event).getBytes(UTF_8) 46 | 47 | deserializer.deserialize("Does not matter", bytes) shouldBe event 48 | } 49 | } 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /jsoniter-scala/src/main/scala/com/ovoenergy/kafka/serialization/jsoniter_scala/JsoniterScalaSerialization.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.jsoniter_scala 18 | 19 | import com.github.plokhotnyuk.jsoniter_scala.core.{JsonValueCodec, _} 20 | import com.ovoenergy.kafka.serialization.core._ 21 | import org.apache.kafka.common.serialization.{Deserializer => KafkaDeserializer, Serializer => KafkaSerializer} 22 | 23 | private[jsoniter_scala] trait JsoniterScalaSerialization { 24 | 25 | def jsoniterScalaSerializer[T: JsonValueCodec](config: WriterConfig = WriterConfig): KafkaSerializer[T] = 26 | serializer((_, data) => writeToArray[T](data, config)) 27 | 28 | def jsoniterScalaDeserializer[T: JsonValueCodec](config: ReaderConfig = ReaderConfig): KafkaDeserializer[T] = 29 | deserializer((_, data) => readFromArray(data, config)) 30 | 31 | } 32 | -------------------------------------------------------------------------------- /jsoniter-scala/src/main/scala/com/ovoenergy/kafka/serialization/jsoniter_scala/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization 18 | 19 | package object jsoniter_scala extends JsoniterScalaSerialization 20 | -------------------------------------------------------------------------------- /jsoniter-scala/src/test/scala/com/ovoenergy/kafka/serialization/jsoniter_scala/JsoniterScalaSerializationSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.jsoniter_scala 18 | 19 | import com.github.plokhotnyuk.jsoniter_scala.core._ 20 | import com.github.plokhotnyuk.jsoniter_scala.macros._ 21 | import com.ovoenergy.kafka.serialization.testkit.UnitSpec 22 | import com.ovoenergy.kafka.serialization.testkit.UnitSpec._ 23 | 24 | class JsoniterScalaSerializationSpec extends UnitSpec with JsoniterScalaSerialization { 25 | 26 | implicit val eventCodec: JsonValueCodec[Event] = JsonCodecMaker.make[Event](CodecMakerConfig) 27 | 28 | "JsoniterScalaSerialization" when { 29 | "serializing" should { 30 | "write compact json" in forAll { event: Event => 31 | val serializer = jsoniterScalaSerializer[Event]() 32 | 33 | val jsonBytes = serializer.serialize("does not matter", event) 34 | 35 | jsonBytes.deep shouldBe writeToArray(event).deep 36 | } 37 | "write prettified json" in forAll { event: Event => 38 | val serializer = jsoniterScalaSerializer[Event](WriterConfig.withIndentionStep(2)) 39 | 40 | val jsonBytes = serializer.serialize("does not matter", event) 41 | 42 | jsonBytes.deep shouldBe writeToArray(event, WriterConfig.withIndentionStep(2)).deep 43 | } 44 | } 45 | 46 | "deserializing" should { 47 | "parse the json" in forAll { event: Event => 48 | val jsonBytes = writeToArray(event) 49 | val deserializer = jsoniterScalaDeserializer[Event]() 50 | 51 | val deserialized = deserializer.deserialize("does not matter", jsonBytes) 52 | 53 | deserialized shouldBe event 54 | } 55 | "throw parse exception with a hex dump in case of invalid input" in { 56 | val deserializer = jsoniterScalaDeserializer[Event]() 57 | 58 | assert(intercept[JsonReaderException] { 59 | deserializer.deserialize( 60 | "does not matter", 61 | """{"name":"vjTjvnkwbdGczk7ylwtsLzfkawxsydRul9Infmapftuhn"}""".getBytes 62 | ) 63 | }.getMessage.contains("""missing required field "id", offset: 0x00000037, buf: 64 | |+----------+-------------------------------------------------+------------------+ 65 | || | 0 1 2 3 4 5 6 7 8 9 a b c d e f | 0123456789abcdef | 66 | |+----------+-------------------------------------------------+------------------+ 67 | || 00000010 | 77 62 64 47 63 7a 6b 37 79 6c 77 74 73 4c 7a 66 | wbdGczk7ylwtsLzf | 68 | || 00000020 | 6b 61 77 78 73 79 64 52 75 6c 39 49 6e 66 6d 61 | kawxsydRul9Infma | 69 | || 00000030 | 70 66 74 75 68 6e 22 7d | pftuhn"} | 70 | |+----------+-------------------------------------------------+------------------+""".stripMargin)) 71 | } 72 | "throw parse exception without a hex dump in case of invalid input" in { 73 | val deserializer = jsoniterScalaDeserializer[Event](ReaderConfig.withAppendHexDumpToParseException(false)) 74 | 75 | assert(intercept[JsonReaderException] { 76 | deserializer.deserialize( 77 | "does not matter", 78 | """{"name":"vjTjvnkwbdGczk7ylwtsLzfkawxsydRul9Infmapftuhn"}""".getBytes 79 | ) 80 | }.getMessage.contains("""missing required field "id", offset: 0x00000037""")) 81 | } 82 | } 83 | } 84 | 85 | } 86 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.9.4 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.9.0") 2 | addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.3.1") 3 | addSbtPlugin("com.geirsson" % "sbt-scalafmt" % "1.5.1") 4 | addSbtPlugin("org.scalameta" % "sbt-mdoc" % "2.2.21") 5 | addSbtPlugin("fr.qux" % "sbt-release-tags-only" % "0.5.0") 6 | addSbtPlugin("com.dwijnand" % "sbt-dynver" % "4.1.1") 7 | addDependencyTreePlugin 8 | 9 | // TO mute sbt-git 10 | libraryDependencies += "org.slf4j" % "slf4j-nop" % "1.7.30" 11 | -------------------------------------------------------------------------------- /spray/src/main/scala/com/ovoenergy/kafka/serialization/spray/SpraySerialization.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.spray 18 | 19 | import java.io.{ByteArrayOutputStream, OutputStreamWriter} 20 | import java.nio.charset.StandardCharsets 21 | 22 | import org.apache.kafka.common.serialization.{Deserializer => KafkaDeserializer, Serializer => KafkaSerializer} 23 | import spray.json._ 24 | import com.ovoenergy.kafka.serialization.core._ 25 | 26 | trait SpraySerialization { 27 | 28 | def spraySerializer[T](implicit format: JsonWriter[T]): KafkaSerializer[T] = serializer { (_, data) => 29 | val bout = new ByteArrayOutputStream() 30 | val osw = new OutputStreamWriter(bout, StandardCharsets.UTF_8) 31 | 32 | // TODO use scala-arm 33 | try { 34 | osw.write(data.toJson.compactPrint) 35 | osw.flush() 36 | } finally { 37 | osw.close() 38 | } 39 | bout.toByteArray 40 | } 41 | 42 | def sprayDeserializer[T](implicit format: JsonReader[T]): KafkaDeserializer[T] = deserializer { (_, data) => 43 | JsonParser(ParserInput(data)).convertTo[T] 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /spray/src/main/scala/com/ovoenergy/kafka/serialization/spray/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization 18 | 19 | package object spray extends SpraySerialization 20 | -------------------------------------------------------------------------------- /spray/src/test/scala/com/ovoenergy/kafka/serialization/spray/SpraySerializationSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.spray 18 | 19 | import java.nio.charset.StandardCharsets.UTF_8 20 | 21 | import com.ovoenergy.kafka.serialization.spray.SpraySerializationSpec._ 22 | import com.ovoenergy.kafka.serialization.testkit.UnitSpec 23 | import com.ovoenergy.kafka.serialization.testkit.UnitSpec._ 24 | import spray.json.DefaultJsonProtocol._ 25 | import spray.json._ 26 | 27 | object SpraySerializationSpec { 28 | 29 | val IgnoredTopic = "ignored" 30 | implicit val EventFormat: RootJsonFormat[Event] = jsonFormat2(Event) 31 | 32 | } 33 | 34 | class SpraySerializationSpec extends UnitSpec with SpraySerialization { 35 | 36 | "SpraySerialization" when { 37 | "serializing" should { 38 | "write the json body" in forAll { event: Event => 39 | val serializer = spraySerializer[Event] 40 | 41 | val bytes = serializer.serialize(IgnoredTopic, event) 42 | 43 | new String(bytes, UTF_8).parseJson shouldBe event.toJson 44 | } 45 | } 46 | 47 | "deserializing" should { 48 | "parse the json" in forAll { event: Event => 49 | val deserializer = sprayDeserializer[Event] 50 | 51 | val bytes = event.toJson.compactPrint.getBytes(UTF_8) 52 | 53 | deserializer.deserialize(IgnoredTopic, bytes) shouldBe event 54 | } 55 | } 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /testkit/src/main/resources/application.conf: -------------------------------------------------------------------------------- 1 | akka { 2 | test.timefactor = 1.0 3 | test.timefactor = ${?TEST_TIMEFACTOR} 4 | } -------------------------------------------------------------------------------- /testkit/src/main/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | target/test.log 5 | true 6 | true 7 | 8 | %-4relative [%thread] %-5level %logger{35} - %msg%n 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /testkit/src/main/scala/com/ovoenergy/kafka/serialization/testkit/UnitSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.testkit 18 | 19 | import org.scalacheck.{Arbitrary, Gen} 20 | import org.scalacheck.Arbitrary._ 21 | import org.scalatest.concurrent.{ScalaFutures, ScaledTimeSpans} 22 | import org.scalatest.prop.PropertyChecks 23 | import org.scalatest.{Matchers, WordSpec} 24 | 25 | object UnitSpec { 26 | 27 | case class Event(id: String, name: String) 28 | 29 | implicit val arbString: Arbitrary[String] = Arbitrary(for { 30 | length <- Gen.chooseNum(3, 64) 31 | chars <- Gen.listOfN(length, Gen.alphaNumChar) 32 | } yield chars.mkString) 33 | 34 | implicit val arbEvent: Arbitrary[Event] = Arbitrary(for { 35 | id <- arbitrary[String] 36 | name <- arbitrary[String] 37 | } yield Event(id, name)) 38 | 39 | } 40 | 41 | abstract class UnitSpec extends WordSpec with Matchers with PropertyChecks with ScalaFutures with ScaledTimeSpans { 42 | 43 | override lazy val spanScaleFactor: Double = 44 | sys.env.get("TEST_TIME_FACTOR").map(_.toDouble).getOrElse(super.spanScaleFactor) 45 | } 46 | -------------------------------------------------------------------------------- /testkit/src/main/scala/com/ovoenergy/kafka/serialization/testkit/WireMockFixture.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.kafka.serialization.testkit 18 | 19 | import com.github.tomakehurst.wiremock.WireMockServer 20 | import com.github.tomakehurst.wiremock.client.WireMock 21 | import com.github.tomakehurst.wiremock.core.WireMockConfiguration 22 | import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, Suite} 23 | 24 | trait WireMockFixture extends BeforeAndAfterAll with BeforeAndAfterEach { self: Suite => 25 | 26 | private lazy val wireMockServer: WireMockServer = new WireMockServer(WireMockConfiguration.options().dynamicPort()) 27 | 28 | val wireMockHost: String = "localhost" 29 | def wireMockPort: Int = wireMockServer.port() 30 | def wireMockEndpoint: String = s"http://$wireMockHost:$wireMockPort" 31 | 32 | override protected def beforeAll(): Unit = { 33 | super.beforeAll() 34 | wireMockServer.start() 35 | WireMock.configureFor(wireMockPort) 36 | } 37 | 38 | override protected def afterAll(): Unit = { 39 | wireMockServer.shutdown() 40 | super.afterAll() 41 | } 42 | 43 | override protected def afterEach(): Unit = { 44 | resetWireMock() 45 | super.afterEach() 46 | } 47 | 48 | override protected def beforeEach(): Unit = { 49 | super.beforeEach() 50 | resetWireMock() 51 | } 52 | 53 | def resetWireMock(): Unit = { 54 | wireMockServer.resetMappings() 55 | wireMockServer.resetRequests() 56 | wireMockServer.resetScenarios() 57 | } 58 | } 59 | --------------------------------------------------------------------------------