├── .github └── workflows │ ├── master.yml │ └── pr.yml ├── .gitignore ├── LICENSE ├── Readme.md ├── build.gradle.kts ├── gradle ├── libs.versions.toml └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle.kts └── src ├── integrationTest └── kotlin │ └── com │ └── github │ └── thake │ └── kafka │ └── avro4k │ └── serializer │ ├── ConfluentCluster.kt │ └── ConfluentIT.kt ├── main └── kotlin │ └── com │ └── github │ └── thake │ └── kafka │ └── avro4k │ └── serializer │ ├── AbstractKafkaAvro4kDeserializer.kt │ ├── AbstractKafkaAvro4kSerDe.kt │ ├── AbstractKafkaAvro4kSerDeConfig.kt │ ├── AbstractKafkaAvro4kSerializer.kt │ ├── Avro4kSchemaUtils.kt │ ├── Avro4kSerde.kt │ ├── ClassExtensions.kt │ ├── KafkaAvro4kDeserializer.kt │ ├── KafkaAvro4kDeserializerConfig.kt │ ├── KafkaAvro4kSerializer.kt │ ├── KafkaAvro4kSerializerConfig.kt │ ├── RecordLookup.kt │ └── TypedKafkaAvro4kDeserializer.kt └── test └── kotlin └── com └── github └── thake └── kafka └── avro4k └── serializer ├── Avro4kSerdeTest.kt ├── ClassloaderTest.kt ├── KafkaAvro4kDeserializerTest.kt ├── KafkaAvro4kSerializerTest.kt ├── PrintConfigDocumentation.kt └── TestNetworkOutageRecovery.kt /.github/workflows/master.yml: -------------------------------------------------------------------------------- 1 | name: build-master 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - release/* 8 | paths-ignore: 9 | - 'doc/**' 10 | - '*.md' 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Checkout the repo 17 | uses: actions/checkout@v2 18 | with: 19 | fetch-depth: 0 20 | 21 | - name: Run tests 22 | run: ./gradlew check 23 | 24 | - name: Bundle the build report 25 | if: failure() 26 | run: find . -type d -name 'reports' | zip -@ -r build-reports.zip 27 | 28 | - name: Upload the build report 29 | if: failure() 30 | uses: actions/upload-artifact@master 31 | with: 32 | name: error-report 33 | path: build-reports.zip 34 | 35 | deploy: 36 | needs: build 37 | runs-on: ubuntu-latest 38 | steps: 39 | - name: Checkout the repo 40 | uses: actions/checkout@v2 41 | 42 | - name: deploy to sonatype snapshots 43 | run: ./gradlew build 44 | 45 | env: 46 | GRADLE_OPTS: -Dorg.gradle.configureondemand=true -Dorg.gradle.parallel=false -Dkotlin.incremental=false -Dorg.gradle.jvmargs="-Xmx3g -XX:MaxPermSize=2048m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8" 47 | -------------------------------------------------------------------------------- /.github/workflows/pr.yml: -------------------------------------------------------------------------------- 1 | name: build-pr 2 | 3 | on: 4 | pull_request: 5 | paths-ignore: 6 | - 'doc/**' 7 | - '*.md' 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout the repo 14 | uses: actions/checkout@v2 15 | with: 16 | fetch-depth: 0 17 | 18 | - name: Run tests 19 | run: ./gradlew check 20 | 21 | - name: Bundle the build report 22 | if: failure() 23 | run: find . -type d -name 'reports' | zip -@ -r build-reports.zip 24 | 25 | - name: Upload the build report 26 | if: failure() 27 | uses: actions/upload-artifact@master 28 | with: 29 | name: error-report 30 | path: build-reports.zip 31 | 32 | env: 33 | GRADLE_OPTS: -Dorg.gradle.configureondemand=true -Dorg.gradle.parallel=false -Dkotlin.incremental=false -Dorg.gradle.jvmargs="-Xmx3g -XX:MaxPermSize=2048m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8" 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Project exclude paths 2 | /.gradle/ 3 | /.idea/ 4 | /build/ 5 | /gradle.properties 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # Kafka avro4k serializer / deserializer 2 | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.github.thake.avro4k/avro4k-kafka-serializer/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.github.thake.avro4k/avro4k-kafka-serializer) 3 | 4 | This project implements a Kafka serializer / deserializer that integrates with the confluent schema registry and 5 | leverages [avro4k](https://github.com/avro-kotlin/avro4k). It is based on confluent's [Kafka Serializer]((https://github.com/confluentinc/schema-registry/tree/master/avro-serializer)). 6 | As such, this implementations can be used to in several projects (i.e. spring) 7 | 8 | This SerDe supports retrying of failed calls to the schema registry (i.e. due to flaky network). Confluent's Serde does not implement this yet. 9 | See https://github.com/confluentinc/schema-registry/issues/928. 10 | 11 | ## Confluent Versions 12 | 13 | Version 0.10.x is compatible with Apache Kafka 2.5.x / Confluent 5.5.x and avro4k < 1.0 14 | 15 | Version 0.11.x+ is compatible with Apache Kafka 2.6.x / Confluent 6.0.0 and avro4k < 1.0 16 | 17 | Version > 0.13 is compatible with Apache Kafka 2.6.x/3.x (Confluent 6.x/7.x) and avro4k 1.x 18 | 19 | ## Example usage 20 | 21 | You can find an example configuration of a Kafka Consumer, Kafka Producer and Kafka Streams application in the [ConfluentIT](./src/integrationTest/kotlin/com/github/thake/kafka/avro4k/serializer/ConfluentIT.kt) integration test. 22 | 23 | ### Spring Cloud Stream with Kafka 24 | 25 | ```javascript 26 | spring: 27 | application: 28 | name: 29 | kafka: 30 | bootstrap-servers: 31 | - 32 | 33 | cloud: 34 | stream: 35 | default: 36 | consumer: 37 | useNativeDecoding: true 38 | producer: 39 | useNativeEncoding: true 40 | kafka: 41 | streams: 42 | binder: 43 | brokers: 44 | configuration: 45 | schema.registry.url: 46 | schema.registry.retry.attemps: 3 47 | schema.registry.retry.jitter.base: 10 48 | schema.registry.retry.jitter.max: 5000 49 | record.packages: 50 | default.key.serde: com.github.thake.kafka.avro4k.serializer.Avro4kSerde 51 | default.value.serde: com.github.thake.kafka.avro4k.serializer.Avro4kSerde 52 | default: 53 | producer: 54 | keySerde: com.github.thake.kafka.avro4k.serializer.Avro4kSerde 55 | valueSerde: com.github.thake.kafka.avro4k.serializer.Avro4kSerde 56 | consumer: 57 | keySerde: com.github.thake.kafka.avro4k.serializer.Avro4kSerde 58 | valueSerde: com.github.thake.kafka.avro4k.serializer.Avro4kSerde 59 | ... 60 | ``` 61 | 62 | ### Maven 63 | 64 | ```xml 65 | 66 | com.github.thake.avro4k 67 | avro4k-kafka-serializer 68 | VERSION 69 | 70 | ``` 71 | 72 | ### Configuration options 73 | 74 | - `schema.registry.url` 75 | Comma-separated list of URLs for schema registry instances that can 76 | be used to register or look up schemas. If you wish to get a 77 | connection to a mocked schema registry for testing, you can specify 78 | a scope using the `mock://` pseudo-protocol. For example, 79 | `mock://my-scope-name` corresponds to 80 | `MockSchemaRegistry.getClientForScope("my-scope-name")`. 81 | 82 | - Type: list 83 | - Importance: high 84 | 85 | - `record.packages` 86 | The packages in which record types annotated with `@AvroName`, 87 | `@AvroAlias` and `@AvroNamespace` can be found. Packages are separated 88 | by a comma `,`. Only needed for deserialization. 89 | 90 | - Type: string 91 | - Default: null 92 | - Importance: high 93 | 94 | - `schema.registry.ssl.key.password` 95 | The password of the private key in the key store file or the PEM key 96 | specified in `ssl.keystore.key`. This is required for clients only 97 | if two-way authentication is configured. 98 | 99 | - Type: password 100 | - Default: null 101 | - Importance: high 102 | 103 | - `schema.registry.ssl.keystore.certificate.chain` 104 | Certificate chain in the format specified by `ssl.keystore.type`. 105 | Default SSL engine factory supports only PEM format with a list of 106 | X.509 certificates 107 | 108 | - Type: password 109 | - Default: null 110 | - Importance: high 111 | 112 | - `schema.registry.ssl.keystore.key` 113 | Private key in the format specified by `ssl.keystore.type`. Default 114 | SSL engine factory supports only PEM format with PKCS\#8 keys. If 115 | the key is encrypted, key password must be specified using 116 | `ssl.key.password` 117 | 118 | - Type: password 119 | - Default: null 120 | - Importance: high 121 | 122 | - `schema.registry.ssl.keystore.location` 123 | The location of the key store file. This is optional for client and 124 | can be used for two-way authentication for client. 125 | 126 | - Type: string 127 | - Default: null 128 | - Importance: high 129 | 130 | - `schema.registry.ssl.keystore.password` 131 | The store password for the key store file. This is optional for 132 | client and only needed if `ssl.keystore.location` is configured. Key 133 | store password is not supported for PEM format. 134 | 135 | - Type: password 136 | - Default: null 137 | - Importance: high 138 | 139 | - `schema.registry.ssl.truststore.certificates` 140 | Trusted certificates in the format specified by 141 | `ssl.truststore.type`. Default SSL engine factory supports only PEM 142 | format with X.509 certificates. 143 | 144 | - Type: password 145 | - Default: null 146 | - Importance: high 147 | 148 | - `schema.registry.ssl.truststore.location` 149 | The location of the trust store file. 150 | 151 | - Type: string 152 | - Default: null 153 | - Importance: high 154 | 155 | - `schema.registry.ssl.truststore.password` 156 | The password for the trust store file. If a password is not set, 157 | trust store file configured will still be used, but integrity 158 | checking is disabled. Trust store password is not supported for PEM 159 | format. 160 | 161 | - Type: password 162 | - Default: null 163 | - Importance: high 164 | 165 | - `auto.register.schemas` 166 | Specify if the Serializer should attempt to register the Schema with 167 | Schema Registry 168 | 169 | - Type: boolean 170 | - Default: true 171 | - Importance: medium 172 | 173 | - `basic.auth.credentials.source` 174 | Specify how to pick the credentials for Basic Auth header. The 175 | supported values are URL, USER\_INFO and SASL\_INHERIT 176 | 177 | - Type: string 178 | - Default: URL 179 | - Importance: medium 180 | 181 | - `basic.auth.user.info` 182 | Specify the user info for Basic Auth in the form of 183 | {username}:{password} 184 | 185 | - Type: password 186 | - Default: \[hidden\] 187 | - Importance: medium 188 | 189 | - `bearer.auth.credentials.source` 190 | Specify how to pick the credentials for Bearer Auth header. 191 | 192 | - Type: string 193 | - Default: STATIC\_TOKEN 194 | - Importance: medium 195 | 196 | - `bearer.auth.token` 197 | Specify the Bearer token to be used for authentication 198 | 199 | - Type: password 200 | - Default: \[hidden\] 201 | - Importance: medium 202 | 203 | - `context.name.strategy` 204 | A class used to determine the schema registry context. 205 | 206 | - Type: class 207 | - Default: 208 | io.confluent.kafka.serializers.context.NullContextNameStrategy 209 | - Importance: medium 210 | 211 | - `key.subject.name.strategy` 212 | Determines how to construct the subject name under which the key 213 | schema is registered with the schema registry. By default, 214 | \-key is used as subject. 215 | 216 | - Type: class 217 | - Default: 218 | io.confluent.kafka.serializers.subject.TopicNameStrategy 219 | - Importance: medium 220 | 221 | - `normalize.schemas` 222 | Whether to normalize schemas, which generally ignores ordering when 223 | it is not significant 224 | 225 | - Type: boolean 226 | - Default: false 227 | - Importance: medium 228 | 229 | - `schema.registry.basic.auth.user.info` 230 | Specify the user info for Basic Auth in the form of 231 | {username}:{password} 232 | 233 | - Type: password 234 | - Default: \[hidden\] 235 | - Importance: medium 236 | 237 | - `schema.registry.ssl.enabled.protocols` 238 | The list of protocols enabled for SSL connections. The default is 239 | `TLSv1.2,TLSv1.3` when running with Java 11 or newer, `TLSv1.2` 240 | otherwise. With the default value for Java 11, clients and servers 241 | will prefer TLSv1.3 if both support it and fallback to TLSv1.2 242 | otherwise (assuming both support at least TLSv1.2). This default 243 | should be fine for most cases. Also see the config documentation for 244 | ssl.protocol. 245 | 246 | - Type: list 247 | - Default: TLSv1.2,TLSv1.3 248 | - Importance: medium 249 | 250 | - `schema.registry.ssl.keystore.type` 251 | The file format of the key store file. This is optional for client. 252 | 253 | - Type: string 254 | - Default: JKS 255 | - Importance: medium 256 | 257 | - `schema.registry.ssl.protocol` 258 | The SSL protocol used to generate the SSLContext. The default is 259 | `TLSv1.3` when running with Java 11 or newer, `TLSv1.2` otherwise. 260 | This value should be fine for most use cases. Allowed values in 261 | recent JVMs are `TLSv1.2` and `TLSv1.3`. `TLS`, `TLSv1.1`, `SSL`, 262 | `SSLv2` and `SSLv3` may be supported in older JVMs, but their usage 263 | is discouraged due to known security vulnerabilities. With the 264 | default value for this config and `ssl.enabled.protocols`, clients 265 | will downgrade to `TLSv1.2` if the server does not support 266 | `TLSv1.3`. If this config is set to `TLSv1.2`, clients will not use 267 | `TLSv1.3` even if it is one of the values in ssl.enabled.protocols 268 | and the server only supports `TLSv1.3`. 269 | 270 | - Type: string 271 | - Default: TLSv1.3 272 | - Importance: medium 273 | 274 | - `schema.registry.ssl.provider` 275 | The name of the security provider used for SSL connections. Default 276 | value is the default security provider of the JVM. 277 | 278 | - Type: string 279 | - Default: null 280 | - Importance: medium 281 | 282 | - `schema.registry.ssl.truststore.type` 283 | The file format of the trust store file. 284 | 285 | - Type: string 286 | - Default: JKS 287 | - Importance: medium 288 | 289 | - `value.subject.name.strategy` 290 | Determines how to construct the subject name under which the value 291 | schema is registered with the schema registry. By default, 292 | \-value is used as subject. 293 | 294 | - Type: class 295 | - Default: 296 | io.confluent.kafka.serializers.subject.TopicNameStrategy 297 | - Importance: medium 298 | 299 | - `id.compatibility.strict` 300 | Whether to check for backward compatibility between the schema with 301 | the given ID and the schema of the object to be serialized 302 | 303 | - Type: boolean 304 | - Default: true 305 | - Importance: low 306 | 307 | - `latest.compatibility.strict` 308 | Whether to check for backward compatibility between the latest 309 | subject version and the schema of the object to be serialized 310 | 311 | - Type: boolean 312 | - Default: true 313 | - Importance: low 314 | 315 | - `max.schemas.per.subject` 316 | Maximum number of schemas to create or cache locally. 317 | 318 | - Type: int 319 | - Default: 1000 320 | - Importance: low 321 | 322 | - `proxy.host` 323 | The hostname, or address, of the proxy server that will be used to 324 | connect to the schema registry instances. 325 | 326 | - Type: string 327 | - Default: "" 328 | - Importance: low 329 | 330 | - `proxy.port` 331 | The port number of the proxy server that will be used to connect to 332 | the schema registry instances. 333 | 334 | - Type: int 335 | - Default: -1 336 | - Importance: low 337 | 338 | - `schema.reflection` 339 | If true, uses the reflection API when serializing/deserializing 340 | 341 | - Type: boolean 342 | - Default: false 343 | - Importance: low 344 | 345 | - `schema.registry.retry.attempts` 346 | Number of retry attempts that will be made if the schema registry 347 | seems to have a problem with requesting a schema. 348 | 349 | - Type: int 350 | - Default: 5 351 | - Importance: low 352 | 353 | - `schema.registry.retry.jitter.base` 354 | Milliseconds that are used as a base for the jitter calculation 355 | (sleep = random\_between(0, min(max, base \* 2 \*\* attempt))) 356 | 357 | - Type: long 358 | - Default: 10 359 | - Importance: low 360 | 361 | - `schema.registry.retry.jitter.max` 362 | Milliseconds that are used as max for the jitter calculation (sleep 363 | = random\_between(0, min(max, base \* 2 \*\* attempt))) 364 | 365 | - Type: long 366 | - Default: 5000 367 | - Importance: low 368 | 369 | - `schema.registry.ssl.cipher.suites` 370 | A list of cipher suites. This is a named combination of 371 | authentication, encryption, MAC and key exchange algorithm used to 372 | negotiate the security settings for a network connection using TLS 373 | or SSL network protocol. By default all the available cipher suites 374 | are supported. 375 | 376 | - Type: list 377 | - Default: null 378 | - Importance: low 379 | 380 | - `schema.registry.ssl.endpoint.identification.algorithm` 381 | The endpoint identification algorithm to validate server hostname 382 | using server certificate. 383 | 384 | - Type: string 385 | - Default: https 386 | - Importance: low 387 | 388 | - `schema.registry.ssl.engine.factory.class` 389 | The class of type 390 | org.apache.kafka.common.security.auth.SslEngineFactory to provide 391 | SSLEngine objects. Default value is 392 | org.apache.kafka.common.security.ssl.DefaultSslEngineFactory 393 | 394 | - Type: class 395 | - Default: null 396 | - Importance: low 397 | 398 | - `schema.registry.ssl.keymanager.algorithm` 399 | The algorithm used by key manager factory for SSL connections. 400 | Default value is the key manager factory algorithm configured for 401 | the Java Virtual Machine. 402 | 403 | - Type: string 404 | - Default: SunX509 405 | - Importance: low 406 | 407 | - `schema.registry.ssl.secure.random.implementation` 408 | The SecureRandom PRNG implementation to use for SSL cryptography 409 | operations. 410 | 411 | - Type: string 412 | - Default: null 413 | - Importance: low 414 | 415 | - `schema.registry.ssl.trustmanager.algorithm` 416 | The algorithm used by trust manager factory for SSL connections. 417 | Default value is the trust manager factory algorithm configured for 418 | the Java Virtual Machine. 419 | 420 | - Type: string 421 | - Default: PKIX 422 | - Importance: low 423 | 424 | - `use.latest.version` 425 | Specify if the Serializer should use the latest subject version for 426 | serialization 427 | 428 | - Type: boolean 429 | - Default: false 430 | - Importance: low 431 | 432 | - `use.schema.id` 433 | Schema ID to use for serialization 434 | - Type: int 435 | - Default: -1 436 | - Importance: low 437 | -------------------------------------------------------------------------------- /build.gradle.kts: -------------------------------------------------------------------------------- 1 | @file:Suppress("UnstableApiUsage") 2 | 3 | group = "com.github.thake.avro4k" 4 | val javaCompatibility = "1.8" 5 | 6 | plugins { 7 | `java-library` 8 | `jvm-test-suite` 9 | idea 10 | `maven-publish` 11 | signing 12 | alias(libs.plugins.kotlin.jvm) 13 | alias(libs.plugins.kotlin.serialization) 14 | alias(libs.plugins.dokka) 15 | alias(libs.plugins.versions) 16 | alias(libs.plugins.release) 17 | } 18 | 19 | repositories { 20 | mavenCentral() 21 | mavenLocal() 22 | maven("https://packages.confluent.io/maven/") 23 | } 24 | 25 | dependencies { 26 | api(libs.kafka.avro.serde) 27 | implementation(libs.kotlin.reflect) 28 | implementation(libs.kotlin.stdlibJdk8) 29 | implementation(libs.kotlinx.serialization) 30 | implementation(libs.kotlinx.coroutines) 31 | implementation(libs.avro) 32 | implementation(libs.kafka.avro.serializer) 33 | implementation(libs.avro4k) 34 | implementation(libs.classgraph) 35 | implementation(libs.retry) 36 | testImplementation(libs.bundles.test) 37 | testRuntimeOnly(libs.junit.runtime) 38 | 39 | } 40 | // Configure existing Dokka task to output HTML to typical Javadoc directory 41 | tasks.dokkaHtml.configure { 42 | outputDirectory.set(buildDir.resolve("javadoc")) 43 | } 44 | 45 | // Create dokka Jar task from dokka task output 46 | val dokkaJar by tasks.creating(Jar::class) { 47 | group = JavaBasePlugin.DOCUMENTATION_GROUP 48 | description = "Assembles Kotlin docs with Dokka" 49 | archiveClassifier.set("javadoc") 50 | // dependsOn(tasks.dokka) not needed; dependency automatically inferred by from(tasks.dokka) 51 | from(tasks.dokkaHtml) 52 | } 53 | 54 | testing { 55 | suites { 56 | val test by getting(JvmTestSuite::class) { 57 | useJUnitJupiter() 58 | } 59 | val integrationTest by registering(JvmTestSuite::class) { 60 | 61 | dependencies { 62 | implementation(project) 63 | implementation(libs.testcontainers) 64 | implementation(libs.testcontainers.kafka) 65 | implementation(libs.kafka.streams) 66 | implementation(libs.kotlinx.serialization) 67 | implementation(libs.bundles.test) 68 | implementation(libs.bundles.logging) 69 | } 70 | 71 | targets { 72 | all { 73 | testTask.configure { 74 | shouldRunAfter(test) 75 | } 76 | } 77 | } 78 | } 79 | } 80 | } 81 | 82 | tasks { 83 | compileJava { 84 | targetCompatibility = javaCompatibility 85 | } 86 | compileKotlin { 87 | kotlinOptions.jvmTarget = javaCompatibility 88 | kotlinOptions.freeCompilerArgs += "-opt-in=kotlin.RequiresOptIn" 89 | } 90 | compileTestJava { 91 | targetCompatibility = javaCompatibility 92 | } 93 | compileTestKotlin { 94 | kotlinOptions.jvmTarget = javaCompatibility 95 | } 96 | beforeReleaseBuild { 97 | dependsOn(testing.suites.named("integrationTest")) 98 | } 99 | 100 | idea { 101 | module { 102 | isDownloadSources = true 103 | isDownloadJavadoc = false 104 | } 105 | } 106 | } 107 | 108 | // Create sources Jar from main kotlin sources 109 | val sourcesJar by tasks.creating(Jar::class) { 110 | group = JavaBasePlugin.DOCUMENTATION_GROUP 111 | description = "Assembles sources JAR" 112 | archiveClassifier.set("sources") 113 | from(sourceSets.main.get().allSource) 114 | } 115 | publishing{ 116 | repositories{ 117 | maven{ 118 | name = "mavenCentral" 119 | url = if (project.isSnapshot) { 120 | uri("https://oss.sonatype.org/content/repositories/snapshots/") 121 | } else { 122 | uri("https://oss.sonatype.org/service/local/staging/deploy/maven2/") 123 | } 124 | credentials { 125 | username = project.findProperty("ossrhUsername") as? String 126 | password = project.findProperty("ossrhPassword") as? String 127 | } 128 | } 129 | } 130 | publications{ 131 | create("mavenJava"){ 132 | from(components["java"]) 133 | artifact(sourcesJar) 134 | artifact(dokkaJar) 135 | //artifact(javadocJar.get()) 136 | pom{ 137 | name.set("Kafka serializer using avro4k") 138 | description.set("Provides Kafka SerDes and Serializer / Deserializer implementations for avro4k") 139 | url.set("https://github.com/thake/avro4k-kafka-serializer") 140 | developers { 141 | developer { 142 | name.set("Thorsten Hake") 143 | email.set("mail@thorsten-hake.com") 144 | } 145 | } 146 | scm { 147 | connection.set("https://github.com/thake/avro4k-kafka-serializer.git") 148 | developerConnection.set("scm:git:ssh://github.com:thake/avro4k-kafka-serializer.git") 149 | url.set("https://github.com/thake/avro4k-kafka-serializer") 150 | } 151 | licenses { 152 | license { 153 | name.set("The Apache License, Version 2.0") 154 | url.set("https://www.apache.org/licenses/LICENSE-2.0.txt") 155 | } 156 | } 157 | } 158 | } 159 | } 160 | } 161 | signing { 162 | useGpgCmd() 163 | isRequired = !isSnapshot 164 | sign(publishing.publications["mavenJava"]) 165 | } 166 | tasks.named("afterReleaseBuild") { 167 | dependsOn("publish") 168 | } 169 | 170 | inline val Project.isSnapshot 171 | get() = version.toString().endsWith("-SNAPSHOT") 172 | -------------------------------------------------------------------------------- /gradle/libs.versions.toml: -------------------------------------------------------------------------------- 1 | [versions] 2 | kotlin = "1.7.10" 3 | avro = "1.11.1" 4 | confluent = "7.2.1" 5 | kafka = "3.2.1" 6 | avro4k = "1.6.0" 7 | junit = "5.9.0" 8 | kotest = "5.4.2" 9 | logback = "1.2.11" 10 | testcontainers = "1.17.3" 11 | 12 | [plugins] 13 | kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } 14 | kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" } 15 | dokka = { id = "org.jetbrains.dokka", version.ref = "kotlin" } 16 | release = { id = "net.researchgate.release", version = "3.0.1" } 17 | versions = { id = "com.github.ben-manes.versions", version = "0.42.0" } 18 | 19 | [libraries] 20 | avro = { module = "org.apache.avro:avro", version.ref = "avro" } 21 | kafka-avro-serializer = { module = "io.confluent:kafka-avro-serializer", version.ref = "confluent" } 22 | kafka-avro-serde = { module = "io.confluent:kafka-streams-avro-serde", version.ref = "confluent" } 23 | avro4k = { module = "com.github.avro-kotlin.avro4k:avro4k-core", version.ref = "avro4k" } 24 | kotlin-reflect = { module = "org.jetbrains.kotlin:kotlin-reflect", version.ref = "kotlin" } 25 | kotlin-stdlibJdk8 = { module = "org.jetbrains.kotlin:kotlin-stdlib-jdk8", version.ref = "kotlin" } 26 | kotlinx-serialization = { module = "org.jetbrains.kotlinx:kotlinx-serialization-core", version = "1.4.0" } 27 | kotlinx-coroutines = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version = "1.6.4" } 28 | classgraph = { module = "io.github.classgraph:classgraph", version = "4.8.149" } 29 | retry = { module = "com.michael-bull.kotlin-retry:kotlin-retry", version = "1.0.9" } 30 | #Test libs 31 | kafka-streams = { module = "org.apache.kafka:kafka-streams", version.ref = "kafka" } 32 | testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } 33 | testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref = "testcontainers" } 34 | junit-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "junit" } 35 | junit-runtime = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit" } 36 | junit-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "junit" } 37 | kotest-runner = { module = "io.kotest:kotest-runner-junit5", version.ref = "kotest" } 38 | kotest-assertions = { module = "io.kotest:kotest-assertions-core", version.ref = "kotest" } 39 | logback-classic = { module = "ch.qos.logback:logback-classic", version.ref = "logback" } 40 | logback-core = { module = "ch.qos.logback:logback-core", version.ref = "logback" } 41 | mockk = { module = "io.mockk:mockk", version = "1.12.7" } 42 | 43 | [bundles] 44 | test = ["junit-api", "junit-params", "kotest-assertions", "kotest-runner", "mockk"] 45 | logging = ["logback-classic", "logback-core"] -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thake/avro4k-kafka-serializer/5f1396521964ea7a03299f7381773cfcabf4f795/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | # 21 | # Gradle start up script for POSIX generated by Gradle. 22 | # 23 | # Important for running: 24 | # 25 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 26 | # noncompliant, but you have some other compliant shell such as ksh or 27 | # bash, then to run this script, type that shell name before the whole 28 | # command line, like: 29 | # 30 | # ksh Gradle 31 | # 32 | # Busybox and similar reduced shells will NOT work, because this script 33 | # requires all of these POSIX shell features: 34 | # * functions; 35 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 36 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 37 | # * compound commands having a testable exit status, especially «case»; 38 | # * various built-in commands including «command», «set», and «ulimit». 39 | # 40 | # Important for patching: 41 | # 42 | # (2) This script targets any POSIX shell, so it avoids extensions provided 43 | # by Bash, Ksh, etc; in particular arrays are avoided. 44 | # 45 | # The "traditional" practice of packing multiple parameters into a 46 | # space-separated string is a well documented source of bugs and security 47 | # problems, so this is (mostly) avoided, by progressively accumulating 48 | # options in "$@", and eventually passing that to Java. 49 | # 50 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 51 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 52 | # see the in-line comments for details. 53 | # 54 | # There are tweaks for specific operating systems such as AIX, CygWin, 55 | # Darwin, MinGW, and NonStop. 56 | # 57 | # (3) This script is generated from the Groovy template 58 | # https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 59 | # within the Gradle project. 60 | # 61 | # You can find Gradle at https://github.com/gradle/gradle/. 62 | # 63 | ############################################################################## 64 | 65 | # Attempt to set APP_HOME 66 | 67 | # Resolve links: $0 may be a link 68 | app_path=$0 69 | 70 | # Need this for daisy-chained symlinks. 71 | while 72 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 73 | [ -h "$app_path" ] 74 | do 75 | ls=$( ls -ld "$app_path" ) 76 | link=${ls#*' -> '} 77 | case $link in #( 78 | /*) app_path=$link ;; #( 79 | *) app_path=$APP_HOME$link ;; 80 | esac 81 | done 82 | 83 | APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit 84 | 85 | APP_NAME="Gradle" 86 | APP_BASE_NAME=${0##*/} 87 | 88 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 89 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 90 | 91 | # Use the maximum available, or set MAX_FD != -1 to use that value. 92 | MAX_FD=maximum 93 | 94 | warn () { 95 | echo "$*" 96 | } >&2 97 | 98 | die () { 99 | echo 100 | echo "$*" 101 | echo 102 | exit 1 103 | } >&2 104 | 105 | # OS specific support (must be 'true' or 'false'). 106 | cygwin=false 107 | msys=false 108 | darwin=false 109 | nonstop=false 110 | case "$( uname )" in #( 111 | CYGWIN* ) cygwin=true ;; #( 112 | Darwin* ) darwin=true ;; #( 113 | MSYS* | MINGW* ) msys=true ;; #( 114 | NONSTOP* ) nonstop=true ;; 115 | esac 116 | 117 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 118 | 119 | 120 | # Determine the Java command to use to start the JVM. 121 | if [ -n "$JAVA_HOME" ] ; then 122 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 123 | # IBM's JDK on AIX uses strange locations for the executables 124 | JAVACMD=$JAVA_HOME/jre/sh/java 125 | else 126 | JAVACMD=$JAVA_HOME/bin/java 127 | fi 128 | if [ ! -x "$JAVACMD" ] ; then 129 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 130 | 131 | Please set the JAVA_HOME variable in your environment to match the 132 | location of your Java installation." 133 | fi 134 | else 135 | JAVACMD=java 136 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 137 | 138 | Please set the JAVA_HOME variable in your environment to match the 139 | location of your Java installation." 140 | fi 141 | 142 | # Increase the maximum file descriptors if we can. 143 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 144 | case $MAX_FD in #( 145 | max*) 146 | MAX_FD=$( ulimit -H -n ) || 147 | warn "Could not query maximum file descriptor limit" 148 | esac 149 | case $MAX_FD in #( 150 | '' | soft) :;; #( 151 | *) 152 | ulimit -n "$MAX_FD" || 153 | warn "Could not set maximum file descriptor limit to $MAX_FD" 154 | esac 155 | fi 156 | 157 | # Collect all arguments for the java command, stacking in reverse order: 158 | # * args from the command line 159 | # * the main class name 160 | # * -classpath 161 | # * -D...appname settings 162 | # * --module-path (only if needed) 163 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 164 | 165 | # For Cygwin or MSYS, switch paths to Windows format before running java 166 | if "$cygwin" || "$msys" ; then 167 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 168 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 169 | 170 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 171 | 172 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 173 | for arg do 174 | if 175 | case $arg in #( 176 | -*) false ;; # don't mess with options #( 177 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 178 | [ -e "$t" ] ;; #( 179 | *) false ;; 180 | esac 181 | then 182 | arg=$( cygpath --path --ignore --mixed "$arg" ) 183 | fi 184 | # Roll the args list around exactly as many times as the number of 185 | # args, so each arg winds up back in the position where it started, but 186 | # possibly modified. 187 | # 188 | # NB: a `for` loop captures its iteration list before it begins, so 189 | # changing the positional parameters here affects neither the number of 190 | # iterations, nor the values presented in `arg`. 191 | shift # remove old arg 192 | set -- "$@" "$arg" # push replacement arg 193 | done 194 | fi 195 | 196 | # Collect all arguments for the java command; 197 | # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of 198 | # shell script including quotes and variable substitutions, so put them in 199 | # double quotes to make sure that they get re-expanded; and 200 | # * put everything else in single quotes, so that it's not re-expanded. 201 | 202 | set -- \ 203 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 204 | -classpath "$CLASSPATH" \ 205 | org.gradle.wrapper.GradleWrapperMain \ 206 | "$@" 207 | 208 | # Use "xargs" to parse quoted args. 209 | # 210 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 211 | # 212 | # In Bash we could simply go: 213 | # 214 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 215 | # set -- "${ARGS[@]}" "$@" 216 | # 217 | # but POSIX shell has neither arrays nor command substitution, so instead we 218 | # post-process each arg (as a line of input to sed) to backslash-escape any 219 | # character that might be a shell metacharacter, then use eval to reverse 220 | # that process (while maintaining the separation between arguments), and wrap 221 | # the whole thing up as a single "set" statement. 222 | # 223 | # This will of course break if any of these variables contains a newline or 224 | # an unmatched quote. 225 | # 226 | 227 | eval "set -- $( 228 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 229 | xargs -n1 | 230 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 231 | tr '\n' ' ' 232 | )" '"$@"' 233 | 234 | exec "$JAVACMD" "$@" 235 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /settings.gradle.kts: -------------------------------------------------------------------------------- 1 | rootProject.name = "avro4k-kafka-serializer" -------------------------------------------------------------------------------- /src/integrationTest/kotlin/com/github/thake/kafka/avro4k/serializer/ConfluentCluster.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | import org.testcontainers.containers.GenericContainer 4 | import org.testcontainers.containers.KafkaContainer 5 | import org.testcontainers.containers.Network 6 | import org.testcontainers.utility.DockerImageName 7 | 8 | class ConfluentCluster(confluentVersion: String) { 9 | 10 | private val kafkaContainer = 11 | KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:$confluentVersion")) 12 | .withNetwork(Network.newNetwork()) 13 | .withNetworkAliases("kafka") 14 | .withEnv("KAFKA_HOST_NAME", "kafka") 15 | .apply { this.start() } 16 | private val schemaRegistry = 17 | GenericContainer("confluentinc/cp-schema-registry:$confluentVersion") 18 | .withNetwork(kafkaContainer.network) 19 | .withEnv("SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS", "PLAINTEXT://kafka:9092") 20 | .withEnv("SCHEMA_REGISTRY_HOST_NAME", "localhost") 21 | .withEnv("SCHEMA_REGISTRY_LISTENERS", "http://0.0.0.0:8081") 22 | .withExposedPorts(8081) 23 | .apply { 24 | this.start() 25 | } 26 | val schemaRegistryUrl = "http://${schemaRegistry.host}:${schemaRegistry.getMappedPort(8081)}" 27 | val bootstrapServers = kafkaContainer.bootstrapServers 28 | 29 | fun stop() { 30 | schemaRegistry.stop() 31 | kafkaContainer.stop() 32 | } 33 | } -------------------------------------------------------------------------------- /src/integrationTest/kotlin/com/github/thake/kafka/avro4k/serializer/ConfluentIT.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig 4 | import io.kotest.matchers.collections.shouldContainInOrder 5 | import kotlinx.serialization.Serializable 6 | import org.apache.kafka.clients.admin.Admin 7 | import org.apache.kafka.clients.admin.AdminClientConfig 8 | import org.apache.kafka.clients.admin.NewTopic 9 | import org.apache.kafka.clients.consumer.ConsumerConfig 10 | import org.apache.kafka.clients.consumer.KafkaConsumer 11 | import org.apache.kafka.clients.producer.KafkaProducer 12 | import org.apache.kafka.clients.producer.Producer 13 | import org.apache.kafka.clients.producer.ProducerConfig 14 | import org.apache.kafka.clients.producer.ProducerRecord 15 | import org.apache.kafka.streams.KafkaStreams 16 | import org.apache.kafka.streams.KeyValue 17 | import org.apache.kafka.streams.StreamsBuilder 18 | import org.apache.kafka.streams.StreamsConfig 19 | import org.junit.jupiter.params.ParameterizedTest 20 | import org.junit.jupiter.params.provider.ValueSource 21 | import java.time.Instant 22 | import java.util.* 23 | import kotlin.time.Duration 24 | import kotlin.time.Duration.Companion.milliseconds 25 | import kotlin.time.ExperimentalTime 26 | import kotlin.time.measureTimedValue 27 | import kotlin.time.toJavaDuration 28 | 29 | 30 | const val inputTopic = "input" 31 | const val outputTopic = "output" 32 | 33 | @Serializable 34 | data class Article( 35 | val title: String, 36 | val content: String 37 | ) 38 | 39 | class KafkaStreamsIT { 40 | 41 | @ParameterizedTest 42 | @ValueSource(strings = ["6.0.9", "6.1.7", "6.2.6", "7.0.5", "7.1.3", "7.2.1"]) 43 | fun testConfluentIntegration(confluentVersion: String) { 44 | val confluentCluster = ConfluentCluster(confluentVersion) 45 | val streamsConfiguration: Properties by lazy { 46 | val streamsConfiguration = Properties() 47 | streamsConfiguration[StreamsConfig.APPLICATION_ID_CONFIG] = "specific-avro-integration-test" 48 | streamsConfiguration[StreamsConfig.BOOTSTRAP_SERVERS_CONFIG] = 49 | confluentCluster.bootstrapServers 50 | streamsConfiguration[StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG] = Avro4kSerde::class.java 51 | streamsConfiguration[StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG] = Avro4kSerde::class.java 52 | streamsConfiguration[AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG] = 53 | confluentCluster.schemaRegistryUrl 54 | streamsConfiguration[KafkaAvro4kDeserializerConfig.RECORD_PACKAGES] = 55 | KafkaStreamsIT::class.java.packageName 56 | streamsConfiguration[ConsumerConfig.AUTO_OFFSET_RESET_CONFIG] = "earliest" 57 | streamsConfiguration 58 | } 59 | val producerConfig: Properties by lazy { 60 | val properties = Properties() 61 | properties[ProducerConfig.BOOTSTRAP_SERVERS_CONFIG] = confluentCluster.bootstrapServers 62 | properties[ProducerConfig.ACKS_CONFIG] = "all" 63 | properties[ProducerConfig.RETRIES_CONFIG] = 0 64 | properties[ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG] = KafkaAvro4kSerializer::class.java 65 | properties[ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG] = KafkaAvro4kSerializer::class.java 66 | properties[AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG] = confluentCluster.schemaRegistryUrl 67 | properties 68 | } 69 | val consumerConfig: Properties by lazy { 70 | val properties = Properties() 71 | properties[ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG] = confluentCluster.bootstrapServers 72 | properties[ConsumerConfig.GROUP_ID_CONFIG] = "kafka-streams-integration-test-standard-consumer" 73 | properties[ConsumerConfig.AUTO_OFFSET_RESET_CONFIG] = "earliest" 74 | properties[ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG] = KafkaAvro4kDeserializer::class.java 75 | properties[ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG] = KafkaAvro4kDeserializer::class.java 76 | properties[AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG] = confluentCluster.schemaRegistryUrl 77 | properties[KafkaAvro4kDeserializerConfig.RECORD_PACKAGES] = KafkaStreamsIT::class.java.packageName 78 | properties 79 | } 80 | val admin = Admin.create(mapOf(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG to confluentCluster.bootstrapServers)) 81 | //Wait for topic creations 82 | admin.createTopic(inputTopic) 83 | admin.createTopic(outputTopic) 84 | 85 | //Input values 86 | val staticInput = listOf( 87 | Article("Kafka Streams and Avro4k", "Just use avro4k-kafka-serializer"), 88 | Article("Lorem ipsum", "another content") 89 | ) 90 | //Now start kafka streams 91 | val streamsBuilder = StreamsBuilder() 92 | streamsBuilder.stream(inputTopic).to(outputTopic) 93 | val streams = KafkaStreams(streamsBuilder.build(), streamsConfiguration) 94 | streams.start() 95 | 96 | //Produce some input 97 | produceArticles(staticInput, producerConfig) 98 | 99 | //Now check output 100 | val values = readValues(consumerConfig) 101 | values.map { it.value }.shouldContainInOrder(staticInput) 102 | values.map { it.key }.shouldContainInOrder(staticInput.map { it.title }) 103 | 104 | //Close the stream after the test 105 | streams.close() 106 | 107 | confluentCluster.stop() 108 | } 109 | 110 | private fun Admin.createTopic(name: String) { 111 | createTopics(listOf(NewTopic(name, 1, 1))).all().get() 112 | } 113 | 114 | private fun produceArticles(articles: Collection
, producerConfig: Properties) { 115 | val producer: Producer = KafkaProducer(producerConfig) 116 | articles.forEach { article -> 117 | producer.send(ProducerRecord(inputTopic, null, Instant.now().toEpochMilli(), article.title, article)).get() 118 | } 119 | producer.flush() 120 | producer.close() 121 | } 122 | 123 | @OptIn(ExperimentalTime::class) 124 | private fun readValues(consumerConfig: Properties): List> { 125 | val consumer: KafkaConsumer = KafkaConsumer(consumerConfig) 126 | consumer.subscribe(listOf(outputTopic)) 127 | val pollInterval = 100.milliseconds.toJavaDuration() 128 | val maxTotalPollTime = 10000.milliseconds 129 | var totalPollTimeMs: Duration = 0.milliseconds 130 | val consumedValues: MutableList> = mutableListOf() 131 | 132 | while (totalPollTimeMs < maxTotalPollTime) { 133 | val timedValue = measureTimedValue { consumer.poll(pollInterval) } 134 | totalPollTimeMs += timedValue.duration 135 | for (record in timedValue.value) { 136 | consumedValues.add(KeyValue(record.key(), record.value())) 137 | } 138 | } 139 | consumer.close() 140 | return consumedValues 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /src/main/kotlin/com/github/thake/kafka/avro4k/serializer/AbstractKafkaAvro4kDeserializer.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | 4 | import com.github.avrokotlin.avro4k.Avro 5 | import com.github.avrokotlin.avro4k.io.AvroDecodeFormat 6 | import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException 7 | import kotlinx.serialization.InternalSerializationApi 8 | import kotlinx.serialization.serializer 9 | import org.apache.avro.Schema 10 | import org.apache.avro.generic.GenericDatumReader 11 | import org.apache.avro.io.BinaryDecoder 12 | import org.apache.avro.io.DecoderFactory 13 | import org.apache.kafka.common.errors.SerializationException 14 | import java.io.ByteArrayInputStream 15 | import java.io.IOException 16 | import java.io.InputStream 17 | import kotlin.reflect.KClass 18 | 19 | abstract class AbstractKafkaAvro4kDeserializer(private val avro: Avro) : AbstractKafkaAvro4kSerDe() { 20 | companion object { 21 | private var specificRecordLookupForClassLoader: MutableMap, ClassLoader>, RecordLookup> = 22 | mutableMapOf() 23 | 24 | private fun getLookup(recordPackages: List, classLoader: ClassLoader) = 25 | specificRecordLookupForClassLoader.getOrPut(Pair(recordPackages, classLoader), 26 | { RecordLookup(recordPackages, classLoader) }) 27 | } 28 | 29 | private var recordPackages: List = emptyList() 30 | private var binaryDecoder: BinaryDecoder? = null 31 | protected val avroSchemaUtils = Avro4kSchemaUtils(avro) 32 | 33 | 34 | protected fun configure(config: KafkaAvro4kDeserializerConfig) { 35 | val configuredPackages = config.getRecordPackages() 36 | if (configuredPackages.isEmpty()) { 37 | throw IllegalArgumentException("${KafkaAvro4kDeserializerConfig.RECORD_PACKAGES} is not set correctly.") 38 | } 39 | recordPackages = configuredPackages 40 | super.configure(config) 41 | } 42 | 43 | protected fun deserializerConfig(props: Map): KafkaAvro4kDeserializerConfig { 44 | return KafkaAvro4kDeserializerConfig(props) 45 | } 46 | 47 | 48 | @Throws(SerializationException::class) 49 | protected fun deserialize( 50 | payload: ByteArray?, readerSchema: Schema? 51 | ): Any? { 52 | 53 | return if (payload == null) { 54 | null 55 | } else { 56 | var id = -1 57 | try { 58 | val buffer = getByteBuffer(payload) 59 | id = buffer.int 60 | val writerSchema = getSchemaByIdWithRetry(id) 61 | ?: throw SerializationException("Could not find schema with id $id in schema registry") 62 | val length = buffer.limit() - 1 - 4 63 | val bytes = ByteArray(length) 64 | buffer[bytes, 0, length] 65 | return ByteArrayInputStream(bytes).use { 66 | deserialize(writerSchema, readerSchema, it) 67 | } 68 | } catch (re: RuntimeException) { 69 | throw SerializationException("Error deserializing Avro message for schema id $id with avro4k", re) 70 | } catch (io: IOException) { 71 | throw SerializationException("Error deserializing Avro message for schema id $id with avro4k", io) 72 | } catch (registry: RestClientException) { 73 | throw SerializationException("Error retrieving Avro schema for id $id from schema registry.", registry) 74 | } 75 | } 76 | } 77 | 78 | private fun deserializeUnion(writerSchema: Schema, readerSchema: Schema?, bytes: InputStream): Any? { 79 | val decoder = DecoderFactory.get().directBinaryDecoder(bytes, binaryDecoder) 80 | val unionTypeIndex = decoder.readInt() 81 | val recordSchema = writerSchema.types[unionTypeIndex] 82 | if (recordSchema.type == Schema.Type.NULL) return null 83 | binaryDecoder = decoder 84 | //Decode avro type as record 85 | return deserialize(recordSchema, readerSchema, decoder.inputStream()) 86 | } 87 | 88 | 89 | private fun deserialize(writerSchema: Schema, readerSchema: Schema?, bytes: InputStream) = 90 | when (writerSchema.type) { 91 | Schema.Type.BYTES -> bytes.readAllBytes() 92 | Schema.Type.UNION -> deserializeUnion(writerSchema, readerSchema, bytes) 93 | Schema.Type.RECORD -> deserializeRecord(writerSchema, readerSchema, bytes) 94 | else -> { 95 | val decoder = DecoderFactory.get().directBinaryDecoder(bytes, null) 96 | val datumReader = GenericDatumReader(writerSchema, readerSchema ?: writerSchema) 97 | val deserialized = datumReader.read(null, decoder) 98 | if (writerSchema.type == Schema.Type.STRING) { 99 | deserialized.toString() 100 | } else { 101 | deserialized 102 | } 103 | } 104 | } 105 | 106 | @OptIn(InternalSerializationApi::class) 107 | private fun deserializeRecord( 108 | writerSchema: Schema, 109 | readerSchema: Schema?, 110 | bytes: InputStream 111 | ): Any { 112 | val deserializedClass = getDeserializedClass(writerSchema) 113 | return avro.openInputStream(deserializedClass.serializer()) { 114 | decodeFormat = AvroDecodeFormat.Binary( 115 | writerSchema = writerSchema, 116 | readerSchema = readerSchema ?: avroSchemaUtils.getSchema(deserializedClass) 117 | ) 118 | }.from(bytes).nextOrThrow() 119 | } 120 | 121 | private fun getLookup(contextClassLoader: ClassLoader) = Companion.getLookup(recordPackages, contextClassLoader) 122 | 123 | protected open fun getDeserializedClass(msgSchema: Schema): KClass<*> { 124 | //First lookup using the context class loader 125 | val contextClassLoader = Thread.currentThread().contextClassLoader 126 | var objectClass: Class<*>? = null 127 | if (contextClassLoader != null) { 128 | objectClass = getLookup(contextClassLoader).lookupType(msgSchema) 129 | } 130 | if (objectClass == null) { 131 | //Fallback to classloader of this class 132 | objectClass = getLookup(AbstractKafkaAvro4kDeserializer::class.java.classLoader).lookupType(msgSchema) 133 | ?: throw SerializationException("Couldn't find matching class for record type ${msgSchema.fullName}. Full schema: $msgSchema") 134 | } 135 | 136 | return objectClass.kotlin 137 | } 138 | 139 | 140 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/github/thake/kafka/avro4k/serializer/AbstractKafkaAvro4kSerDe.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | import com.github.michaelbull.retry.ContinueRetrying 4 | import com.github.michaelbull.retry.StopRetrying 5 | import com.github.michaelbull.retry.context.retryStatus 6 | import com.github.michaelbull.retry.policy.RetryPolicy 7 | import com.github.michaelbull.retry.policy.fullJitterBackoff 8 | import com.github.michaelbull.retry.policy.limitAttempts 9 | import com.github.michaelbull.retry.policy.plus 10 | import com.github.michaelbull.retry.retry 11 | import io.confluent.kafka.schemaregistry.ParsedSchema 12 | import io.confluent.kafka.schemaregistry.avro.AvroSchema 13 | import io.confluent.kafka.schemaregistry.avro.AvroSchemaProvider 14 | import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException 15 | import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDe 16 | import kotlinx.coroutines.runBlocking 17 | import org.apache.avro.Schema 18 | import org.apache.kafka.common.errors.SerializationException 19 | import org.slf4j.LoggerFactory 20 | import kotlin.coroutines.coroutineContext 21 | 22 | private val logger = LoggerFactory.getLogger(AbstractKafkaAvro4kSerDe::class.java) 23 | 24 | abstract class AbstractKafkaAvro4kSerDe : AbstractKafkaSchemaSerDe() { 25 | var retryAttempts = AbstractKafkaAvro4kSerDeConfig.SCHEMA_REGISTRY_RETRY_ATTEMPTS_DEFAULT 26 | private set 27 | var retryJitterBase = AbstractKafkaAvro4kSerDeConfig.SCHEMA_REGISTRY_RETRY_JITTER_BASE_DEFAULT 28 | private set 29 | var retryJitterMax = AbstractKafkaAvro4kSerDeConfig.SCHEMA_REGISTRY_RETRY_JITTER_MAX_DEFAULT 30 | private set 31 | 32 | private val retryRestClientException: RetryPolicy = { 33 | if (reason is RestClientException) { 34 | val retryStatus = coroutineContext.retryStatus 35 | logger.warn( 36 | "Caught exception while trying to talk to the schema registry. Retry attempt ${retryStatus.attempt}", 37 | reason 38 | ) 39 | ContinueRetrying 40 | } else StopRetrying 41 | } 42 | var retryPolicy = calcRetryPolicy() 43 | private set 44 | 45 | private fun calcRetryPolicy(): RetryPolicy { 46 | return retryRestClientException + limitAttempts(retryAttempts) + fullJitterBackoff( 47 | base = retryJitterBase, 48 | max = retryJitterMax 49 | ) 50 | } 51 | 52 | fun configure(config: AbstractKafkaAvro4kSerDeConfig) { 53 | this.retryAttempts = config.schemaRegistryRetryAttempts 54 | this.retryJitterBase = config.schemaRegistryRetryJitterBase 55 | this.retryJitterMax = config.schemaRegistryRetryJitterMax 56 | this.retryPolicy = calcRetryPolicy() 57 | this.configureClientProperties(config, AvroSchemaProvider()) 58 | } 59 | 60 | private fun doCallToSchemaRegistry( 61 | errMsgProvider: (e: RestClientException) -> String = { "Error calling schema registry" }, 62 | block: () -> T 63 | ): T { 64 | return try { 65 | runBlocking { 66 | retry(retryPolicy) { 67 | block.invoke() 68 | } 69 | } 70 | } catch (e: RestClientException) { 71 | throw SerializationException(errMsgProvider.invoke(e), e) 72 | } 73 | } 74 | 75 | fun registerWithRetry(subject: String?, schema: Schema?): Int { 76 | return doCallToSchemaRegistry({ "Error registering Avro schema in schema registry for subject '$subject': $schema" }) { 77 | register(subject, AvroSchema(schema)) 78 | } 79 | } 80 | 81 | fun getSchemaIdWithRetry(subject: String?, schema: Schema?): Int { 82 | return doCallToSchemaRegistry({ "Error retrieving Avro schema id from schema registry for subject '$subject' and schema: $schema" }) { 83 | schemaRegistry.getId(subject, AvroSchema(schema) as ParsedSchema?) 84 | } 85 | } 86 | 87 | fun getSchemaByIdWithRetry(id: Int): Schema? { 88 | return doCallToSchemaRegistry({ "Error retrieving Avro schema by id $id from schema registry." }) { 89 | (schemaRegistry.getSchemaById(id) as? AvroSchema)?.rawSchema() 90 | } 91 | } 92 | 93 | 94 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/github/thake/kafka/avro4k/serializer/AbstractKafkaAvro4kSerDeConfig.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | 4 | import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig 5 | import org.apache.kafka.common.config.ConfigDef 6 | 7 | abstract class AbstractKafkaAvro4kSerDeConfig(configDef: ConfigDef, props: Map) : 8 | AbstractKafkaSchemaSerDeConfig(configDef, props) { 9 | companion object { 10 | const val SCHEMA_REGISTRY_RETRY_ATTEMPTS_CONFIG = "schema.registry.retry.attempts" 11 | const val SCHEMA_REGISTRY_RETRY_ATTEMPTS_DEFAULT = 5 12 | const val SCHEMA_REGISTRY_RETRY_ATTEMPTS_DOC = 13 | "Number of retry attempts that will be made if the schema registry seems to have a problem with requesting a schema." 14 | const val SCHEMA_REGISTRY_RETRY_JITTER_BASE_CONFIG = "schema.registry.retry.jitter.base" 15 | const val SCHEMA_REGISTRY_RETRY_JITTER_BASE_DOC = 16 | "Milliseconds that are used as a base for the jitter calculation (sleep = random_between(0, min(max, base * 2 ** attempt)))" 17 | const val SCHEMA_REGISTRY_RETRY_JITTER_BASE_DEFAULT = 10L 18 | const val SCHEMA_REGISTRY_RETRY_JITTER_MAX_CONFIG = "schema.registry.retry.jitter.max" 19 | const val SCHEMA_REGISTRY_RETRY_JITTER_MAX_DOC = 20 | "Milliseconds that are used as max for the jitter calculation (sleep = random_between(0, min(max, base * 2 ** attempt)))" 21 | const val SCHEMA_REGISTRY_RETRY_JITTER_MAX_DEFAULT = 5000L 22 | 23 | fun baseConfigDef(): ConfigDef = AbstractKafkaSchemaSerDeConfig.baseConfigDef() 24 | .define( 25 | SCHEMA_REGISTRY_RETRY_ATTEMPTS_CONFIG, 26 | ConfigDef.Type.INT, 27 | SCHEMA_REGISTRY_RETRY_ATTEMPTS_DEFAULT, 28 | ConfigDef.Importance.LOW, 29 | SCHEMA_REGISTRY_RETRY_ATTEMPTS_DOC 30 | ) 31 | .define( 32 | SCHEMA_REGISTRY_RETRY_JITTER_BASE_CONFIG, 33 | ConfigDef.Type.LONG, 34 | SCHEMA_REGISTRY_RETRY_JITTER_BASE_DEFAULT, 35 | ConfigDef.Importance.LOW, 36 | SCHEMA_REGISTRY_RETRY_JITTER_BASE_DOC 37 | ) 38 | .define( 39 | SCHEMA_REGISTRY_RETRY_JITTER_MAX_CONFIG, 40 | ConfigDef.Type.LONG, 41 | SCHEMA_REGISTRY_RETRY_JITTER_MAX_DEFAULT, 42 | ConfigDef.Importance.LOW, 43 | SCHEMA_REGISTRY_RETRY_JITTER_MAX_DOC 44 | ) 45 | } 46 | 47 | val schemaRegistryRetryAttempts: Int 48 | get() = this.get(SCHEMA_REGISTRY_RETRY_ATTEMPTS_CONFIG) as Int 49 | val schemaRegistryRetryJitterBase: Long 50 | get() = this.get(SCHEMA_REGISTRY_RETRY_JITTER_BASE_CONFIG) as Long 51 | val schemaRegistryRetryJitterMax: Long 52 | get() = this.get(SCHEMA_REGISTRY_RETRY_JITTER_MAX_CONFIG) as Long 53 | } 54 | -------------------------------------------------------------------------------- /src/main/kotlin/com/github/thake/kafka/avro4k/serializer/AbstractKafkaAvro4kSerializer.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | 4 | import com.github.avrokotlin.avro4k.Avro 5 | import com.github.avrokotlin.avro4k.io.AvroEncodeFormat 6 | import io.confluent.kafka.serializers.NonRecordContainer 7 | import kotlinx.serialization.InternalSerializationApi 8 | import kotlinx.serialization.KSerializer 9 | import kotlinx.serialization.serializer 10 | import org.apache.avro.Schema 11 | import org.apache.avro.generic.GenericDatumWriter 12 | import org.apache.avro.io.EncoderFactory 13 | import org.apache.kafka.common.errors.SerializationException 14 | import java.io.ByteArrayOutputStream 15 | import java.io.IOException 16 | import java.nio.ByteBuffer 17 | 18 | abstract class AbstractKafkaAvro4kSerializer(private val avro: Avro) : AbstractKafkaAvro4kSerDe() { 19 | private var autoRegisterSchema = false 20 | 21 | protected val avroSchemaUtils = Avro4kSchemaUtils(avro) 22 | protected fun configure(config: KafkaAvro4kSerializerConfig) { 23 | autoRegisterSchema = config.autoRegisterSchema() 24 | super.configure(config) 25 | } 26 | 27 | protected fun serializerConfig(props: Map): KafkaAvro4kSerializerConfig { 28 | return KafkaAvro4kSerializerConfig(props) 29 | } 30 | 31 | @Throws(SerializationException::class) 32 | protected fun serializeImpl(subject: String?, obj: Any?): ByteArray? { 33 | return if (obj == null) { 34 | null 35 | } else { 36 | try { 37 | val currentSchema = avroSchemaUtils.getSchema(obj) 38 | val id = getSchemaId(subject, currentSchema) 39 | val out = ByteArrayOutputStream() 40 | writeSchemaId(out, id) 41 | if (obj is ByteArray) { 42 | out.write(obj) 43 | } else { 44 | serializeValue(out, obj, currentSchema) 45 | } 46 | val bytes = out.toByteArray() 47 | out.close() 48 | bytes 49 | } catch (re: RuntimeException) { 50 | throw SerializationException("Error serializing Avro message with avro4k", re) 51 | } catch (io: IOException) { 52 | throw SerializationException("Error serializing Avro message with avro4k", io) 53 | } 54 | } 55 | } 56 | 57 | @OptIn(InternalSerializationApi::class) 58 | fun serializeValue(out: ByteArrayOutputStream, obj: Any, currentSchema: Schema) { 59 | val value = 60 | if (obj is NonRecordContainer) obj.value else obj 61 | if (currentSchema.type == Schema.Type.RECORD) { 62 | @Suppress("UNCHECKED_CAST") 63 | avro.openOutputStream(value::class.serializer() as KSerializer) { 64 | encodeFormat = AvroEncodeFormat.Binary 65 | schema = currentSchema 66 | }.to(out).write(obj).close() 67 | } else { 68 | val encoder = EncoderFactory.get().directBinaryEncoder(out, null) 69 | val datumWriter = GenericDatumWriter(currentSchema) 70 | datumWriter.write(obj, encoder) 71 | encoder.flush() 72 | } 73 | } 74 | 75 | private fun writeSchemaId(out: ByteArrayOutputStream, id: Int) { 76 | out.write(0) 77 | out.write(ByteBuffer.allocate(4).putInt(id).array()) 78 | } 79 | 80 | private fun getSchemaId( 81 | subject: String?, 82 | schema: Schema? 83 | ): Int { 84 | return if (autoRegisterSchema) { 85 | registerWithRetry(subject, schema) 86 | } else { 87 | getSchemaIdWithRetry(subject, schema) 88 | } 89 | } 90 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/github/thake/kafka/avro4k/serializer/Avro4kSchemaUtils.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | import com.github.avrokotlin.avro4k.Avro 4 | import kotlinx.serialization.InternalSerializationApi 5 | import kotlinx.serialization.serializer 6 | import org.apache.avro.Schema 7 | import java.util.concurrent.ConcurrentHashMap 8 | import kotlin.reflect.KClass 9 | import kotlin.reflect.full.isSuperclassOf 10 | 11 | 12 | class Avro4kSchemaUtils(private val avro: Avro) { 13 | private val NULL_SCHEMA = Schema.create(Schema.Type.NULL) 14 | private val BOOLEAN_SCHEMA = Schema.create(Schema.Type.BOOLEAN) 15 | private val INTEGER_SCHEMA = Schema.create(Schema.Type.INT) 16 | private val LONG_SCHEMA = Schema.create(Schema.Type.LONG) 17 | private val FLOAT_SCHEMA = Schema.create(Schema.Type.FLOAT) 18 | private val DOUBLE_SCHEMA = Schema.create(Schema.Type.DOUBLE) 19 | private val STRING_SCHEMA = Schema.create(Schema.Type.STRING) 20 | private val BYTES_SCHEMA = Schema.create(Schema.Type.BYTES) 21 | private val cachedSchemas = ConcurrentHashMap, Schema>() 22 | 23 | 24 | @OptIn(InternalSerializationApi::class) 25 | fun getSchema(clazz: KClass<*>): Schema { 26 | return when { 27 | Boolean::class.isSuperclassOf(clazz) -> BOOLEAN_SCHEMA 28 | Int::class.isSuperclassOf(clazz) -> INTEGER_SCHEMA 29 | Long::class.isSuperclassOf(clazz) -> LONG_SCHEMA 30 | Float::class.isSuperclassOf(clazz) -> FLOAT_SCHEMA 31 | Double::class.isSuperclassOf(clazz) -> DOUBLE_SCHEMA 32 | CharSequence::class.isSuperclassOf(clazz) -> STRING_SCHEMA 33 | ByteArray::class.isSuperclassOf(clazz) -> BYTES_SCHEMA 34 | else -> cachedSchemas.computeIfAbsent(clazz) { avro.schema(it.serializer()) } 35 | } 36 | } 37 | 38 | 39 | fun getSchema(obj: Any?): Schema { 40 | return if (obj == null) NULL_SCHEMA else getSchema(obj::class) 41 | } 42 | 43 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/github/thake/kafka/avro4k/serializer/Avro4kSerde.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | import com.github.avrokotlin.avro4k.Avro 4 | import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient 5 | import org.apache.kafka.common.serialization.Deserializer 6 | import org.apache.kafka.common.serialization.Serde 7 | import org.apache.kafka.common.serialization.Serdes 8 | import org.apache.kafka.common.serialization.Serializer 9 | 10 | class Avro4kSerde(client: SchemaRegistryClient? = null, avro: Avro = Avro.default) : Serde { 11 | @Suppress("UNCHECKED_CAST") 12 | private val inner: Serde = Serdes.serdeFrom( 13 | KafkaAvro4kSerializer(client, avro = avro) as Serializer, 14 | KafkaAvro4kDeserializer(client, avro = avro) as Deserializer 15 | ) 16 | 17 | 18 | override fun serializer(): Serializer { 19 | return inner.serializer() 20 | } 21 | 22 | override fun deserializer(): Deserializer { 23 | return inner.deserializer() 24 | } 25 | 26 | override fun configure(configs: Map?, isSerdeForRecordKeys: Boolean) { 27 | inner.serializer().configure(configs, isSerdeForRecordKeys) 28 | inner.deserializer().configure(configs, isSerdeForRecordKeys) 29 | } 30 | 31 | override fun close() { 32 | inner.serializer().close() 33 | inner.deserializer().close() 34 | } 35 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/github/thake/kafka/avro4k/serializer/ClassExtensions.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | 4 | import com.github.avrokotlin.avro4k.AnnotationExtractor 5 | import com.github.avrokotlin.avro4k.RecordNaming 6 | import kotlinx.serialization.ExperimentalSerializationApi 7 | import kotlinx.serialization.InternalSerializationApi 8 | import kotlinx.serialization.serializer 9 | 10 | /** 11 | * A list containing all avro record names that represent this class. 12 | */ 13 | @OptIn(ExperimentalSerializationApi::class, InternalSerializationApi::class) 14 | val Class<*>.avroRecordNames: List 15 | get() { 16 | val descriptor = this.kotlin.serializer().descriptor 17 | val naming = RecordNaming(descriptor) 18 | val aliases = AnnotationExtractor(descriptor.annotations).aliases() 19 | val normalNameMapping = "${naming.namespace}.${naming.name}" 20 | return if (aliases.isNotEmpty()) { 21 | val mappings = mutableListOf(normalNameMapping) 22 | aliases.forEach { alias -> 23 | mappings.add( 24 | if (alias.contains('.')) { 25 | alias 26 | } else { 27 | "${naming.namespace}.$alias" 28 | } 29 | ) 30 | } 31 | mappings 32 | } else { 33 | listOf(normalNameMapping) 34 | } 35 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/github/thake/kafka/avro4k/serializer/KafkaAvro4kDeserializer.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | import com.github.avrokotlin.avro4k.Avro 4 | import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient 5 | import org.apache.avro.Schema 6 | import org.apache.kafka.common.serialization.Deserializer 7 | 8 | class KafkaAvro4kDeserializer( 9 | client: SchemaRegistryClient? = null, 10 | props: Map? = null, 11 | avro: Avro = Avro.default 12 | ) : AbstractKafkaAvro4kDeserializer(avro), Deserializer { 13 | private var isKey = false 14 | 15 | init { 16 | props?.let { configure(this.deserializerConfig(it)) } 17 | //Set the registry client explicitly after the configuration has been applied to override client from configuration 18 | if (client != null) this.schemaRegistry = client 19 | } 20 | 21 | override fun configure(configs: Map, isKey: Boolean) { 22 | this.isKey = isKey 23 | this.configure(KafkaAvro4kDeserializerConfig(configs)) 24 | } 25 | 26 | 27 | override fun deserialize(s: String?, bytes: ByteArray?): Any? { 28 | return this.deserialize(s, bytes, null) 29 | } 30 | 31 | fun deserialize(@Suppress("UNUSED_PARAMETER") topic: String?, data: ByteArray?, readerSchema: Schema?): Any? { 32 | return this.deserialize(data, readerSchema) 33 | } 34 | 35 | override fun close() {} 36 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/github/thake/kafka/avro4k/serializer/KafkaAvro4kDeserializerConfig.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | import org.apache.kafka.common.config.ConfigDef 4 | 5 | 6 | class KafkaAvro4kDeserializerConfig(props: Map) : AbstractKafkaAvro4kSerDeConfig(baseConfigDef(), props) { 7 | fun getRecordPackages() = getString(RECORD_PACKAGES)?.split(",")?.map { it.trim() }?.toList() ?: emptyList() 8 | 9 | companion object { 10 | const val RECORD_PACKAGES = "record.packages" 11 | fun baseConfigDef() = AbstractKafkaAvro4kSerDeConfig.baseConfigDef().define( 12 | RECORD_PACKAGES, 13 | ConfigDef.Type.STRING, 14 | null, 15 | ConfigDef.Importance.HIGH, 16 | "The packages in which record types annotated with @AvroName, @AvroAlias and @AvroNamespace can be found. Packages are separated by a comma ','." 17 | ) 18 | } 19 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/github/thake/kafka/avro4k/serializer/KafkaAvro4kSerializer.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | import com.github.avrokotlin.avro4k.Avro 4 | import io.confluent.kafka.schemaregistry.avro.AvroSchema 5 | import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient 6 | import org.apache.kafka.common.serialization.Serializer 7 | 8 | class KafkaAvro4kSerializer( 9 | client : SchemaRegistryClient? = null, 10 | props : Map? = null, 11 | avro: Avro = Avro.default 12 | ) : AbstractKafkaAvro4kSerializer(avro), Serializer { 13 | private var isKey = false 14 | 15 | init { 16 | props?.let { configure(this.serializerConfig(it)) } 17 | //Set the registry client explicitly after configuration has been applied to override client from configuration 18 | if (client != null) this.schemaRegistry = client 19 | } 20 | 21 | override fun configure(configs: Map, isKey: Boolean) { 22 | this.isKey = isKey 23 | this.configure(KafkaAvro4kSerializerConfig(configs)) 24 | } 25 | 26 | 27 | override fun serialize(topic: String?, record: Any?): ByteArray? { 28 | return record?.let { 29 | this.serializeImpl( 30 | this.getSubjectName( 31 | topic, 32 | isKey, 33 | it, 34 | AvroSchema(avroSchemaUtils.getSchema(it)) 35 | ), it 36 | ) 37 | } 38 | } 39 | 40 | override fun close() {} 41 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/github/thake/kafka/avro4k/serializer/KafkaAvro4kSerializerConfig.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | class KafkaAvro4kSerializerConfig(props: Map) : AbstractKafkaAvro4kSerDeConfig(baseConfigDef(), props) { 4 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/github/thake/kafka/avro4k/serializer/RecordLookup.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | 4 | import com.github.avrokotlin.avro4k.AvroAlias 5 | import com.github.avrokotlin.avro4k.AvroAliases 6 | import com.github.avrokotlin.avro4k.AvroName 7 | import com.github.avrokotlin.avro4k.AvroNamespace 8 | import io.github.classgraph.ClassGraph 9 | import org.apache.avro.Schema 10 | import org.apache.avro.specific.SpecificData 11 | 12 | class RecordLookup( 13 | recordPackages: List, 14 | private val classLoader: ClassLoader 15 | ) { 16 | private val specificData = SpecificData(classLoader) 17 | private val recordNameToType: Map> by lazy { 18 | if (recordPackages.isNotEmpty()) { 19 | val annotationNames = arrayOf( 20 | AvroName::class, 21 | AvroNamespace::class, 22 | AvroAlias::class, 23 | AvroAliases::class 24 | ).map { it.java.name } 25 | val avroTypes = ClassGraph().enableClassInfo().enableAnnotationInfo().ignoreClassVisibility() 26 | .acceptClasses(*annotationNames.toTypedArray()) 27 | .acceptPackages(*recordPackages.toTypedArray()) 28 | .addClassLoader(classLoader) 29 | .scan().use { scanResult -> 30 | annotationNames 31 | .flatMap { scanResult.getClassesWithAnnotation(it) } 32 | .map { it.loadClass() } 33 | .toSet() 34 | } 35 | avroTypes.flatMap { type -> 36 | type.avroRecordNames.map { Pair(it, type) } 37 | }.toMap() 38 | } else { 39 | emptyMap() 40 | } 41 | } 42 | 43 | fun lookupType(msgSchema: Schema): Class<*>? { 44 | //Use the context classloader to not run into https://github.com/spring-projects/spring-boot/issues/14622 when 45 | //using Spring boot devtools 46 | var objectClass: Class<*>? = specificData.getClass(msgSchema) 47 | if (objectClass == null) { 48 | objectClass = recordNameToType[msgSchema.fullName] 49 | } 50 | return objectClass 51 | } 52 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/github/thake/kafka/avro4k/serializer/TypedKafkaAvro4kDeserializer.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | import com.github.avrokotlin.avro4k.Avro 4 | import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient 5 | import org.apache.avro.Schema 6 | import org.apache.kafka.common.errors.SerializationException 7 | import org.apache.kafka.common.serialization.Deserializer 8 | import kotlin.jvm.internal.Reflection 9 | import kotlin.reflect.KClass 10 | 11 | class TypedKafkaAvro4kDeserializer( 12 | private val type: Class, client : SchemaRegistryClient? = null, 13 | avro: Avro = Avro.default 14 | ) : AbstractKafkaAvro4kDeserializer(avro), Deserializer { 15 | private val typeNames = type.avroRecordNames 16 | init { 17 | this.schemaRegistry = client 18 | } 19 | override fun getDeserializedClass(msgSchema: Schema): KClass<*> { 20 | return if (typeNames.contains(msgSchema.fullName)) { 21 | Reflection.getOrCreateKotlinClass(type) 22 | } else { 23 | throw SerializationException("Could not convert to type $type with schema record name ${msgSchema.fullName}") 24 | } 25 | } 26 | 27 | override fun deserialize(topic: String?, data: ByteArray?): T? { 28 | @Suppress("UNCHECKED_CAST") 29 | return deserialize(data, avroSchemaUtils.getSchema(type)) as T? 30 | } 31 | } -------------------------------------------------------------------------------- /src/test/kotlin/com/github/thake/kafka/avro4k/serializer/Avro4kSerdeTest.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | 4 | import com.github.avrokotlin.avro4k.AvroName 5 | import com.github.avrokotlin.avro4k.AvroNamespace 6 | import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig 7 | import io.kotest.matchers.types.shouldBeInstanceOf 8 | import kotlinx.serialization.Serializable 9 | import org.junit.jupiter.api.Assertions 10 | import org.junit.jupiter.api.Assertions.assertEquals 11 | import org.junit.jupiter.params.ParameterizedTest 12 | import org.junit.jupiter.params.provider.MethodSource 13 | import java.util.stream.Stream 14 | 15 | class Avro4kSerdeTest { 16 | @Serializable 17 | private data class TestRecord( 18 | val str : String 19 | ) 20 | @Serializable 21 | private data class TestRecordWithNull( 22 | val nullableStr : String? = null, 23 | val intValue : Int 24 | ) 25 | @Serializable 26 | @AvroNamespace("custom.namespace.serde") 27 | private data class TestRecordWithNamespace( 28 | val float : Float 29 | ) 30 | @Serializable 31 | @AvroName("AnotherName") 32 | private data class TestRecordWithDifferentName( 33 | val double : Double 34 | ) 35 | 36 | companion object{ 37 | @JvmStatic 38 | fun createSerializableObjects(): Stream { 39 | return Stream.of( 40 | TestRecord("STTR"), 41 | TestRecordWithNull(null, 2), 42 | TestRecordWithNull("33", 1), 43 | TestRecordWithNamespace(4f), 44 | TestRecordWithDifferentName(2.0), 45 | 1, 46 | 2.0, 47 | "STR", 48 | true, 49 | 2.0f, 50 | 1L, 51 | byteArrayOf(0xC, 0xA, 0xF, 0xE) 52 | ) 53 | } 54 | } 55 | 56 | @ParameterizedTest() 57 | @MethodSource("createSerializableObjects") 58 | fun testRecordSerDeRoundtrip(toSerialize: Any) { 59 | val config = mapOf( 60 | KafkaAvro4kDeserializerConfig.RECORD_PACKAGES to this::class.java.packageName, 61 | AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG to "mock://registry" 62 | ) 63 | val serde = Avro4kSerde() 64 | val topic = "My-Topic" 65 | serde.configure(config, false) 66 | val result = serde.serializer().serialize(topic, toSerialize) 67 | 68 | Assertions.assertNotNull(result) 69 | result ?: throw Exception("") 70 | 71 | val deserializedValue = serde.deserializer().deserialize(topic, result) 72 | if (toSerialize is ByteArray) { 73 | deserializedValue.shouldBeInstanceOf() 74 | toSerialize.forEachIndexed { i, value -> 75 | assertEquals(value, deserializedValue[i]) 76 | } 77 | } else { 78 | assertEquals(toSerialize, deserializedValue) 79 | } 80 | } 81 | } -------------------------------------------------------------------------------- /src/test/kotlin/com/github/thake/kafka/avro4k/serializer/ClassloaderTest.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | 4 | import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig 5 | import kotlinx.serialization.Serializable 6 | import org.junit.jupiter.api.Assertions.assertEquals 7 | import org.junit.jupiter.api.Assertions.assertNotNull 8 | import org.junit.jupiter.api.Test 9 | import java.io.ByteArrayOutputStream 10 | import java.io.File 11 | import java.io.IOException 12 | import java.util.concurrent.CountDownLatch 13 | 14 | @Serializable 15 | data class SimpleTest( 16 | val str: String 17 | ) 18 | 19 | class ClassloaderTest { 20 | private val config: Map = mapOf( 21 | KafkaAvro4kDeserializerConfig.RECORD_PACKAGES to this::class.java.packageName, 22 | AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG to "mock://registry" 23 | ) 24 | private val serializer = KafkaAvro4kSerializer(null, config).apply { configure(config, false) } 25 | private val deserializer = KafkaAvro4kDeserializer(null, config).apply { configure(config, false) } 26 | 27 | @Test 28 | fun deserializeWithDifferentClassloader() { 29 | val byteArray = serializer.serialize("A", SimpleTest("AAA")) 30 | val newClassLoader = object : ClassLoader() { 31 | 32 | override fun loadClass(name: String): Class<*> { 33 | return if (name.contains(SimpleTest::class.java.name)) findClass(name) 34 | else super.loadClass(name) 35 | } 36 | 37 | @Throws(ClassNotFoundException::class) 38 | override fun findClass(name: String): Class<*> { 39 | return if (name.contains(SimpleTest::class.java.name)) { 40 | val b = loadClassFromFile(name) 41 | defineClass(name, b, 0, b.size, null) 42 | } else { 43 | super.findClass(name) 44 | } 45 | } 46 | 47 | private fun loadClassFromFile(fileName: String): ByteArray { 48 | val inputStream = javaClass.classLoader 49 | .getResourceAsStream(fileName.replace('.', File.separatorChar) + ".class") 50 | ?: error("Couldn't load $fileName as Stream") 51 | val buffer: ByteArray 52 | val byteStream = ByteArrayOutputStream() 53 | var nextValue: Int 54 | try { 55 | while (inputStream.read().also { nextValue = it } != -1) { 56 | byteStream.write(nextValue) 57 | } 58 | } catch (e: IOException) { 59 | e.printStackTrace() 60 | } 61 | buffer = byteStream.toByteArray() 62 | return buffer 63 | } 64 | } 65 | val countDown = CountDownLatch(1) 66 | var result: Any? = null 67 | val testThread = Thread { 68 | try { 69 | assertEquals(newClassLoader, Thread.currentThread().contextClassLoader) 70 | result = deserializer.deserialize("s", byteArray) 71 | } finally { 72 | countDown.countDown() 73 | } 74 | } 75 | testThread.contextClassLoader = newClassLoader 76 | testThread.start() 77 | countDown.await() 78 | 79 | assertNotNull(result) 80 | assertEquals(newClassLoader, result!!.javaClass.classLoader) 81 | } 82 | 83 | @Test 84 | fun deserializeWithoutContextClassloader() { 85 | val byteArray = serializer.serialize("A", SimpleTest("AAA")) 86 | val countDown = CountDownLatch(1) 87 | var result: Any? = null 88 | val testThread = Thread { 89 | result = deserializer.deserialize("s", byteArray) 90 | countDown.countDown() 91 | } 92 | testThread.contextClassLoader = null 93 | testThread.start() 94 | countDown.await() 95 | 96 | assertNotNull(result) 97 | assertEquals(ClassloaderTest::class.java.classLoader, result!!.javaClass.classLoader) 98 | } 99 | 100 | 101 | @Test 102 | fun deserializeWithNormalClassloader() { 103 | val byteArray = serializer.serialize("A", SimpleTest("AAA")) 104 | val result = deserializer.deserialize("s", byteArray) 105 | 106 | assertNotNull(result) 107 | assertEquals(ClassloaderTest::class.java.classLoader, result!!.javaClass.classLoader) 108 | } 109 | } -------------------------------------------------------------------------------- /src/test/kotlin/com/github/thake/kafka/avro4k/serializer/KafkaAvro4kDeserializerTest.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | 4 | import com.github.avrokotlin.avro4k.Avro 5 | import io.confluent.kafka.schemaregistry.avro.AvroSchema 6 | import io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient 7 | import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig 8 | import io.kotest.matchers.shouldBe 9 | import io.mockk.every 10 | import io.mockk.spyk 11 | import kotlinx.serialization.Serializable 12 | import org.apache.avro.Schema 13 | import org.apache.avro.generic.GenericDatumWriter 14 | import org.apache.avro.generic.GenericRecord 15 | import org.apache.avro.generic.GenericRecordBuilder 16 | import org.apache.avro.io.EncoderFactory 17 | import org.junit.jupiter.api.Assertions 18 | import org.junit.jupiter.api.Test 19 | import java.io.ByteArrayOutputStream 20 | import java.nio.ByteBuffer 21 | 22 | class KafkaAvro4kDeserializerTest { 23 | private val registryMock = spyk(MockSchemaRegistryClient()) 24 | 25 | @Serializable 26 | private data class DeserializerTestRecord( 27 | val str: String 28 | ) 29 | 30 | @Test 31 | fun testNullRecordInUnion() { 32 | val deserializer = KafkaAvro4kDeserializer(registryMock) 33 | deserializer.configure( 34 | mapOf( 35 | AbstractKafkaSchemaSerDeConfig.AUTO_REGISTER_SCHEMAS to "true", 36 | AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG to "mock://registry", 37 | KafkaAvro4kDeserializerConfig.RECORD_PACKAGES to this::class.java.packageName, 38 | ), 39 | false 40 | ) 41 | val avroSchema = Avro.Companion.default.schema(DeserializerTestRecord.serializer()) 42 | val unionSchema = Schema.createUnion(Schema.create(Schema.Type.NULL), avroSchema) 43 | val schemaId = 1 44 | val buffer = ByteBuffer.allocate(8) 45 | buffer.put(0x00) //Magic byte as in AbstractKafkaSchemaSerDe.MAGIC_BYTE 46 | buffer.putInt(schemaId) //Schema ID 47 | buffer.put(0x00) //Actual record. 0x00 indicating the first type of the union. This is "NULL". 48 | val byteArray: ByteArray = buffer.array() 49 | every { 50 | registryMock.getSchemaById(schemaId) 51 | }.returns(AvroSchema(unionSchema)) 52 | val topic = "My-Topic" 53 | val result = deserializer.deserialize(topic, byteArray) 54 | Assertions.assertNull(result) 55 | } 56 | 57 | @Test 58 | fun testNonNullRecordInUnion() { 59 | val deserializer = KafkaAvro4kDeserializer(registryMock) 60 | deserializer.configure( 61 | mapOf( 62 | AbstractKafkaSchemaSerDeConfig.AUTO_REGISTER_SCHEMAS to "true", 63 | AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG to "mock://registry", 64 | KafkaAvro4kDeserializerConfig.RECORD_PACKAGES to this::class.java.packageName, 65 | ), 66 | false 67 | ) 68 | val avroSchema = Avro.Companion.default.schema(DeserializerTestRecord.serializer()) 69 | val unionSchema = Schema.createUnion(Schema.create(Schema.Type.NULL), avroSchema) 70 | val topic = "My-Topic" 71 | val subject = "$topic-value" 72 | val schemaId = registryMock.register(subject, AvroSchema(unionSchema)) 73 | 74 | val outputStream = ByteArrayOutputStream() 75 | //Avro Kafka format 76 | outputStream.write(0x00)//Magic byte as in AbstractKafkaSchemaSerDe.MAGIC_BYTE 77 | outputStream.write(ByteBuffer.allocate(4).putInt(schemaId).array()) //Schema ID encoded as normal Int 78 | //Avro data starts 79 | val serializedRecord = DeserializerTestRecord("test") 80 | val record = 81 | GenericRecordBuilder(avroSchema).set(DeserializerTestRecord::str.name, serializedRecord.str).build() 82 | val datumWriter = GenericDatumWriter(unionSchema) 83 | val encoder = EncoderFactory.get().directBinaryEncoder(outputStream, null) 84 | datumWriter.write(record, encoder) 85 | val byteArray: ByteArray = outputStream.toByteArray() 86 | val result = deserializer.deserialize(topic, byteArray) 87 | result.shouldBe(serializedRecord) 88 | } 89 | } -------------------------------------------------------------------------------- /src/test/kotlin/com/github/thake/kafka/avro4k/serializer/KafkaAvro4kSerializerTest.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | 4 | import com.github.avrokotlin.avro4k.Avro 5 | import com.github.avrokotlin.avro4k.AvroName 6 | import com.github.avrokotlin.avro4k.AvroNamespace 7 | import io.confluent.kafka.schemaregistry.ParsedSchema 8 | import io.confluent.kafka.schemaregistry.avro.AvroSchema 9 | import io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient 10 | import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig 11 | import io.kotest.matchers.shouldBe 12 | import io.mockk.spyk 13 | import io.mockk.verify 14 | import kotlinx.serialization.Serializable 15 | import org.apache.avro.Schema 16 | import org.junit.jupiter.api.Assertions.assertEquals 17 | import org.junit.jupiter.api.Assertions.assertNotNull 18 | import org.junit.jupiter.api.Test 19 | import org.junit.jupiter.params.ParameterizedTest 20 | import org.junit.jupiter.params.provider.MethodSource 21 | import java.util.stream.Stream 22 | 23 | class KafkaAvro4kSerializerTest { 24 | private val registryMock = spyk(MockSchemaRegistryClient()) 25 | @Serializable 26 | private data class TestRecord( 27 | val str : String 28 | ) 29 | @Serializable 30 | private data class TestRecordWithNull( 31 | val nullableStr : String? = null, 32 | val intValue : Int 33 | ) 34 | @Serializable 35 | @AvroNamespace("custom.namespace") 36 | private data class TestRecordWithNamespace( 37 | val float : Double 38 | ) 39 | @Serializable 40 | @AvroName("AnotherName") 41 | private data class TestRecordWithDifferentName( 42 | val double : Double 43 | ) 44 | 45 | companion object{ 46 | @JvmStatic 47 | fun createSerializableObjects(): Stream { 48 | return Stream.of( 49 | TestRecord("STTR"), 50 | TestRecordWithNull(null, 2), 51 | TestRecordWithNull("33", 1), 52 | TestRecordWithNamespace(4.0), 53 | TestRecordWithDifferentName(2.0) 54 | ) 55 | } 56 | } 57 | 58 | @Test 59 | fun testRecordSerializedWithNull() { 60 | val serializer = KafkaAvro4kSerializer(registryMock) 61 | serializer.configure( 62 | mapOf( 63 | AbstractKafkaSchemaSerDeConfig.AUTO_REGISTER_SCHEMAS to "true", 64 | AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG to "mock://registry" 65 | ), 66 | false 67 | ) 68 | val avroSchema = Avro.default.schema(TestRecord.serializer()) 69 | val unionSchema = Schema.createUnion(Schema.create(Schema.Type.NULL), avroSchema) 70 | val topic = "My-Topic" 71 | val subjectName = "$topic-value" 72 | registryMock.register(subjectName, AvroSchema(unionSchema)) 73 | val result = serializer.serialize(topic, null) 74 | result.shouldBe(null) 75 | } 76 | 77 | @ParameterizedTest() 78 | @MethodSource("createSerializableObjects") 79 | fun testRecordSerDeRoundtrip(toSerialize: Any?) { 80 | val serializer = KafkaAvro4kSerializer(registryMock) 81 | serializer.configure( 82 | mapOf( 83 | AbstractKafkaSchemaSerDeConfig.AUTO_REGISTER_SCHEMAS to "true", 84 | AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG to "mock://registry" 85 | ), 86 | false 87 | ) 88 | val topic = "My-Topic" 89 | val result = serializer.serialize(topic, toSerialize) 90 | assertNotNull(result) 91 | result ?: throw Exception("") 92 | verify { 93 | registryMock.register(any(), any()) 94 | } 95 | verify(inverse = true) { 96 | registryMock.getId("$topic-value", any()) 97 | } 98 | 99 | 100 | val deserializer = KafkaAvro4kDeserializer( 101 | registryMock, 102 | mapOf( 103 | KafkaAvro4kDeserializerConfig.RECORD_PACKAGES to this::class.java.`package`.name, 104 | AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG to "mock://registry" 105 | ) 106 | ) 107 | 108 | val deserializedValue = deserializer.deserialize(topic, result) 109 | assertEquals(toSerialize, deserializedValue) 110 | } 111 | } -------------------------------------------------------------------------------- /src/test/kotlin/com/github/thake/kafka/avro4k/serializer/PrintConfigDocumentation.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | fun main() { 4 | print(KafkaAvro4kDeserializerConfig.Companion.baseConfigDef().toEnrichedRst()) 5 | 6 | 7 | } -------------------------------------------------------------------------------- /src/test/kotlin/com/github/thake/kafka/avro4k/serializer/TestNetworkOutageRecovery.kt: -------------------------------------------------------------------------------- 1 | package com.github.thake.kafka.avro4k.serializer 2 | 3 | import io.confluent.kafka.schemaregistry.ParsedSchema 4 | import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient 5 | import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException 6 | import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig 7 | import io.mockk.clearAllMocks 8 | import io.mockk.every 9 | import io.mockk.mockk 10 | import io.mockk.verify 11 | import kotlinx.serialization.Serializable 12 | import org.apache.kafka.common.errors.SerializationException 13 | import org.junit.jupiter.api.BeforeEach 14 | import org.junit.jupiter.api.Test 15 | import org.junit.jupiter.api.assertThrows 16 | 17 | class TestNetworkOutageRecovery { 18 | private val mockedRegistry = mockk() 19 | 20 | @Serializable 21 | private data class TestRecord( 22 | val str: String 23 | ) 24 | 25 | @BeforeEach 26 | fun resetMocks() { 27 | clearAllMocks() 28 | } 29 | 30 | @Test 31 | fun testSuccessAfterRetry() { 32 | val serializer = KafkaAvro4kSerializer(mockedRegistry) 33 | serializer.configure( 34 | mapOf( 35 | AbstractKafkaSchemaSerDeConfig.AUTO_REGISTER_SCHEMAS to "true", 36 | AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG to "mock://registry", 37 | AbstractKafkaAvro4kSerDeConfig.SCHEMA_REGISTRY_RETRY_ATTEMPTS_CONFIG to "2" 38 | ), 39 | false 40 | ) 41 | val topic = "My-Topic" 42 | //on first call throw an exception 43 | every { 44 | mockedRegistry.register(any(), any()) 45 | }.throws(RestClientException("Could not contact registry", 404, 4)) 46 | .andThen(1) 47 | serializer.serialize(topic, TestRecord("BLA")) 48 | verify(exactly = 2) { 49 | mockedRegistry.register(any(), any()) 50 | } 51 | } 52 | 53 | @Test 54 | fun testNoInfiniteRetry() { 55 | val serializer = KafkaAvro4kSerializer(mockedRegistry) 56 | serializer.configure( 57 | mapOf( 58 | AbstractKafkaSchemaSerDeConfig.AUTO_REGISTER_SCHEMAS to "true", 59 | AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG to "mock://registry", 60 | AbstractKafkaAvro4kSerDeConfig.SCHEMA_REGISTRY_RETRY_ATTEMPTS_CONFIG to "2" 61 | ), 62 | false 63 | ) 64 | val topic = "My-Topic" 65 | //on first call throw an exception 66 | every { 67 | mockedRegistry.register(any(), any()) 68 | }.throws(RestClientException("Could not contact registry", 404, 4)) 69 | .andThenThrows(RestClientException("Still no contact", 404, 4)) 70 | .andThen(2) 71 | assertThrows { 72 | serializer.serialize(topic, TestRecord("BLA")) 73 | } 74 | verify(exactly = 2) { 75 | mockedRegistry.register(any(), any()) 76 | } 77 | } 78 | } --------------------------------------------------------------------------------