├── .gitignore ├── Jenkinsfile ├── LICENSE ├── README.md ├── bin └── debug.sh ├── config └── connect-avro-docker.properties ├── docker-compose.yml ├── pom.xml └── src ├── main └── java │ └── com │ └── github │ └── jcustenborder │ └── kafka │ └── connect │ └── json │ ├── CustomTimestampFormatValidator.java │ ├── DecimalFormatValidator.java │ ├── FromConnectConversionKey.java │ ├── FromConnectSchemaConverter.java │ ├── FromConnectState.java │ ├── FromConnectVisitor.java │ ├── FromJson.java │ ├── FromJsonConfig.java │ ├── FromJsonConversionKey.java │ ├── FromJsonSchemaConverter.java │ ├── FromJsonSchemaConverterFactory.java │ ├── FromJsonState.java │ ├── FromJsonVisitor.java │ ├── JacksonFactory.java │ ├── JsonConfig.java │ ├── JsonSchemaConverter.java │ ├── JsonSchemaConverterConfig.java │ ├── Utils.java │ └── package-info.java └── test ├── java └── com │ └── github │ └── jcustenborder │ └── kafka │ └── connect │ └── json │ ├── DocumentationTest.java │ ├── FromConnectSchemaConverterTest.java │ ├── FromJsonSchemaConverterTest.java │ ├── FromJsonTest.java │ ├── JsonSchemaConverterTest.java │ └── TestUtils.java └── resources ├── com └── github │ └── jcustenborder │ └── kafka │ └── connect │ └── json │ ├── FromJson │ └── inline.json │ ├── SchemaConverterTest │ ├── nested.schema.json │ ├── product.schema.json │ └── wikimedia.recentchange.schema.json │ ├── basic.data.json │ ├── basic.schema.json │ ├── customdate.data.json │ ├── customdate.schema.json │ ├── geo.data.json │ ├── geo.schema.json │ ├── wikimedia.recentchange.data.json │ └── wikimedia.recentchange.schema.json └── logback.xml /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.nar 17 | *.ear 18 | *.zip 19 | *.tar.gz 20 | *.rar 21 | 22 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 23 | hs_err_pid* 24 | target 25 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | #!groovy 2 | @Library('jenkins-pipeline') import com.github.jcustenborder.jenkins.pipeline.KafkaConnectPipeline 3 | 4 | def pipe = new KafkaConnectPipeline() 5 | pipe.execute() -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | [Documentation](https://jcustenborder.github.io/kafka-connect-documentation/projects/kafka-connect-json-schema) 3 | 4 | Installation through the [Confluent Hub Client](https://docs.confluent.io/current/connect/managing/confluent-hub/client.html) 5 | 6 | This plugin is used to add additional JSON parsing functionality to Kafka Connect. 7 | 8 | ## [From Json transformation](https://jcustenborder.github.io/kafka-connect-documentation/projects/kafka-connect-json-schema/transformations/FromJson.html) 9 | 10 | The FromJson will read JSON data that is in string on byte form and parse the data to a connect structure based on the JSON schema provided. 11 | 12 | # Development 13 | 14 | ## Building the source 15 | 16 | ```bash 17 | mvn clean package 18 | ``` -------------------------------------------------------------------------------- /bin/debug.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | : ${SUSPEND:='n'} 19 | 20 | set -e 21 | 22 | mvn clean package 23 | export KAFKA_DEBUG='y' 24 | connect-standalone config/connect-avro-docker.properties config/sink.properties -------------------------------------------------------------------------------- /config/connect-avro-docker.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | bootstrap.servers=kafka:9092 18 | key.converter=io.confluent.connect.avro.AvroConverter 19 | key.converter.schema.registry.url=http://schema-registry:8081 20 | value.converter=io.confluent.connect.avro.AvroConverter 21 | value.converter.schema.registry.url=http://schema-registry:8081 22 | internal.key.converter=org.apache.kafka.connect.json.JsonConverter 23 | internal.value.converter=org.apache.kafka.connect.json.JsonConverter 24 | internal.key.converter.schemas.enable=false 25 | internal.value.converter.schemas.enable=false 26 | offset.storage.file.filename=/tmp/connect.offsets 27 | plugin.path=target/kafka-connect-target/usr/share/kafka-connect -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | version: "2" 18 | services: 19 | zookeeper: 20 | image: confluentinc/cp-zookeeper:5.4.0 21 | ports: 22 | - "2181:2181" 23 | environment: 24 | ZOOKEEPER_CLIENT_PORT: 2181 25 | kafka: 26 | image: confluentinc/cp-kafka:5.4.0 27 | depends_on: 28 | - zookeeper 29 | ports: 30 | - "9092:9092" 31 | environment: 32 | KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" 33 | KAFKA_ADVERTISED_LISTENERS: "plaintext://kafka:9092" 34 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 35 | schema-registry: 36 | image: confluentinc/cp-schema-registry:5.4.0 37 | depends_on: 38 | - kafka 39 | - zookeeper 40 | ports: 41 | - "8081:8081" 42 | environment: 43 | SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: "zookeeper:2181" 44 | SCHEMA_REGISTRY_HOST_NAME: schema-registry 45 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 23 | 4.0.0 24 | 25 | com.github.jcustenborder.kafka.connect 26 | kafka-connect-parent 27 | 2.4.0 28 | 29 | kafka-connect-json-schema 30 | 0.2-SNAPSHOT 31 | kafka-connect-json-schema 32 | This project provides a mechanism to marshall data from JSON to a Kafka Connect struct based on a user provided JSON schema. This is accomplished by using the FromJson transformation which will convert data as a transformation. The JsonSchemaConverter transformation can be used to read and write data as Json. 33 | https://github.com/jcustenborder/kafka-connect-json-schema 34 | 2020 35 | 36 | 37 | The Apache License, Version 2.0 38 | https://www.apache.org/licenses/LICENSE-2.0 39 | repo 40 | 41 | 42 | 43 | 44 | jcustenborder 45 | Jeremy Custenborder 46 | https://github.com/jcustenborder 47 | 48 | Committer 49 | 50 | 51 | 52 | 53 | scm:git:https://github.com/jcustenborder/kafka-connect-json-schema.git 54 | scm:git:git@github.com:jcustenborder/kafka-connect-json-schema.git 55 | 56 | https://github.com/jcustenborder/kafka-connect-json-schema 57 | 58 | 59 | github 60 | https://github.com/jcustenborder/kafka-connect-json-schema/issues 61 | 62 | 63 | 64 | com.github.everit-org.json-schema 65 | org.everit.json.schema 66 | 1.12.1 67 | 68 | 69 | 70 | 71 | jitpack.io 72 | https://jitpack.io 73 | 74 | 75 | 76 | 77 | 78 | io.confluent 79 | kafka-connect-maven-plugin 80 | 81 | true 82 | https://jcustenborder.github.io/kafka-connect-documentation/ 83 | 84 | transform 85 | converter 86 | 87 | 88 | Transform 89 | JSON 90 | 91 | Kafka Connect JSON Schema Transformations 92 | ${pom.issueManagement.url} 93 | Support provided through community involvement. 94 | 95 | org.reflections:reflections 96 | 97 | 98 | 99 | 100 | 101 | 102 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/CustomTimestampFormatValidator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import org.everit.json.schema.FormatValidator; 19 | 20 | import java.util.Optional; 21 | 22 | public class CustomTimestampFormatValidator implements FormatValidator { 23 | @Override 24 | public Optional validate(String s) { 25 | return Optional.empty(); 26 | } 27 | 28 | @Override 29 | public String formatName() { 30 | return "custom-timestamp"; 31 | } 32 | } -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/DecimalFormatValidator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import org.everit.json.schema.FormatValidator; 19 | 20 | import java.util.Optional; 21 | 22 | class DecimalFormatValidator implements FormatValidator { 23 | @Override 24 | public Optional validate(String s) { 25 | return Optional.empty(); 26 | } 27 | 28 | @Override 29 | public String formatName() { 30 | return "decimal"; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/FromConnectConversionKey.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import com.google.common.base.MoreObjects; 19 | import com.google.common.base.Objects; 20 | import org.apache.kafka.connect.data.Decimal; 21 | import org.apache.kafka.connect.data.Schema; 22 | 23 | public class FromConnectConversionKey { 24 | public final Schema.Type type; 25 | public final String schemaName; 26 | public final Integer scale; 27 | 28 | private FromConnectConversionKey(Schema.Type type, String schemaName, Integer scale) { 29 | this.type = type; 30 | this.schemaName = schemaName; 31 | this.scale = scale; 32 | } 33 | 34 | public static FromConnectConversionKey of(Schema schema) { 35 | Integer scale; 36 | if (Decimal.LOGICAL_NAME.equals(schema.name())) { 37 | String scaleText = schema.parameters().get(Decimal.SCALE_FIELD); 38 | scale = Integer.parseInt(scaleText); 39 | } else { 40 | scale = null; 41 | } 42 | return of(schema.type(), schema.name(), scale); 43 | } 44 | 45 | public static FromConnectConversionKey of(Schema.Type type) { 46 | return new FromConnectConversionKey(type, null, null); 47 | } 48 | 49 | public static FromConnectConversionKey of(Schema.Type type, String schemaName) { 50 | return new FromConnectConversionKey(type, schemaName, null); 51 | } 52 | 53 | public static FromConnectConversionKey of(Schema.Type type, String schemaName, Integer scale) { 54 | return new FromConnectConversionKey(type, schemaName, scale); 55 | } 56 | 57 | @Override 58 | public boolean equals(Object o) { 59 | if (this == o) return true; 60 | if (o == null || getClass() != o.getClass()) return false; 61 | FromConnectConversionKey that = (FromConnectConversionKey) o; 62 | return type == that.type && 63 | Objects.equal(schemaName, that.schemaName) && 64 | Objects.equal(scale, that.scale); 65 | } 66 | 67 | @Override 68 | public int hashCode() { 69 | return Objects.hashCode(type, schemaName, scale); 70 | } 71 | 72 | @Override 73 | public String toString() { 74 | return MoreObjects.toStringHelper(this) 75 | .omitNullValues() 76 | .add("type", type) 77 | .add("schemaName", schemaName) 78 | .add("scale", scale) 79 | .toString(); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/FromConnectSchemaConverter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import com.google.common.base.Charsets; 19 | import com.google.common.base.MoreObjects; 20 | import com.google.common.base.Strings; 21 | import com.google.common.collect.ImmutableMap; 22 | import org.apache.kafka.common.header.Header; 23 | import org.apache.kafka.common.header.internals.RecordHeader; 24 | import org.apache.kafka.connect.data.Date; 25 | import org.apache.kafka.connect.data.Decimal; 26 | import org.apache.kafka.connect.data.Schema; 27 | import org.apache.kafka.connect.data.Schema.Type; 28 | import org.apache.kafka.connect.data.Time; 29 | import org.apache.kafka.connect.data.Timestamp; 30 | import org.json.JSONObject; 31 | import org.slf4j.Logger; 32 | import org.slf4j.LoggerFactory; 33 | 34 | import java.util.ArrayList; 35 | import java.util.LinkedHashMap; 36 | import java.util.List; 37 | import java.util.Map; 38 | import java.util.stream.IntStream; 39 | import java.util.stream.Stream; 40 | 41 | public class FromConnectSchemaConverter { 42 | static final Map> PRIMITIVE_TYPES; 43 | static final Map PRIMITIVE_VISITORS; 44 | private static final Logger log = LoggerFactory.getLogger(FromConnectSchemaConverter.class); 45 | 46 | private static void addType( 47 | Map primitiveVisitors, 48 | Map> primitiveTypes, 49 | FromConnectConversionKey key, 50 | FromConnectVisitor visitor, 51 | ImmutableMap properties) { 52 | 53 | primitiveTypes.put(key, properties); 54 | primitiveVisitors.put(key, visitor); 55 | } 56 | 57 | static { 58 | Map visitors = new LinkedHashMap<>(); 59 | Map> types = new LinkedHashMap<>(); 60 | addType( 61 | visitors, types, 62 | FromConnectConversionKey.of(Type.BOOLEAN), 63 | new FromConnectVisitor.BooleanVisitor(), 64 | ImmutableMap.of("type", "boolean") 65 | ); 66 | addType( 67 | visitors, types, 68 | FromConnectConversionKey.of(Type.BYTES), 69 | new FromConnectVisitor.BytesVisitor(), 70 | ImmutableMap.of("type", "string", "contentEncoding", "base64") 71 | ); 72 | addType( 73 | visitors, types, 74 | FromConnectConversionKey.of(Type.FLOAT32), 75 | new FromConnectVisitor.FloatVisitor(), 76 | ImmutableMap.of("type", "number") 77 | ); 78 | addType( 79 | visitors, types, 80 | FromConnectConversionKey.of(Type.FLOAT64), 81 | new FromConnectVisitor.FloatVisitor(), 82 | ImmutableMap.of("type", "number") 83 | ); 84 | Stream.of(Type.INT8, Type.INT16, Type.INT32, Type.INT64) 85 | .forEach(type -> { 86 | addType( 87 | visitors, types, 88 | FromConnectConversionKey.of(type), 89 | new FromConnectVisitor.IntegerVisitor(), 90 | ImmutableMap.of("type", "integer") 91 | ); 92 | }); 93 | IntStream.range(0, 100) 94 | .forEach(scale -> { 95 | Schema decimalSchema = Decimal.schema(scale); 96 | addType( 97 | visitors, types, 98 | FromConnectConversionKey.of(decimalSchema), 99 | new FromConnectVisitor.DecimalVisitor(scale), 100 | ImmutableMap.of( 101 | "type", "string", 102 | "format", "decimal", 103 | "scale", Integer.toString(scale) 104 | ) 105 | ); 106 | }); 107 | 108 | visitors.put(FromConnectConversionKey.of(Type.STRING), new FromConnectVisitor.StringVisitor()); 109 | types.put(FromConnectConversionKey.of(Type.STRING), ImmutableMap.of("type", "string")); 110 | 111 | visitors.put(FromConnectConversionKey.of(Date.SCHEMA), new FromConnectVisitor.DateVisitor()); 112 | types.put(FromConnectConversionKey.of(Date.SCHEMA), ImmutableMap.of("type", "string", "format", "date")); 113 | 114 | visitors.put(FromConnectConversionKey.of(Time.SCHEMA), new FromConnectVisitor.TimeVisitor()); 115 | types.put(FromConnectConversionKey.of(Time.SCHEMA), ImmutableMap.of("type", "string", "format", "time")); 116 | 117 | visitors.put(FromConnectConversionKey.of(Timestamp.SCHEMA), new FromConnectVisitor.DateTimeVisitor()); 118 | types.put(FromConnectConversionKey.of(Timestamp.SCHEMA), ImmutableMap.of("type", "string", "format", "date-time")); 119 | PRIMITIVE_TYPES = ImmutableMap.copyOf(types); 120 | PRIMITIVE_VISITORS = ImmutableMap.copyOf(visitors); 121 | } 122 | 123 | public static FromConnectState toJsonSchema(org.apache.kafka.connect.data.Schema schema, String headerName) { 124 | Map definitions = new LinkedHashMap<>(); 125 | List visitors = new ArrayList<>(); 126 | JSONObject result = toJsonSchema(schema, definitions, visitors); 127 | result.put("$schema", "http://json-schema.org/draft-07/schema#"); 128 | if (!definitions.isEmpty()) { 129 | //definitions 130 | JSONObject definitionsObject = new JSONObject(); 131 | definitions.forEach((definitionName, definition) -> { 132 | definitionsObject.put(definition.name(), definition.jsonSchema()); 133 | }); 134 | result.put("definitions", definitionsObject); 135 | } 136 | 137 | Header header = new RecordHeader( 138 | headerName, 139 | result.toString().getBytes(Charsets.UTF_8) 140 | ); 141 | 142 | FromConnectVisitor visitor = visitors.get(0); 143 | return FromConnectState.of(header, visitor); 144 | } 145 | 146 | private static JSONObject toJsonSchema(org.apache.kafka.connect.data.Schema schema, Map definitions, List visitors) { 147 | JSONObject result = new JSONObject(); 148 | if (!Strings.isNullOrEmpty(schema.doc())) { 149 | result.put("description", schema.doc()); 150 | } 151 | FromConnectConversionKey key = FromConnectConversionKey.of(schema); 152 | log.trace("toJsonSchema() - Checking for '{}'", key); 153 | Map primitiveType = PRIMITIVE_TYPES.get(key); 154 | if (null != primitiveType) { 155 | primitiveType.forEach(result::put); 156 | FromConnectVisitor visitor = PRIMITIVE_VISITORS.get(key); 157 | visitors.add(visitor); 158 | return result; 159 | } 160 | 161 | if (!Strings.isNullOrEmpty(schema.name())) { 162 | result.put("title", schema.name()); 163 | } 164 | 165 | 166 | if (Type.ARRAY == schema.type()) { 167 | result.put("type", "array"); 168 | FromConnectVisitor elementVisitor; 169 | if (Type.STRUCT == schema.valueSchema().type()) { 170 | Definition definition = definitions.computeIfAbsent(schema.valueSchema(), s -> { 171 | List childVisitors = new ArrayList<>(); 172 | JSONObject fieldJsonSchema = toJsonSchema(schema.valueSchema(), definitions, childVisitors); 173 | String definitionName = schema.valueSchema().name().toLowerCase(); 174 | return Definition.of(fieldJsonSchema, definitionName, childVisitors); 175 | }); 176 | result.put("items", definition.ref()); 177 | elementVisitor = definition.visitors().get(0); 178 | } else { 179 | List childVisitors = new ArrayList<>(); 180 | JSONObject arrayValueSchema = toJsonSchema(schema.valueSchema(), definitions, childVisitors); 181 | elementVisitor = childVisitors.get(0); 182 | result.put("items", arrayValueSchema); 183 | } 184 | visitors.add(new FromConnectVisitor.ArrayVisitor(elementVisitor)); 185 | } 186 | if (Type.STRUCT == schema.type()) { 187 | List requiredFields = new ArrayList<>(schema.fields().size()); 188 | Map properties = new LinkedHashMap<>(schema.fields().size()); 189 | Map structVisitors = new LinkedHashMap<>(); 190 | schema.fields().forEach(field -> { 191 | log.trace("toJsonSchema() - field:{} type:{}", field.name(), field.schema().type()); 192 | List childVisitors = new ArrayList<>(); 193 | if (!field.schema().isOptional()) { 194 | requiredFields.add(field.name()); 195 | } 196 | if (Type.STRUCT == field.schema().type()) { 197 | Definition definition = definitions.computeIfAbsent(field.schema(), s -> { 198 | List definitionVisitors = new ArrayList<>(); 199 | JSONObject fieldJsonSchema = toJsonSchema(field.schema(), definitions, definitionVisitors); 200 | String definitionName = field.schema().name().toLowerCase(); 201 | return Definition.of(fieldJsonSchema, definitionName, definitionVisitors); 202 | }); 203 | childVisitors.addAll(definition.visitors()); 204 | properties.put(field.name(), definition.ref()); 205 | } else { 206 | JSONObject fieldJsonSchema = toJsonSchema(field.schema(), definitions, childVisitors); 207 | properties.put(field.name(), fieldJsonSchema); 208 | } 209 | FromConnectVisitor fieldVisitor = childVisitors.get(0); 210 | structVisitors.put(field.name(), fieldVisitor); 211 | }); 212 | result.put("properties", properties); 213 | result.put("required", requiredFields); 214 | result.put("type", "object"); 215 | visitors.add(new FromConnectVisitor.StructVisitor(structVisitors)); 216 | } 217 | 218 | 219 | log.trace("toJsonSchema() - '{}' is not primitive.", schema.type()); 220 | 221 | return result; 222 | } 223 | 224 | static class Definition { 225 | private final JSONObject jsonSchema; 226 | private final String name; 227 | private final List visitors; 228 | 229 | 230 | private Definition(JSONObject jsonSchema, String name, List visitors) { 231 | this.jsonSchema = jsonSchema; 232 | this.name = name; 233 | this.visitors = visitors; 234 | } 235 | 236 | public static Definition of(JSONObject jsonSchema, String ref, List visitors) { 237 | return new Definition(jsonSchema, ref, visitors); 238 | } 239 | 240 | public JSONObject jsonSchema() { 241 | return this.jsonSchema; 242 | } 243 | 244 | public String name() { 245 | return this.name; 246 | } 247 | 248 | public JSONObject ref() { 249 | return new JSONObject() 250 | .put("$ref", String.format("#/definitions/%s", this.name)); 251 | } 252 | 253 | public List visitors() { 254 | return this.visitors; 255 | } 256 | 257 | @Override 258 | public String toString() { 259 | return MoreObjects.toStringHelper(this) 260 | .add("jsonSchema", jsonSchema) 261 | .add("name", name) 262 | .toString(); 263 | } 264 | } 265 | 266 | 267 | } 268 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/FromConnectState.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import org.apache.kafka.common.header.Header; 19 | 20 | class FromConnectState { 21 | final Header header; 22 | final FromConnectVisitor visitor; 23 | 24 | 25 | private FromConnectState(Header header, FromConnectVisitor visitor) { 26 | this.header = header; 27 | this.visitor = visitor; 28 | } 29 | 30 | public static FromConnectState of(Header header, FromConnectVisitor visitor) { 31 | return new FromConnectState(header, visitor); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/FromConnectVisitor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import com.fasterxml.jackson.core.JsonGenerator; 19 | import com.google.common.io.BaseEncoding; 20 | import org.apache.kafka.connect.data.Struct; 21 | 22 | import java.io.IOException; 23 | import java.math.BigDecimal; 24 | import java.text.DecimalFormat; 25 | import java.util.Date; 26 | import java.util.List; 27 | import java.util.Map; 28 | 29 | public abstract class FromConnectVisitor { 30 | public abstract void doVisit(JsonGenerator jsonGenerator, T value) throws IOException; 31 | 32 | public static class StringVisitor extends FromConnectVisitor { 33 | @Override 34 | public void doVisit(JsonGenerator jsonGenerator, String value) throws IOException { 35 | jsonGenerator.writeString(value); 36 | } 37 | } 38 | 39 | public static class StructVisitor extends FromConnectVisitor { 40 | final Map visitors; 41 | 42 | public StructVisitor(Map visitors) { 43 | this.visitors = visitors; 44 | } 45 | 46 | @Override 47 | public void doVisit(JsonGenerator jsonGenerator, Struct value) throws IOException { 48 | jsonGenerator.writeStartObject(); 49 | for (Map.Entry e : this.visitors.entrySet()) { 50 | final String fieldName = e.getKey(); 51 | final FromConnectVisitor visitor = e.getValue(); 52 | final Object fieldValue = value.get(fieldName); 53 | jsonGenerator.writeFieldName(fieldName); 54 | visitor.doVisit(jsonGenerator, fieldValue); 55 | } 56 | jsonGenerator.writeEndObject(); 57 | } 58 | } 59 | 60 | public static class BooleanVisitor extends FromConnectVisitor { 61 | @Override 62 | public void doVisit(JsonGenerator jsonGenerator, Boolean value) throws IOException { 63 | jsonGenerator.writeBoolean(value); 64 | } 65 | } 66 | 67 | public static class BytesVisitor extends FromConnectVisitor { 68 | @Override 69 | public void doVisit(JsonGenerator jsonGenerator, byte[] value) throws IOException { 70 | jsonGenerator.writeString( 71 | BaseEncoding.base64().encode(value) 72 | ); 73 | } 74 | } 75 | 76 | public static class FloatVisitor extends FromConnectVisitor { 77 | 78 | @Override 79 | public void doVisit(JsonGenerator jsonGenerator, Number value) throws IOException { 80 | jsonGenerator.writeNumber(value.doubleValue()); 81 | } 82 | } 83 | 84 | public static class DateTimeVisitor extends FromConnectVisitor { 85 | @Override 86 | public void doVisit(JsonGenerator jsonGenerator, Date value) throws IOException { 87 | jsonGenerator.writeString( 88 | Utils.TIMESTAMP_FORMATTER.format( 89 | value.toInstant() 90 | ) 91 | ); 92 | } 93 | } 94 | 95 | public static class DateVisitor extends FromConnectVisitor { 96 | @Override 97 | public void doVisit(JsonGenerator jsonGenerator, Date value) throws IOException { 98 | jsonGenerator.writeString( 99 | Utils.DATE_FORMATTER.format( 100 | value.toInstant() 101 | ) 102 | ); 103 | } 104 | } 105 | 106 | public static class TimeVisitor extends FromConnectVisitor { 107 | @Override 108 | public void doVisit(JsonGenerator jsonGenerator, Date value) throws IOException { 109 | jsonGenerator.writeString( 110 | Utils.TIME_FORMATTER.format( 111 | value.toInstant() 112 | ) 113 | ); 114 | } 115 | } 116 | 117 | public static class IntegerVisitor extends FromConnectVisitor { 118 | 119 | 120 | @Override 121 | public void doVisit(JsonGenerator jsonGenerator, Number value) throws IOException { 122 | jsonGenerator.writeNumber(value.longValue()); 123 | } 124 | } 125 | 126 | public static class ArrayVisitor extends FromConnectVisitor { 127 | final FromConnectVisitor elementVisitor; 128 | 129 | public ArrayVisitor(FromConnectVisitor elementVisitor) { 130 | this.elementVisitor = elementVisitor; 131 | } 132 | 133 | @Override 134 | public void doVisit(JsonGenerator jsonGenerator, List value) throws IOException { 135 | jsonGenerator.writeStartArray(); 136 | for (Object o : value) { 137 | this.elementVisitor.doVisit(jsonGenerator, o); 138 | } 139 | jsonGenerator.writeEndArray(); 140 | } 141 | } 142 | 143 | public static class DecimalVisitor extends FromConnectVisitor { 144 | DecimalFormat decimalFormat; 145 | 146 | public DecimalVisitor(int scale) { 147 | this.decimalFormat = new DecimalFormat("#"); 148 | this.decimalFormat.setMaximumFractionDigits(scale); 149 | } 150 | 151 | @Override 152 | public void doVisit(JsonGenerator jsonGenerator, BigDecimal value) throws IOException { 153 | jsonGenerator.writeString( 154 | this.decimalFormat.format(value) 155 | ); 156 | } 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/FromJson.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import com.fasterxml.jackson.databind.JsonNode; 19 | import com.fasterxml.jackson.databind.ObjectMapper; 20 | import com.github.jcustenborder.kafka.connect.utils.config.Description; 21 | import com.github.jcustenborder.kafka.connect.utils.config.DocumentationTip; 22 | import com.github.jcustenborder.kafka.connect.utils.config.Title; 23 | import com.github.jcustenborder.kafka.connect.utils.transformation.BaseKeyValueTransformation; 24 | import org.apache.kafka.common.config.ConfigDef; 25 | import org.apache.kafka.common.config.ConfigException; 26 | import org.apache.kafka.connect.connector.ConnectRecord; 27 | import org.apache.kafka.connect.data.Schema; 28 | import org.apache.kafka.connect.data.SchemaAndValue; 29 | import org.apache.kafka.connect.errors.DataException; 30 | import org.everit.json.schema.ValidationException; 31 | import org.json.JSONObject; 32 | import org.slf4j.Logger; 33 | import org.slf4j.LoggerFactory; 34 | 35 | import java.io.ByteArrayInputStream; 36 | import java.io.IOException; 37 | import java.io.InputStream; 38 | import java.io.Reader; 39 | import java.io.StringReader; 40 | import java.util.Map; 41 | 42 | @Title("From Json transformation") 43 | @Description("The FromJson will read JSON data that is in string on byte form and parse the data to " + 44 | "a connect structure based on the JSON schema provided.") 45 | @DocumentationTip("This transformation expects data to be in either String or Byte format. You are " + 46 | "most likely going to use the ByteArrayConverter or the StringConverter.") 47 | public class FromJson> extends BaseKeyValueTransformation { 48 | private static final Logger log = LoggerFactory.getLogger(FromJson.class); 49 | FromJsonConfig config; 50 | 51 | protected FromJson(boolean isKey) { 52 | super(isKey); 53 | } 54 | 55 | @Override 56 | public ConfigDef config() { 57 | return FromJsonConfig.config(); 58 | } 59 | 60 | @Override 61 | public void close() { 62 | 63 | } 64 | 65 | SchemaAndValue processJsonNode(R record, Schema inputSchema, JsonNode node) { 66 | Object result = this.fromJsonState.visitor.visit(node); 67 | return new SchemaAndValue(this.fromJsonState.schema, result); 68 | } 69 | 70 | 71 | void validateJson(JSONObject jsonObject) { 72 | try { 73 | this.fromJsonState.jsonSchema.validate(jsonObject); 74 | } catch (ValidationException ex) { 75 | StringBuilder builder = new StringBuilder(); 76 | builder.append( 77 | String.format( 78 | "Could not validate JSON. Found %s violations(s).", 79 | ex.getViolationCount() 80 | ) 81 | ); 82 | for (ValidationException message : ex.getCausingExceptions()) { 83 | log.error("Validation exception", message); 84 | builder.append("\n"); 85 | builder.append(message.getMessage()); 86 | } 87 | throw new DataException( 88 | builder.toString(), 89 | ex 90 | ); 91 | } 92 | } 93 | 94 | 95 | @Override 96 | protected SchemaAndValue processBytes(R record, Schema inputSchema, byte[] input) { 97 | try { 98 | if (this.config.validateJson) { 99 | try (InputStream inputStream = new ByteArrayInputStream(input)) { 100 | JSONObject jsonObject = Utils.loadObject(inputStream); 101 | validateJson(jsonObject); 102 | } 103 | } 104 | JsonNode node = this.objectMapper.readValue(input, JsonNode.class); 105 | return processJsonNode(record, inputSchema, node); 106 | } catch (IOException e) { 107 | throw new DataException(e); 108 | } 109 | } 110 | 111 | @Override 112 | protected SchemaAndValue processString(R record, Schema inputSchema, String input) { 113 | try { 114 | if (this.config.validateJson) { 115 | try (Reader reader = new StringReader(input)) { 116 | JSONObject jsonObject = Utils.loadObject(reader); 117 | validateJson(jsonObject); 118 | } 119 | } 120 | JsonNode node = this.objectMapper.readValue(input, JsonNode.class); 121 | return processJsonNode(record, inputSchema, node); 122 | } catch (IOException e) { 123 | throw new DataException(e); 124 | } 125 | } 126 | 127 | FromJsonState fromJsonState; 128 | FromJsonSchemaConverterFactory fromJsonSchemaConverterFactory; 129 | ObjectMapper objectMapper; 130 | 131 | @Override 132 | public void configure(Map map) { 133 | this.config = new FromJsonConfig(map); 134 | this.fromJsonSchemaConverterFactory = new FromJsonSchemaConverterFactory(config); 135 | 136 | org.everit.json.schema.Schema schema; 137 | if (JsonConfig.SchemaLocation.Url == this.config.schemaLocation) { 138 | try { 139 | try (InputStream inputStream = this.config.schemaUrl.openStream()) { 140 | schema = Utils.loadSchema(inputStream); 141 | } 142 | } catch (IOException e) { 143 | ConfigException exception = new ConfigException(JsonConfig.SCHEMA_URL_CONF, this.config.schemaUrl, "exception while loading schema"); 144 | exception.initCause(e); 145 | throw exception; 146 | } 147 | } else if (JsonConfig.SchemaLocation.Inline == this.config.schemaLocation) { 148 | schema = Utils.loadSchema(this.config.schemaText); 149 | } else { 150 | throw new ConfigException( 151 | JsonConfig.SCHEMA_LOCATION_CONF, 152 | this.config.schemaLocation.toString(), 153 | "Location is not supported" 154 | ); 155 | } 156 | 157 | this.fromJsonState = this.fromJsonSchemaConverterFactory.fromJSON(schema); 158 | this.objectMapper = JacksonFactory.create(); 159 | } 160 | 161 | public static class Key> extends FromJson { 162 | public Key() { 163 | super(true); 164 | } 165 | } 166 | 167 | public static class Value> extends FromJson { 168 | public Value() { 169 | super(false); 170 | } 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/FromJsonConfig.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import org.apache.kafka.common.config.ConfigDef; 19 | 20 | import java.util.Map; 21 | 22 | class FromJsonConfig extends JsonConfig { 23 | 24 | 25 | public FromJsonConfig(Map originals) { 26 | super(config(), originals); 27 | } 28 | 29 | public static ConfigDef config() { 30 | return JsonConfig.config(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/FromJsonConversionKey.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import com.google.common.base.MoreObjects; 19 | import com.google.common.base.Objects; 20 | import org.everit.json.schema.NumberSchema; 21 | import org.everit.json.schema.Schema; 22 | import org.everit.json.schema.StringSchema; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | class FromJsonConversionKey { 27 | static final String UNNAMED_FORMAT = "unnamed-format"; 28 | private static final Logger log = LoggerFactory.getLogger(FromJsonConversionKey.class); 29 | final Class schemaClass; 30 | final String format; 31 | final Boolean requiresInteger; 32 | final String contentEncoding; 33 | 34 | private FromJsonConversionKey(Class schemaClass, String format, Boolean requiresInteger, String contentEncoding) { 35 | this.schemaClass = schemaClass; 36 | this.format = format; 37 | this.requiresInteger = requiresInteger; 38 | this.contentEncoding = contentEncoding; 39 | } 40 | 41 | public static FromJsonConversionKey of(org.everit.json.schema.Schema jsonSchema) { 42 | String format; 43 | Boolean requiresInteger; 44 | String contentEncoding; 45 | if (jsonSchema instanceof StringSchema) { 46 | StringSchema stringSchema = (StringSchema) jsonSchema; 47 | format = UNNAMED_FORMAT.equals(stringSchema.getFormatValidator().formatName()) ? null : stringSchema.getFormatValidator().formatName(); 48 | contentEncoding = (String) stringSchema.getUnprocessedProperties().get("contentEncoding"); 49 | requiresInteger = null; 50 | log.trace("jsonSchema = '{}' format = '{}'", jsonSchema, format); 51 | } else if (jsonSchema instanceof NumberSchema) { 52 | NumberSchema numberSchema = (NumberSchema) jsonSchema; 53 | requiresInteger = numberSchema.requiresInteger(); 54 | format = null; 55 | contentEncoding = null; 56 | } else { 57 | format = null; 58 | requiresInteger = null; 59 | contentEncoding = null; 60 | } 61 | 62 | return new FromJsonConversionKey(jsonSchema.getClass(), format, requiresInteger, contentEncoding); 63 | } 64 | 65 | public static Builder from(Class schemaClass) { 66 | return new Builder().schemaClass(schemaClass); 67 | } 68 | 69 | @Override 70 | public boolean equals(Object o) { 71 | if (this == o) return true; 72 | if (o == null || getClass() != o.getClass()) return false; 73 | FromJsonConversionKey that = (FromJsonConversionKey) o; 74 | return Objects.equal(schemaClass, that.schemaClass) && 75 | Objects.equal(format, that.format) && 76 | Objects.equal(requiresInteger, that.requiresInteger) && 77 | Objects.equal(contentEncoding, that.contentEncoding); 78 | } 79 | 80 | @Override 81 | public int hashCode() { 82 | return Objects.hashCode(schemaClass, format, requiresInteger, contentEncoding); 83 | } 84 | 85 | // public static ConversionKey of(Class schemaClass) { 86 | // return new ConversionKey(schemaClass, null, null, contentMediaType); 87 | // } 88 | // 89 | // public static ConversionKey of(Class schemaClass, String format) { 90 | // return new ConversionKey(schemaClass, format, null, contentMediaType); 91 | // } 92 | // 93 | // public static ConversionKey of(Class schemaClass, Boolean requiesInteger) { 94 | // return new ConversionKey(schemaClass, null, requiesInteger, contentMediaType); 95 | // } 96 | 97 | @Override 98 | public String toString() { 99 | return MoreObjects.toStringHelper(this) 100 | .add("schemaClass", schemaClass) 101 | .add("format", format) 102 | .add("requiresInteger", requiresInteger) 103 | .add("contentEncoding", contentEncoding) 104 | .toString(); 105 | } 106 | 107 | static class Builder { 108 | Class schemaClass; 109 | String format; 110 | Boolean requiresInteger; 111 | String contentEncoding; 112 | private Builder() { 113 | } 114 | 115 | public Builder schemaClass(Class schemaClass) { 116 | this.schemaClass = schemaClass; 117 | return this; 118 | } 119 | 120 | public Builder format(String format) { 121 | this.format = format; 122 | return this; 123 | } 124 | 125 | public Builder requiresInteger(Boolean requiresInteger) { 126 | this.requiresInteger = requiresInteger; 127 | return this; 128 | } 129 | 130 | public Builder contentEncoding(String contentMediaType) { 131 | this.contentEncoding = contentMediaType; 132 | return this; 133 | } 134 | 135 | public FromJsonConversionKey build() { 136 | return new FromJsonConversionKey(this.schemaClass, this.format, this.requiresInteger, this.contentEncoding); 137 | } 138 | } 139 | 140 | } 141 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/FromJsonSchemaConverter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import com.fasterxml.jackson.databind.JsonNode; 19 | import com.fasterxml.jackson.databind.node.ArrayNode; 20 | import com.fasterxml.jackson.databind.node.BooleanNode; 21 | import com.fasterxml.jackson.databind.node.NumericNode; 22 | import com.fasterxml.jackson.databind.node.ObjectNode; 23 | import com.fasterxml.jackson.databind.node.TextNode; 24 | import com.google.common.base.Preconditions; 25 | import com.google.common.collect.ImmutableSet; 26 | import org.apache.kafka.connect.data.Date; 27 | import org.apache.kafka.connect.data.Decimal; 28 | import org.apache.kafka.connect.data.Schema; 29 | import org.apache.kafka.connect.data.SchemaBuilder; 30 | import org.apache.kafka.connect.data.Struct; 31 | import org.apache.kafka.connect.data.Time; 32 | import org.apache.kafka.connect.data.Timestamp; 33 | import org.everit.json.schema.ArraySchema; 34 | import org.everit.json.schema.BooleanSchema; 35 | import org.everit.json.schema.NumberSchema; 36 | import org.everit.json.schema.ObjectSchema; 37 | import org.everit.json.schema.StringSchema; 38 | import org.slf4j.Logger; 39 | import org.slf4j.LoggerFactory; 40 | 41 | import java.util.List; 42 | import java.util.Map; 43 | import java.util.Set; 44 | 45 | public abstract class FromJsonSchemaConverter { 46 | private static final Logger log = LoggerFactory.getLogger(FromJsonSchemaConverter.class); 47 | protected final FromJsonSchemaConverterFactory factory; 48 | protected final JsonConfig config; 49 | 50 | protected FromJsonSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) { 51 | this.factory = factory; 52 | this.config = config; 53 | } 54 | 55 | protected abstract SchemaBuilder schemaBuilder(T schema); 56 | 57 | protected abstract FromJsonConversionKey key(); 58 | 59 | protected abstract FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors); 60 | 61 | protected abstract void fromJSON(SchemaBuilder builder, T jsonSchema, Map visitors); 62 | 63 | static class BooleanSchemaConverter extends FromJsonSchemaConverter { 64 | BooleanSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) { 65 | super(factory, config); 66 | } 67 | 68 | @Override 69 | protected SchemaBuilder schemaBuilder(BooleanSchema schema) { 70 | return SchemaBuilder.bool(); 71 | } 72 | 73 | @Override 74 | protected FromJsonConversionKey key() { 75 | return FromJsonConversionKey.from(BooleanSchema.class).build(); 76 | } 77 | 78 | @Override 79 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) { 80 | return new FromJsonVisitor.BooleanVisitor(connectSchema); 81 | } 82 | 83 | @Override 84 | protected void fromJSON(SchemaBuilder builder, BooleanSchema jsonSchema, Map visitors) { 85 | 86 | } 87 | } 88 | 89 | static class ObjectSchemaConverter extends FromJsonSchemaConverter { 90 | 91 | ObjectSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) { 92 | super(factory, config); 93 | } 94 | 95 | @Override 96 | protected SchemaBuilder schemaBuilder(ObjectSchema schema) { 97 | return SchemaBuilder.struct(); 98 | } 99 | 100 | @Override 101 | protected FromJsonConversionKey key() { 102 | return FromJsonConversionKey.from(ObjectSchema.class).build(); 103 | } 104 | 105 | @Override 106 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) { 107 | return new FromJsonVisitor.StructVisitor(connectSchema, visitors); 108 | } 109 | 110 | static final Set EXCLUDE_PROPERTIES = ImmutableSet.of("$schema"); 111 | 112 | @Override 113 | protected void fromJSON(SchemaBuilder builder, ObjectSchema jsonSchema, Map visitors) { 114 | Set requiredProperties = ImmutableSet.copyOf(jsonSchema.getRequiredProperties()); 115 | jsonSchema.getPropertySchemas() 116 | .entrySet() 117 | .stream() 118 | .filter(e -> { 119 | boolean result = !EXCLUDE_PROPERTIES.contains(e.getKey()); 120 | log.trace("fromJson() - filtering '{}' result = '{}'", e.getKey(), result); 121 | return result; 122 | }) 123 | .filter(e -> { 124 | String schemaLocation = e.getValue().getSchemaLocation(); 125 | boolean result = !this.config.excludeLocations.contains(schemaLocation); 126 | log.trace("fromJson() - filtering '{}' location='{}' result = '{}'", e.getKey(), e.getValue().getSchemaLocation(), result); 127 | return result; 128 | }) 129 | .sorted(Map.Entry.comparingByKey()) 130 | .forEach(e -> { 131 | final String propertyName = e.getKey(); 132 | final org.everit.json.schema.Schema propertyJsonSchema = e.getValue(); 133 | final boolean isOptional = !requiredProperties.contains(propertyName); 134 | log.trace("fromJson() - Processing property '{}' '{}'", propertyName, propertyJsonSchema); 135 | FromJsonState state = this.factory.fromJSON(propertyJsonSchema, isOptional); 136 | builder.field(propertyName, state.schema); 137 | visitors.put(propertyName, state.visitor); 138 | }); 139 | } 140 | } 141 | 142 | static class IntegerSchemaConverter extends FromJsonSchemaConverter { 143 | 144 | IntegerSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) { 145 | super(factory, config); 146 | } 147 | 148 | @Override 149 | protected SchemaBuilder schemaBuilder(NumberSchema schema) { 150 | return SchemaBuilder.int64(); 151 | } 152 | 153 | @Override 154 | protected FromJsonConversionKey key() { 155 | return FromJsonConversionKey.from(NumberSchema.class) 156 | .requiresInteger(true) 157 | .build(); 158 | } 159 | 160 | @Override 161 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) { 162 | return new FromJsonVisitor.IntegerVisitor(connectSchema); 163 | } 164 | 165 | @Override 166 | protected void fromJSON(SchemaBuilder builder, NumberSchema jsonSchema, Map visitors) { 167 | log.trace("fromJson() - Processing '{}'", jsonSchema); 168 | } 169 | } 170 | 171 | static class FloatSchemaConverter extends FromJsonSchemaConverter { 172 | 173 | FloatSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) { 174 | super(factory, config); 175 | } 176 | 177 | @Override 178 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) { 179 | return new FromJsonVisitor.FloatVisitor(connectSchema); 180 | } 181 | 182 | @Override 183 | protected SchemaBuilder schemaBuilder(NumberSchema schema) { 184 | return SchemaBuilder.float64(); 185 | } 186 | 187 | @Override 188 | protected FromJsonConversionKey key() { 189 | return FromJsonConversionKey.from(NumberSchema.class) 190 | .requiresInteger(false) 191 | .build(); 192 | } 193 | 194 | @Override 195 | protected void fromJSON(SchemaBuilder builder, NumberSchema jsonSchema, Map visitors) { 196 | log.trace("fromJson() - Processing '{}'", jsonSchema); 197 | } 198 | } 199 | 200 | static class StringSchemaConverter extends FromJsonSchemaConverter { 201 | 202 | StringSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) { 203 | super(factory, config); 204 | } 205 | 206 | @Override 207 | protected SchemaBuilder schemaBuilder(StringSchema schema) { 208 | return SchemaBuilder.string(); 209 | } 210 | 211 | @Override 212 | protected FromJsonConversionKey key() { 213 | return FromJsonConversionKey.from(StringSchema.class).build(); 214 | } 215 | 216 | @Override 217 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) { 218 | return new FromJsonVisitor.StringVisitor(connectSchema); 219 | } 220 | 221 | @Override 222 | protected void fromJSON(SchemaBuilder builder, StringSchema jsonSchema, Map visitors) { 223 | log.trace("fromJson() - Processing '{}'", jsonSchema); 224 | } 225 | } 226 | 227 | static class DateSchemaConverter extends FromJsonSchemaConverter { 228 | 229 | DateSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) { 230 | super(factory, config); 231 | } 232 | 233 | @Override 234 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) { 235 | return new FromJsonVisitor.DateVisitor(connectSchema); 236 | } 237 | 238 | @Override 239 | protected SchemaBuilder schemaBuilder(StringSchema schema) { 240 | return Date.builder(); 241 | } 242 | 243 | @Override 244 | protected FromJsonConversionKey key() { 245 | return FromJsonConversionKey.from(StringSchema.class) 246 | .format("date") 247 | .build(); 248 | } 249 | 250 | @Override 251 | protected void fromJSON(SchemaBuilder builder, StringSchema jsonSchema, Map visitors) { 252 | log.trace("fromJson() - Processing '{}'", jsonSchema); 253 | } 254 | } 255 | 256 | static class TimeSchemaConverter extends FromJsonSchemaConverter { 257 | 258 | TimeSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) { 259 | super(factory, config); 260 | } 261 | 262 | @Override 263 | protected SchemaBuilder schemaBuilder(StringSchema schema) { 264 | return Time.builder(); 265 | } 266 | 267 | @Override 268 | protected FromJsonConversionKey key() { 269 | return FromJsonConversionKey.from(StringSchema.class) 270 | .format("time") 271 | .build(); 272 | } 273 | 274 | @Override 275 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) { 276 | return new FromJsonVisitor.TimeVisitor(connectSchema); 277 | } 278 | 279 | @Override 280 | protected void fromJSON(SchemaBuilder builder, StringSchema jsonSchema, Map visitors) { 281 | log.trace("fromJson() - Processing '{}'", jsonSchema); 282 | } 283 | } 284 | 285 | static class DateTimeSchemaConverter extends FromJsonSchemaConverter { 286 | 287 | DateTimeSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) { 288 | super(factory, config); 289 | } 290 | 291 | @Override 292 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) { 293 | return new FromJsonVisitor.DateTimeVisitor(connectSchema); 294 | } 295 | 296 | @Override 297 | protected SchemaBuilder schemaBuilder(StringSchema schema) { 298 | return Timestamp.builder(); 299 | } 300 | 301 | @Override 302 | protected FromJsonConversionKey key() { 303 | return FromJsonConversionKey.from(StringSchema.class) 304 | .format("date-time") 305 | .build(); 306 | } 307 | 308 | @Override 309 | protected void fromJSON(SchemaBuilder builder, StringSchema jsonSchema, Map visitors) { 310 | log.trace("fromJson() - Processing '{}'", jsonSchema); 311 | } 312 | } 313 | 314 | static class BytesSchemaConverter extends FromJsonSchemaConverter { 315 | 316 | BytesSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) { 317 | super(factory, config); 318 | } 319 | 320 | @Override 321 | protected SchemaBuilder schemaBuilder(StringSchema schema) { 322 | return SchemaBuilder.bytes(); 323 | } 324 | 325 | @Override 326 | protected FromJsonConversionKey key() { 327 | return FromJsonConversionKey.from(StringSchema.class) 328 | .contentEncoding("base64") 329 | .build(); 330 | } 331 | 332 | @Override 333 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) { 334 | return new FromJsonVisitor.BytesVisitor(connectSchema); 335 | } 336 | 337 | @Override 338 | protected void fromJSON(SchemaBuilder builder, StringSchema jsonSchema, Map visitors) { 339 | 340 | } 341 | } 342 | 343 | static class DecimalSchemaConverter extends FromJsonSchemaConverter { 344 | public DecimalSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) { 345 | super(factory, config); 346 | } 347 | 348 | @Override 349 | protected SchemaBuilder schemaBuilder(StringSchema schema) { 350 | int scale = Utils.scale(schema); 351 | return Decimal.builder(scale); 352 | } 353 | 354 | @Override 355 | protected FromJsonConversionKey key() { 356 | return FromJsonConversionKey.from(StringSchema.class) 357 | .format("decimal") 358 | .build(); 359 | } 360 | 361 | @Override 362 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) { 363 | int scale = Utils.scale(connectSchema); 364 | return new FromJsonVisitor.DecimalVisitor(connectSchema, scale); 365 | } 366 | 367 | @Override 368 | protected void fromJSON(SchemaBuilder builder, StringSchema jsonSchema, Map visitors) { 369 | 370 | } 371 | } 372 | 373 | static class ArraySchemaConverter extends FromJsonSchemaConverter { 374 | 375 | ArraySchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) { 376 | super(factory, config); 377 | } 378 | 379 | @Override 380 | protected SchemaBuilder schemaBuilder(ArraySchema schema) { 381 | FromJsonState state = this.factory.fromJSON(schema.getAllItemSchema()); 382 | return SchemaBuilder.array(state.schema); 383 | } 384 | 385 | @Override 386 | protected FromJsonConversionKey key() { 387 | return FromJsonConversionKey.from(ArraySchema.class).build(); 388 | } 389 | 390 | @Override 391 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) { 392 | FromJsonVisitor visitor = visitors.get("item"); 393 | return new FromJsonVisitor.ArrayVisitor(connectSchema, visitor); 394 | } 395 | 396 | @Override 397 | protected void fromJSON(SchemaBuilder builder, ArraySchema jsonSchema, Map visitors) { 398 | FromJsonState state = this.factory.fromJSON(jsonSchema.getAllItemSchema()); 399 | visitors.put("item", state.visitor); 400 | } 401 | } 402 | 403 | static class CustomTimestampConverter extends FromJsonSchemaConverter { 404 | 405 | CustomTimestampConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) { 406 | super(factory, config); 407 | } 408 | 409 | @Override 410 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) { 411 | return new FromJsonVisitor.CustomDateVisitor(connectSchema); 412 | } 413 | 414 | @Override 415 | protected SchemaBuilder schemaBuilder(StringSchema schema) { 416 | Object dateTimeFormat = schema.getUnprocessedProperties().get("dateTimeFormat"); 417 | Preconditions.checkNotNull(dateTimeFormat, "dateTimeFormat cannot be null"); 418 | return Timestamp.builder() 419 | .parameter("dateFormat", dateTimeFormat.toString()); 420 | } 421 | 422 | @Override 423 | protected FromJsonConversionKey key() { 424 | return FromJsonConversionKey.from(StringSchema.class) 425 | .format("custom-timestamp") 426 | .build(); 427 | } 428 | 429 | @Override 430 | protected void fromJSON(SchemaBuilder builder, StringSchema jsonSchema, Map visitors) { 431 | log.trace("fromJson() - Processing '{}'", jsonSchema); 432 | } 433 | } 434 | } 435 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/FromJsonSchemaConverterFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import com.fasterxml.jackson.databind.JsonNode; 19 | import com.google.common.base.Joiner; 20 | import com.google.common.base.Strings; 21 | import org.apache.kafka.connect.data.SchemaBuilder; 22 | import org.everit.json.schema.CombinedSchema; 23 | import org.everit.json.schema.NullSchema; 24 | import org.everit.json.schema.ObjectSchema; 25 | import org.everit.json.schema.ReferenceSchema; 26 | import org.everit.json.schema.Schema; 27 | import org.everit.json.schema.StringSchema; 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | 31 | import java.util.Collections; 32 | import java.util.LinkedHashMap; 33 | import java.util.List; 34 | import java.util.Map; 35 | import java.util.stream.Collectors; 36 | import java.util.stream.Stream; 37 | 38 | public class FromJsonSchemaConverterFactory { 39 | private static final Logger log = LoggerFactory.getLogger(FromJsonSchemaConverterFactory.class); 40 | private final Map< 41 | FromJsonConversionKey, 42 | FromJsonSchemaConverter 43 | > lookup; 44 | private final JsonConfig config; 45 | private final FromJsonConversionKey genericStringKey; 46 | 47 | public FromJsonSchemaConverterFactory(JsonConfig config) { 48 | this.config = config; 49 | FromJsonSchemaConverter.StringSchemaConverter stringSchemaConverter = new FromJsonSchemaConverter.StringSchemaConverter(this, config); 50 | genericStringKey = stringSchemaConverter.key(); 51 | lookup = Stream.of( 52 | stringSchemaConverter, 53 | new FromJsonSchemaConverter.ObjectSchemaConverter(this, config), 54 | new FromJsonSchemaConverter.IntegerSchemaConverter(this, config), 55 | new FromJsonSchemaConverter.BooleanSchemaConverter(this, config), 56 | new FromJsonSchemaConverter.TimeSchemaConverter(this, config), 57 | new FromJsonSchemaConverter.DateSchemaConverter(this, config), 58 | new FromJsonSchemaConverter.DateTimeSchemaConverter(this, config), 59 | new FromJsonSchemaConverter.FloatSchemaConverter(this, config), 60 | new FromJsonSchemaConverter.ArraySchemaConverter(this, config), 61 | new FromJsonSchemaConverter.BytesSchemaConverter(this, config), 62 | new FromJsonSchemaConverter.DecimalSchemaConverter(this, config), 63 | new FromJsonSchemaConverter.CustomTimestampConverter(this, config) 64 | ).collect(Collectors.toMap(FromJsonSchemaConverter::key, c -> c)); 65 | } 66 | 67 | public FromJsonState fromJSON(org.everit.json.schema.Schema jsonSchema) { 68 | return fromJSON(jsonSchema, false); 69 | } 70 | 71 | public FromJsonState fromJSON(org.everit.json.schema.Schema jsonSchema, boolean isOptional) { 72 | final String description; 73 | 74 | if (jsonSchema instanceof ReferenceSchema) { 75 | ReferenceSchema referenceSchema = (ReferenceSchema) jsonSchema; 76 | jsonSchema = referenceSchema.getReferredSchema(); 77 | description = jsonSchema.getDescription(); 78 | } else if (jsonSchema instanceof CombinedSchema) { 79 | CombinedSchema combinedSchema = (CombinedSchema) jsonSchema; 80 | description = combinedSchema.getDescription(); 81 | List nonNullSubSchemas = combinedSchema 82 | .getSubschemas() 83 | .stream() 84 | .filter(s -> !(s instanceof NullSchema)) 85 | .collect(Collectors.toList()); 86 | if (1 != nonNullSubSchemas.size()) { 87 | throw new UnsupportedOperationException( 88 | String.format( 89 | "More than one choice for non null schemas. Schema location %s: %s", 90 | jsonSchema.getSchemaLocation(), 91 | Joiner.on(", ").join(nonNullSubSchemas) 92 | ) 93 | ); 94 | } 95 | jsonSchema = nonNullSubSchemas.get(0); 96 | } else { 97 | description = jsonSchema.getDescription(); 98 | } 99 | FromJsonConversionKey key = FromJsonConversionKey.of(jsonSchema); 100 | 101 | FromJsonSchemaConverter converter = lookup.get(key); 102 | 103 | if (null == converter && jsonSchema instanceof StringSchema) { 104 | log.trace("fromJSON() - falling back to string passthrough for {}", jsonSchema); 105 | converter = lookup.get(genericStringKey); 106 | } 107 | 108 | if (null == converter) { 109 | throw new UnsupportedOperationException( 110 | String.format("Schema type is not supported. %s:%s", jsonSchema.getClass().getName(), jsonSchema) 111 | ); 112 | } 113 | 114 | SchemaBuilder builder = converter.schemaBuilder(jsonSchema); 115 | if (jsonSchema instanceof ObjectSchema) { 116 | ObjectSchema objectSchema = (ObjectSchema) jsonSchema; 117 | String schemaName = schemaName(objectSchema); 118 | builder.name(schemaName); 119 | } 120 | 121 | if (!Strings.isNullOrEmpty(description)) { 122 | builder.doc(description); 123 | } 124 | if (isOptional) { 125 | builder.optional(); 126 | } 127 | Map visitors = new LinkedHashMap<>(); 128 | converter.fromJSON(builder, jsonSchema, visitors); 129 | org.apache.kafka.connect.data.Schema schema = builder.build(); 130 | FromJsonVisitor visitor = converter.jsonVisitor(schema, visitors); 131 | return FromJsonState.of(jsonSchema, schema, visitor); 132 | } 133 | 134 | private List clean(String text) { 135 | List result; 136 | 137 | if (Strings.isNullOrEmpty(text)) { 138 | result = Collections.EMPTY_LIST; 139 | } else { 140 | result = Stream.of(text.split("[#\\\\/\\.]+")) 141 | .filter(p -> !Strings.isNullOrEmpty(p)) 142 | .collect(Collectors.toList()); 143 | } 144 | 145 | return result; 146 | } 147 | 148 | private String schemaName(ObjectSchema objectSchema) { 149 | final List parts; 150 | 151 | if (!Strings.isNullOrEmpty(objectSchema.getTitle())) { 152 | parts = clean(objectSchema.getTitle()); 153 | } else if (!Strings.isNullOrEmpty(objectSchema.getSchemaLocation())) { 154 | parts = clean(objectSchema.getSchemaLocation()); 155 | } else { 156 | parts = Collections.EMPTY_LIST; 157 | } 158 | 159 | return parts.isEmpty() ? null : Joiner.on('.').join(parts); 160 | } 161 | 162 | private String cleanName(String title) { 163 | String result = title.replaceAll("[#\\\\/]+", "."); 164 | return result; 165 | } 166 | 167 | 168 | } 169 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/FromJsonState.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | 19 | class FromJsonState { 20 | public final org.everit.json.schema.Schema jsonSchema; 21 | public final org.apache.kafka.connect.data.Schema schema; 22 | public final FromJsonVisitor visitor; 23 | 24 | FromJsonState(org.everit.json.schema.Schema jsonSchema, org.apache.kafka.connect.data.Schema schema, FromJsonVisitor visitor) { 25 | this.jsonSchema = jsonSchema; 26 | this.schema = schema; 27 | this.visitor = visitor; 28 | } 29 | 30 | public static FromJsonState of(org.everit.json.schema.Schema jsonSchema, org.apache.kafka.connect.data.Schema schema, FromJsonVisitor visitor) { 31 | return new FromJsonState(jsonSchema, schema, visitor); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/FromJsonVisitor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import com.fasterxml.jackson.databind.JsonNode; 19 | import com.fasterxml.jackson.databind.node.ArrayNode; 20 | import com.fasterxml.jackson.databind.node.BooleanNode; 21 | import com.fasterxml.jackson.databind.node.NumericNode; 22 | import com.fasterxml.jackson.databind.node.ObjectNode; 23 | import com.fasterxml.jackson.databind.node.TextNode; 24 | import com.google.common.io.BaseEncoding; 25 | import org.apache.kafka.connect.data.Schema; 26 | import org.apache.kafka.connect.data.Struct; 27 | import org.apache.kafka.connect.errors.DataException; 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | 31 | import java.text.DecimalFormat; 32 | import java.text.ParseException; 33 | import java.time.Instant; 34 | import java.time.LocalDate; 35 | import java.time.LocalDateTime; 36 | import java.time.LocalTime; 37 | import java.time.ZoneOffset; 38 | import java.time.format.DateTimeFormatter; 39 | import java.util.ArrayList; 40 | import java.util.Date; 41 | import java.util.List; 42 | import java.util.Map; 43 | 44 | public abstract class FromJsonVisitor { 45 | protected final Schema schema; 46 | 47 | protected FromJsonVisitor(Schema schema) { 48 | this.schema = schema; 49 | } 50 | 51 | public V visit(T node) { 52 | V result; 53 | 54 | if (null == node || node.isNull()) { 55 | result = null; 56 | } else { 57 | result = doVisit(node); 58 | } 59 | 60 | return result; 61 | } 62 | 63 | protected abstract V doVisit(T node); 64 | 65 | public static class StringVisitor extends FromJsonVisitor { 66 | public StringVisitor(Schema schema) { 67 | super(schema); 68 | } 69 | 70 | @Override 71 | public String doVisit(TextNode node) { 72 | return node.textValue(); 73 | } 74 | } 75 | 76 | public static class BooleanVisitor extends FromJsonVisitor { 77 | public BooleanVisitor(Schema schema) { 78 | super(schema); 79 | } 80 | 81 | @Override 82 | protected Boolean doVisit(BooleanNode node) { 83 | return node.booleanValue(); 84 | } 85 | } 86 | 87 | public static class StructVisitor extends FromJsonVisitor { 88 | private final Map visitors; 89 | 90 | public StructVisitor(Schema schema, Map visitors) { 91 | super(schema); 92 | this.visitors = visitors; 93 | } 94 | 95 | @Override 96 | protected Struct doVisit(ObjectNode node) { 97 | Struct result = new Struct(this.schema); 98 | visitors.forEach((fieldName, visitor) -> { 99 | try { 100 | JsonNode rawValue = node.get(fieldName); 101 | Object convertedValue = visitor.visit(rawValue); 102 | result.put(fieldName, convertedValue); 103 | } catch (Exception ex) { 104 | throw new IllegalStateException( 105 | String.format("Exception thrown while reading %s:%s", this.schema.name(), fieldName), 106 | ex 107 | ); 108 | } 109 | }); 110 | 111 | return result; 112 | } 113 | } 114 | 115 | public static class IntegerVisitor extends FromJsonVisitor { 116 | public IntegerVisitor(Schema schema) { 117 | super(schema); 118 | } 119 | 120 | @Override 121 | protected Number doVisit(NumericNode node) { 122 | return node.longValue(); 123 | } 124 | } 125 | 126 | public static class FloatVisitor extends FromJsonVisitor { 127 | public FloatVisitor(Schema schema) { 128 | super(schema); 129 | } 130 | 131 | @Override 132 | protected Number doVisit(NumericNode node) { 133 | return node.doubleValue(); 134 | } 135 | } 136 | 137 | public static class DateTimeVisitor extends FromJsonVisitor { 138 | private static final Logger log = LoggerFactory.getLogger(DateTimeVisitor.class); 139 | 140 | public DateTimeVisitor(Schema schema) { 141 | super(schema); 142 | } 143 | 144 | @Override 145 | protected Date doVisit(TextNode node) { 146 | log.trace(node.asText()); 147 | LocalDateTime localDateTime = LocalDateTime.parse(node.asText(), Utils.TIMESTAMP_FORMATTER); 148 | Instant instant = localDateTime.toInstant(ZoneOffset.UTC); 149 | return Date.from(instant); 150 | } 151 | } 152 | 153 | public static class DateVisitor extends FromJsonVisitor { 154 | private static final Logger log = LoggerFactory.getLogger(DateTimeVisitor.class); 155 | 156 | public DateVisitor(Schema schema) { 157 | super(schema); 158 | } 159 | 160 | @Override 161 | protected Date doVisit(TextNode node) { 162 | log.trace(node.asText()); 163 | LocalDate localDateTime = LocalDate.parse(node.asText(), Utils.DATE_FORMATTER); 164 | Instant instant = localDateTime.atStartOfDay().toInstant(ZoneOffset.UTC); 165 | return Date.from(instant); 166 | } 167 | } 168 | 169 | public static class TimeVisitor extends FromJsonVisitor { 170 | private static final Logger log = LoggerFactory.getLogger(DateTimeVisitor.class); 171 | 172 | public TimeVisitor(Schema schema) { 173 | super(schema); 174 | } 175 | 176 | @Override 177 | protected Date doVisit(TextNode node) { 178 | log.trace(node.asText()); 179 | LocalTime localDateTime = LocalTime.parse(node.asText(), Utils.TIME_FORMATTER); 180 | Instant instant = LocalDate.ofEpochDay(0).atTime(localDateTime).toInstant(ZoneOffset.UTC); 181 | return Date.from(instant); 182 | } 183 | } 184 | 185 | public static class DecimalVisitor extends FromJsonVisitor { 186 | final int scale; 187 | final DecimalFormat decimalFormat; 188 | 189 | protected DecimalVisitor(Schema schema, int scale) { 190 | super(schema); 191 | this.scale = scale; 192 | this.decimalFormat = new DecimalFormat("#"); 193 | this.decimalFormat.setParseBigDecimal(true); 194 | this.decimalFormat.setMinimumFractionDigits(scale); 195 | } 196 | 197 | @Override 198 | protected Number doVisit(TextNode node) { 199 | try { 200 | return this.decimalFormat.parse(node.asText()); 201 | } catch (ParseException e) { 202 | throw new DataException(e); 203 | } 204 | } 205 | } 206 | 207 | public static class ArrayVisitor extends FromJsonVisitor { 208 | final FromJsonVisitor itemVisitor; 209 | 210 | public ArrayVisitor(Schema schema, FromJsonVisitor itemVisitor) { 211 | super(schema); 212 | this.itemVisitor = itemVisitor; 213 | } 214 | 215 | @Override 216 | protected List doVisit(ArrayNode node) { 217 | List result = new ArrayList(); 218 | for (JsonNode jsonNode : node) { 219 | Object value = itemVisitor.visit(jsonNode); 220 | result.add(value); 221 | } 222 | return result; 223 | } 224 | } 225 | 226 | public static class BytesVisitor extends FromJsonVisitor { 227 | public BytesVisitor(Schema schema) { 228 | super(schema); 229 | } 230 | 231 | @Override 232 | protected byte[] doVisit(TextNode node) { 233 | return BaseEncoding.base64().decode(node.textValue()); 234 | } 235 | } 236 | 237 | public static class CustomDateVisitor extends FromJsonVisitor { 238 | private static final Logger log = LoggerFactory.getLogger(DateTimeVisitor.class); 239 | 240 | final DateTimeFormatter dateTimeFormatter; 241 | 242 | public CustomDateVisitor(Schema schema) { 243 | super(schema); 244 | String pattern = schema.parameters().get("dateFormat"); 245 | this.dateTimeFormatter = DateTimeFormatter.ofPattern(pattern); 246 | } 247 | 248 | @Override 249 | protected Date doVisit(TextNode node) { 250 | log.trace(node.asText()); 251 | LocalDateTime localDateTime = LocalDateTime.parse(node.asText(), this.dateTimeFormatter); 252 | Instant instant = localDateTime.toInstant(ZoneOffset.UTC); 253 | return Date.from(instant); 254 | } 255 | } 256 | } 257 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/JacksonFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import com.fasterxml.jackson.core.JsonGenerator; 19 | import com.fasterxml.jackson.databind.JsonSerializer; 20 | import com.fasterxml.jackson.databind.ObjectMapper; 21 | import com.fasterxml.jackson.databind.SerializerProvider; 22 | import com.fasterxml.jackson.databind.module.SimpleModule; 23 | 24 | import java.io.IOException; 25 | import java.text.DecimalFormat; 26 | 27 | public class JacksonFactory { 28 | 29 | public static ObjectMapper create() { 30 | ObjectMapper objectMapper = new ObjectMapper(); 31 | return objectMapper; 32 | } 33 | 34 | static class SerializationModule extends SimpleModule { 35 | public SerializationModule() { 36 | addSerializer(double.class, new DoubleSerializer()); 37 | } 38 | } 39 | 40 | static class DoubleSerializer extends JsonSerializer { 41 | final DecimalFormat decimalFormat; 42 | 43 | public DoubleSerializer() { 44 | this.decimalFormat = new DecimalFormat("#"); 45 | // this.df.setMaximumFractionDigits(8); 46 | } 47 | 48 | 49 | @Override 50 | public void serialize(Double aDouble, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException { 51 | jsonGenerator.writeRaw(this.decimalFormat.format(aDouble)); 52 | } 53 | } 54 | 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/JsonConfig.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import com.github.jcustenborder.kafka.connect.utils.config.ConfigKeyBuilder; 19 | import com.github.jcustenborder.kafka.connect.utils.config.ConfigUtils; 20 | import com.github.jcustenborder.kafka.connect.utils.config.Description; 21 | import com.github.jcustenborder.kafka.connect.utils.config.recommenders.Recommenders; 22 | import com.github.jcustenborder.kafka.connect.utils.config.validators.Validators; 23 | import org.apache.kafka.common.config.AbstractConfig; 24 | import org.apache.kafka.common.config.ConfigDef; 25 | 26 | import java.net.URL; 27 | import java.util.Collections; 28 | import java.util.Map; 29 | import java.util.Set; 30 | 31 | class JsonConfig extends AbstractConfig { 32 | public static final String SCHEMA_URL_CONF = "json.schema.url"; 33 | public static final String SCHEMA_INLINE_CONF = "json.schema.inline"; 34 | public static final String SCHEMA_LOCATION_CONF = "json.schema.location"; 35 | public static final String VALIDATE_JSON_ENABLED_CONF = "json.schema.validation.enabled"; 36 | public static final String EXCLUDE_LOCATIONS_CONF = "json.exclude.locations"; 37 | static final String SCHEMA_URL_DOC = "Url to retrieve the schema from. Urls can be anything that is " + 38 | "supported by URL.openStream(). For example the local filesystem file:///schemas/something.json. " + 39 | "A web address https://www.schemas.com/something.json"; 40 | static final String SCHEMA_INLINE_DOC = "The JSON schema to use as an escaped string."; 41 | static final String SCHEMA_LOCATION_DOC = "Location to retrieve the schema from. " + 42 | ConfigUtils.enumDescription(SchemaLocation.class); 43 | static final String VALIDATE_JSON_ENABLED_DOC = "Flag to determine if the JSON should be validated " + 44 | "against the schema."; 45 | static final String EXCLUDE_LOCATIONS_DOC = "Location(s) in the schema to exclude. This is primarily " + 46 | "because connect cannot support those locations. For example types that would require a union type."; 47 | 48 | public JsonConfig(ConfigDef definition, Map originals) { 49 | super(definition, originals); 50 | this.schemaUrl = ConfigUtils.url(this, SCHEMA_URL_CONF); 51 | this.schemaLocation = ConfigUtils.getEnum(SchemaLocation.class, this, SCHEMA_LOCATION_CONF); 52 | this.schemaText = getString(SCHEMA_INLINE_CONF); 53 | this.validateJson = getBoolean(VALIDATE_JSON_ENABLED_CONF); 54 | this.excludeLocations = ConfigUtils.getSet(this, EXCLUDE_LOCATIONS_CONF); 55 | 56 | } 57 | 58 | 59 | public enum SchemaLocation { 60 | @Description("Loads the schema from the url specified in `" + SCHEMA_URL_CONF + "`.") 61 | Url, 62 | @Description("Loads the schema from `" + SCHEMA_INLINE_CONF + "` as an inline string.") 63 | Inline 64 | } 65 | 66 | public final URL schemaUrl; 67 | public final SchemaLocation schemaLocation; 68 | public final String schemaText; 69 | public final boolean validateJson; 70 | public final Set excludeLocations; 71 | 72 | public static ConfigDef config() { 73 | return new ConfigDef().define( 74 | ConfigKeyBuilder.of(SCHEMA_URL_CONF, ConfigDef.Type.STRING) 75 | .documentation(SCHEMA_URL_DOC) 76 | .validator(Validators.validUrl()) 77 | .importance(ConfigDef.Importance.HIGH) 78 | .recommender(Recommenders.visibleIf(SCHEMA_LOCATION_CONF, SchemaLocation.Url.toString())) 79 | .defaultValue("File:///doesNotExist") 80 | .build() 81 | ).define( 82 | ConfigKeyBuilder.of(SCHEMA_LOCATION_CONF, ConfigDef.Type.STRING) 83 | .documentation(SCHEMA_LOCATION_DOC) 84 | .validator(Validators.validEnum(SchemaLocation.class)) 85 | .recommender(Recommenders.enumValues(SchemaLocation.class)) 86 | .importance(ConfigDef.Importance.HIGH) 87 | .defaultValue(SchemaLocation.Url.toString()) 88 | .build() 89 | ).define( 90 | ConfigKeyBuilder.of(VALIDATE_JSON_ENABLED_CONF, ConfigDef.Type.BOOLEAN) 91 | .documentation(VALIDATE_JSON_ENABLED_DOC) 92 | .importance(ConfigDef.Importance.MEDIUM) 93 | .defaultValue(false) 94 | .build() 95 | ).define( 96 | ConfigKeyBuilder.of(SCHEMA_INLINE_CONF, ConfigDef.Type.STRING) 97 | .documentation(SCHEMA_INLINE_DOC) 98 | .recommender(Recommenders.visibleIf(SCHEMA_LOCATION_CONF, SchemaLocation.Inline.toString())) 99 | .importance(ConfigDef.Importance.HIGH) 100 | .defaultValue("") 101 | .build() 102 | ).define( 103 | ConfigKeyBuilder.of(EXCLUDE_LOCATIONS_CONF, ConfigDef.Type.LIST) 104 | .documentation(EXCLUDE_LOCATIONS_DOC) 105 | .defaultValue(Collections.EMPTY_LIST) 106 | .importance(ConfigDef.Importance.LOW) 107 | .build() 108 | ); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/JsonSchemaConverter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import com.fasterxml.jackson.core.JsonGenerator; 19 | import com.fasterxml.jackson.databind.JsonNode; 20 | import com.fasterxml.jackson.databind.ObjectMapper; 21 | import com.google.common.base.Charsets; 22 | import com.google.common.hash.Hashing; 23 | import com.google.common.io.ByteStreams; 24 | import org.apache.kafka.common.config.ConfigException; 25 | import org.apache.kafka.common.errors.SerializationException; 26 | import org.apache.kafka.common.header.Header; 27 | import org.apache.kafka.common.header.Headers; 28 | import org.apache.kafka.common.header.internals.RecordHeader; 29 | import org.apache.kafka.connect.data.Schema; 30 | import org.apache.kafka.connect.data.SchemaAndValue; 31 | import org.apache.kafka.connect.errors.DataException; 32 | import org.apache.kafka.connect.storage.Converter; 33 | 34 | import java.io.ByteArrayOutputStream; 35 | import java.io.IOException; 36 | import java.io.InputStream; 37 | import java.nio.charset.Charset; 38 | import java.util.HashMap; 39 | import java.util.Map; 40 | 41 | public class JsonSchemaConverter implements Converter { 42 | private static final String KEY_HEADER = "json.key.schema"; 43 | private static final String VALUE_HEADER = "json.value.schema"; 44 | JsonSchemaConverterConfig config; 45 | String jsonSchemaHeader; 46 | Charset encodingCharset; 47 | ObjectMapper objectMapper; 48 | FromJsonSchemaConverterFactory fromJsonSchemaConverterFactory; 49 | Map fromConnectStateLookup = new HashMap<>(); 50 | Map toConnectStateLookup = new HashMap<>(); 51 | Header fallbackHeader; 52 | 53 | @Override 54 | public void configure(Map settings, boolean isKey) { 55 | this.config = new JsonSchemaConverterConfig(settings); 56 | this.jsonSchemaHeader = isKey ? KEY_HEADER : VALUE_HEADER; 57 | this.encodingCharset = Charsets.UTF_8; 58 | this.objectMapper = JacksonFactory.create(); 59 | this.fromJsonSchemaConverterFactory = new FromJsonSchemaConverterFactory(config); 60 | 61 | if (this.config.insertSchema) { 62 | byte[] headerValue; 63 | if (JsonConfig.SchemaLocation.Url == this.config.schemaLocation) { 64 | try { 65 | try (InputStream inputStream = this.config.schemaUrl.openStream()) { 66 | headerValue = ByteStreams.toByteArray(inputStream); 67 | } 68 | } catch (IOException e) { 69 | ConfigException exception = new ConfigException(JsonConfig.SCHEMA_URL_CONF, this.config.schemaUrl, "exception while loading schema"); 70 | exception.initCause(e); 71 | throw exception; 72 | } 73 | } else if (JsonConfig.SchemaLocation.Inline == this.config.schemaLocation) { 74 | headerValue = this.jsonSchemaHeader.getBytes(Charsets.UTF_8); 75 | } else { 76 | throw new ConfigException( 77 | JsonConfig.SCHEMA_LOCATION_CONF, 78 | this.config.schemaLocation.toString(), 79 | "Location is not supported" 80 | ); 81 | } 82 | this.fallbackHeader = new RecordHeader(this.jsonSchemaHeader, headerValue); 83 | } else { 84 | fallbackHeader = null; 85 | } 86 | } 87 | 88 | @Override 89 | public byte[] fromConnectData(String s, Schema schema, Object o) { 90 | throw new UnsupportedOperationException( 91 | "This converter requires Kafka 2.4.0 or higher with header support." 92 | ); 93 | } 94 | 95 | @Override 96 | public byte[] fromConnectData(String topic, Headers headers, Schema schema, Object value) { 97 | if (null == value) { 98 | return null; 99 | } 100 | 101 | 102 | try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { 103 | try (JsonGenerator jsonGenerator = objectMapper.getFactory().createGenerator(outputStream)) { 104 | FromConnectState fromConnectState = fromConnectStateLookup.computeIfAbsent(schema, s -> FromConnectSchemaConverter.toJsonSchema(schema, jsonSchemaHeader)); 105 | headers.add(fromConnectState.header); 106 | fromConnectState.visitor.doVisit(jsonGenerator, value); 107 | } 108 | return outputStream.toByteArray(); 109 | } catch (IOException ex) { 110 | throw new SerializationException(ex); 111 | } 112 | } 113 | 114 | @Override 115 | public SchemaAndValue toConnectData(String s, byte[] bytes) { 116 | throw new UnsupportedOperationException( 117 | "This converter requires Kafka 2.4.0 or higher with header support." 118 | ); 119 | } 120 | 121 | Header schemaHeader(Headers headers) { 122 | Header schemaHeader = headers.lastHeader(this.jsonSchemaHeader); 123 | if (null == schemaHeader) { 124 | schemaHeader = this.fallbackHeader; 125 | } 126 | return schemaHeader; 127 | } 128 | 129 | @Override 130 | public SchemaAndValue toConnectData(String topic, Headers headers, byte[] value) { 131 | if (null == value) { 132 | return SchemaAndValue.NULL; 133 | } 134 | 135 | final Header schemaHeader = schemaHeader(headers); 136 | 137 | if (null == schemaHeader) { 138 | throw new DataException( 139 | String.format( 140 | "Record does not have '{}' header and '%s' is not enabled.", 141 | this.jsonSchemaHeader, 142 | JsonSchemaConverterConfig.INSERT_SCHEMA_ENABLED_CONF 143 | ) 144 | ); 145 | } 146 | 147 | String hash = Hashing.goodFastHash(32) 148 | .hashBytes(schemaHeader.value()) 149 | .toString(); 150 | FromJsonState state = this.toConnectStateLookup.computeIfAbsent(hash, h -> { 151 | org.everit.json.schema.Schema schema = Utils.loadSchema(schemaHeader); 152 | return this.fromJsonSchemaConverterFactory.fromJSON(schema); 153 | }); 154 | 155 | JsonNode jsonNode; 156 | try { 157 | jsonNode = this.objectMapper.readValue(value, JsonNode.class); 158 | } catch (IOException ex) { 159 | throw new SerializationException(ex); 160 | } 161 | Object result = state.visitor.visit(jsonNode); 162 | return new SchemaAndValue(state.schema, result); 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/JsonSchemaConverterConfig.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import com.github.jcustenborder.kafka.connect.utils.config.ConfigKeyBuilder; 19 | import org.apache.kafka.common.config.ConfigDef; 20 | 21 | import java.util.Map; 22 | 23 | class JsonSchemaConverterConfig extends JsonConfig { 24 | public final boolean insertSchema; 25 | 26 | public final static String INSERT_SCHEMA_ENABLED_CONF = "json.insert.schema.enabled"; 27 | final static String INSERT_SCHEMA_ENABLED_DOC = "Flag to determine if the schema specified should be " + 28 | "used if there is no schema header found. This allows a connector to consume a topic " + 29 | "that does not have schema headers and apply an external header."; 30 | 31 | public JsonSchemaConverterConfig(Map originals) { 32 | super(config(), originals); 33 | this.insertSchema = getBoolean(INSERT_SCHEMA_ENABLED_CONF); 34 | } 35 | 36 | public static ConfigDef config() { 37 | return JsonConfig.config() 38 | .define( 39 | ConfigKeyBuilder.of(INSERT_SCHEMA_ENABLED_CONF, ConfigDef.Type.BOOLEAN) 40 | .documentation(INSERT_SCHEMA_ENABLED_DOC) 41 | .importance(ConfigDef.Importance.HIGH) 42 | .defaultValue(false) 43 | .build() 44 | ); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/Utils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import org.apache.kafka.common.header.Header; 19 | import org.apache.kafka.connect.data.Decimal; 20 | import org.apache.kafka.connect.errors.DataException; 21 | import org.everit.json.schema.Schema; 22 | import org.everit.json.schema.StringSchema; 23 | import org.everit.json.schema.internal.DateFormatValidator; 24 | import org.everit.json.schema.internal.DateTimeFormatValidator; 25 | import org.everit.json.schema.internal.TimeFormatValidator; 26 | import org.everit.json.schema.loader.SchemaLoader; 27 | import org.json.JSONObject; 28 | import org.json.JSONTokener; 29 | 30 | import java.io.ByteArrayInputStream; 31 | import java.io.IOException; 32 | import java.io.InputStream; 33 | import java.io.Reader; 34 | import java.io.StringReader; 35 | import java.time.ZoneId; 36 | import java.time.format.DateTimeFormatter; 37 | 38 | public class Utils { 39 | 40 | static final ZoneId ZONE_ID = ZoneId.of("UTC"); 41 | public static final DateTimeFormatter TIMESTAMP_FORMATTER = DateTimeFormatter.ISO_INSTANT 42 | .withZone(ZONE_ID); 43 | public static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ISO_LOCAL_DATE 44 | .withZone(ZONE_ID); 45 | public static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ISO_LOCAL_TIME 46 | .withZone(ZONE_ID); 47 | 48 | 49 | public static JSONObject loadObject(Header header) { 50 | try (InputStream inputStream = new ByteArrayInputStream(header.value())) { 51 | return loadObject(inputStream); 52 | } catch (IOException ex) { 53 | throw new DataException("Could not load schema", ex); 54 | } 55 | } 56 | 57 | public static JSONObject loadObject(InputStream inputStream) { 58 | return new JSONObject(new JSONTokener(inputStream)); 59 | } 60 | 61 | public static Schema loadSchema(InputStream inputStream) { 62 | JSONObject rawSchema = loadObject(inputStream); 63 | return loadSchema(rawSchema); 64 | } 65 | 66 | public static org.everit.json.schema.Schema loadSchema(JSONObject rawSchema) { 67 | return SchemaLoader.builder() 68 | .draftV7Support() 69 | .addFormatValidator(new DateFormatValidator()) 70 | .addFormatValidator(new TimeFormatValidator()) 71 | .addFormatValidator(new DateTimeFormatValidator()) 72 | .addFormatValidator(new DecimalFormatValidator()) 73 | .addFormatValidator(new CustomTimestampFormatValidator()) 74 | .schemaJson(rawSchema) 75 | .build() 76 | .load() 77 | .build(); 78 | } 79 | 80 | public static org.everit.json.schema.Schema loadSchema(Header header) { 81 | JSONObject rawSchema = loadObject(header); 82 | return loadSchema(rawSchema); 83 | } 84 | 85 | 86 | public static int scale(StringSchema schema) { 87 | String scale = schema.getUnprocessedProperties().get("scale").toString(); 88 | return scale(scale); 89 | } 90 | 91 | private static int scale(String scale) { 92 | return Integer.parseInt(scale); 93 | } 94 | 95 | public static int scale(org.apache.kafka.connect.data.Schema connectSchema) { 96 | String scale = connectSchema.parameters().get(Decimal.SCALE_FIELD); 97 | return scale(scale); 98 | } 99 | 100 | public static JSONObject loadObject(Reader reader) { 101 | return new JSONObject(new JSONTokener(reader)); 102 | } 103 | 104 | public static Schema loadSchema(String schemaText) { 105 | try (Reader reader = new StringReader(schemaText)) { 106 | JSONObject rawSchema = loadObject(reader); 107 | return loadSchema(rawSchema); 108 | } catch (IOException ex) { 109 | throw new DataException("Could not load schema", ex); 110 | } 111 | } 112 | 113 | 114 | } 115 | -------------------------------------------------------------------------------- /src/main/java/com/github/jcustenborder/kafka/connect/json/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | @Introduction("This plugin is used to add additional JSON parsing functionality to Kafka Connect.") 17 | @Title("Json Schema") 18 | @PluginOwner("jcustenborder") 19 | @PluginName("kafka-connect-json-schema") 20 | package com.github.jcustenborder.kafka.connect.json; 21 | 22 | import com.github.jcustenborder.kafka.connect.utils.config.Introduction; 23 | import com.github.jcustenborder.kafka.connect.utils.config.PluginName; 24 | import com.github.jcustenborder.kafka.connect.utils.config.PluginOwner; 25 | import com.github.jcustenborder.kafka.connect.utils.config.Title; -------------------------------------------------------------------------------- /src/test/java/com/github/jcustenborder/kafka/connect/json/DocumentationTest.java: -------------------------------------------------------------------------------- 1 | package com.github.jcustenborder.kafka.connect.json; 2 | 3 | import com.github.jcustenborder.kafka.connect.utils.BaseDocumentationTest; 4 | 5 | public class DocumentationTest extends BaseDocumentationTest { 6 | 7 | } 8 | -------------------------------------------------------------------------------- /src/test/java/com/github/jcustenborder/kafka/connect/json/FromConnectSchemaConverterTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import com.google.common.base.Strings; 19 | import org.apache.kafka.connect.data.Schema; 20 | import org.apache.kafka.connect.data.SchemaBuilder; 21 | import org.json.JSONObject; 22 | import org.junit.jupiter.api.DynamicTest; 23 | import org.junit.jupiter.api.Test; 24 | import org.junit.jupiter.api.TestFactory; 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | 28 | import java.util.stream.Stream; 29 | 30 | import static org.junit.jupiter.api.Assertions.assertEquals; 31 | import static org.junit.jupiter.api.Assertions.assertNotNull; 32 | import static org.junit.jupiter.api.DynamicTest.dynamicTest; 33 | 34 | public class FromConnectSchemaConverterTest { 35 | private static final Logger log = LoggerFactory.getLogger(FromConnectSchemaConverterTest.class); 36 | 37 | @Test 38 | public void array() { 39 | Schema addressSchema = SchemaBuilder.struct() 40 | .name("Address") 41 | .optional() 42 | .field("city", SchemaBuilder.string().build()) 43 | .field("state", SchemaBuilder.string().build()) 44 | .field("street_address", SchemaBuilder.string().build()) 45 | .build(); 46 | Schema arraySchema = SchemaBuilder.array(addressSchema); 47 | Schema expected = SchemaBuilder.struct() 48 | .name("Person") 49 | .field("previous_addresses", arraySchema) 50 | .build(); 51 | FromConnectState state = FromConnectSchemaConverter.toJsonSchema(expected, "foo"); 52 | assertNotNull(state, "rawJsonSchema should not be null."); 53 | JSONObject rawJsonSchema = Utils.loadObject(state.header); 54 | log.trace("rawJsonSchema = {}", rawJsonSchema.toString(2)); 55 | org.everit.json.schema.Schema jsonSchema = TestUtils.jsonSchema(rawJsonSchema); 56 | assertNotNull(jsonSchema); 57 | } 58 | 59 | @Test 60 | public void struct() { 61 | Schema addressSchema = SchemaBuilder.struct() 62 | .name("Address") 63 | .doc("An object to store an address.") 64 | .optional() 65 | .field("city", SchemaBuilder.string().doc("city of the address.").build()) 66 | .field("state", SchemaBuilder.string().doc("state of the address.").build()) 67 | .field("street_address", SchemaBuilder.string().doc("street address of the address.").build()) 68 | .build(); 69 | Schema expected = SchemaBuilder.struct() 70 | .name("Customer") 71 | .field("first_name", SchemaBuilder.string().doc("First name of the customer").build()) 72 | .field("billing_address", addressSchema) 73 | .field("shipping_address", addressSchema) 74 | .build(); 75 | 76 | FromConnectState state = FromConnectSchemaConverter.toJsonSchema(expected, "foo"); 77 | assertNotNull(state, "rawJsonSchema should not be null."); 78 | JSONObject rawJsonSchema = Utils.loadObject(state.header); 79 | log.trace("rawJsonSchema = {}", rawJsonSchema.toString(2)); 80 | org.everit.json.schema.Schema jsonSchema = TestUtils.jsonSchema(rawJsonSchema); 81 | assertNotNull(jsonSchema); 82 | } 83 | 84 | @TestFactory 85 | public Stream primitives() { 86 | return FromConnectSchemaConverter.PRIMITIVE_TYPES.entrySet() 87 | .stream() 88 | .filter(e-> Strings.isNullOrEmpty(e.getKey().schemaName)) 89 | .map(e -> dynamicTest(e.getKey().toString(), () -> { 90 | String description = String.format("This schema represents a %s", e.getKey()); 91 | Schema expected = SchemaBuilder.type(e.getKey().type) 92 | .doc(description) 93 | .build(); 94 | FromConnectState state = FromConnectSchemaConverter.toJsonSchema(expected, "foo"); 95 | assertNotNull(state, "rawJsonSchema should not be null."); 96 | JSONObject rawJsonSchema = Utils.loadObject(state.header); 97 | assertNotNull(rawJsonSchema, "rawJsonSchema should not be null."); 98 | log.trace("rawJsonSchema = {}", rawJsonSchema.toString(2)); 99 | e.getValue().forEach((propertyName, expectedValue) -> { 100 | Object propertyValue = rawJsonSchema.get(propertyName); 101 | assertEquals(expectedValue, propertyValue); 102 | }); 103 | assertEquals(description, rawJsonSchema.getString("description")); 104 | org.everit.json.schema.Schema jsonSchema = TestUtils.jsonSchema(rawJsonSchema); 105 | 106 | })); 107 | } 108 | 109 | 110 | } 111 | -------------------------------------------------------------------------------- /src/test/java/com/github/jcustenborder/kafka/connect/json/FromJsonSchemaConverterTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import com.google.common.collect.ImmutableMap; 19 | import org.apache.kafka.connect.data.Date; 20 | import org.apache.kafka.connect.data.Schema; 21 | import org.apache.kafka.connect.data.SchemaBuilder; 22 | import org.apache.kafka.connect.data.Time; 23 | import org.apache.kafka.connect.data.Timestamp; 24 | import org.everit.json.schema.internal.DateFormatValidator; 25 | import org.everit.json.schema.internal.DateTimeFormatValidator; 26 | import org.everit.json.schema.internal.TimeFormatValidator; 27 | import org.everit.json.schema.loader.SchemaLoader; 28 | import org.json.JSONObject; 29 | import org.json.JSONTokener; 30 | import org.junit.jupiter.api.BeforeEach; 31 | import org.junit.jupiter.api.DynamicTest; 32 | import org.junit.jupiter.api.Test; 33 | import org.junit.jupiter.api.TestFactory; 34 | import org.slf4j.Logger; 35 | import org.slf4j.LoggerFactory; 36 | 37 | import java.io.IOException; 38 | import java.io.InputStream; 39 | import java.util.LinkedHashMap; 40 | import java.util.Map; 41 | import java.util.stream.Stream; 42 | 43 | import static com.github.jcustenborder.kafka.connect.utils.AssertSchema.assertSchema; 44 | import static org.junit.jupiter.api.DynamicTest.dynamicTest; 45 | 46 | public class FromJsonSchemaConverterTest { 47 | private static final Logger log = LoggerFactory.getLogger(FromJsonSchemaConverterTest.class); 48 | 49 | JsonConfig config; 50 | FromJsonSchemaConverterFactory factory; 51 | 52 | @BeforeEach 53 | public void before() { 54 | config = new FromJsonConfig(ImmutableMap.of( 55 | FromJsonConfig.SCHEMA_INLINE_CONF, "\"string\"", 56 | FromJsonConfig.SCHEMA_LOCATION_CONF, JsonConfig.SchemaLocation.Inline.toString(), 57 | JsonConfig.EXCLUDE_LOCATIONS_CONF, "#/properties/log_params" 58 | )); 59 | this.factory = new FromJsonSchemaConverterFactory(config); 60 | } 61 | 62 | 63 | org.everit.json.schema.Schema jsonSchema(String type) { 64 | JSONObject rawSchema = new JSONObject(); 65 | rawSchema.put("type", type); 66 | return TestUtils.jsonSchema(rawSchema); 67 | } 68 | 69 | org.everit.json.schema.Schema jsonSchema(String type, String key1, String value1) { 70 | JSONObject rawSchema = new JSONObject(); 71 | rawSchema.put("type", type); 72 | rawSchema.put(key1, value1); 73 | return TestUtils.jsonSchema(rawSchema); 74 | } 75 | 76 | org.everit.json.schema.Schema jsonSchema(String type, String key1, String value1, String key2, String value2) { 77 | JSONObject rawSchema = new JSONObject(); 78 | rawSchema.put("type", type); 79 | rawSchema.put(key1, value1); 80 | rawSchema.put(key2, value2); 81 | return TestUtils.jsonSchema(rawSchema); 82 | } 83 | 84 | 85 | void assertJsonSchema(org.apache.kafka.connect.data.Schema expected, org.everit.json.schema.Schema input) { 86 | FromJsonState state = this.factory.fromJSON(input); 87 | 88 | 89 | log.trace("schema:\n{}", state.schema); 90 | assertSchema(expected, state.schema); 91 | } 92 | 93 | @Test 94 | public void booleanSchema() { 95 | org.everit.json.schema.Schema jsonSchema = jsonSchema("boolean"); 96 | assertJsonSchema(Schema.BOOLEAN_SCHEMA, jsonSchema); 97 | } 98 | 99 | @Test 100 | public void stringSchema() { 101 | org.everit.json.schema.Schema jsonSchema = jsonSchema("string"); 102 | assertJsonSchema(Schema.STRING_SCHEMA, jsonSchema); 103 | } 104 | 105 | @Test 106 | public void integerSchema() { 107 | org.everit.json.schema.Schema jsonSchema = jsonSchema("integer"); 108 | assertJsonSchema(Schema.INT64_SCHEMA, jsonSchema); 109 | } 110 | 111 | @Test 112 | public void numberSchema() { 113 | org.everit.json.schema.Schema jsonSchema = jsonSchema("number"); 114 | assertJsonSchema(Schema.FLOAT64_SCHEMA, jsonSchema); 115 | } 116 | 117 | @Test 118 | public void dateSchema() { 119 | org.everit.json.schema.Schema jsonSchema = jsonSchema("string", "format", "date"); 120 | assertJsonSchema(Date.SCHEMA, jsonSchema); 121 | } 122 | 123 | @Test 124 | public void timeSchema() { 125 | org.everit.json.schema.Schema jsonSchema = jsonSchema("string", "format", "time"); 126 | assertJsonSchema(Time.SCHEMA, jsonSchema); 127 | } 128 | 129 | @Test 130 | public void datetimeSchema() { 131 | org.everit.json.schema.Schema jsonSchema = jsonSchema("string", "format", "date-time"); 132 | assertJsonSchema(Timestamp.SCHEMA, jsonSchema); 133 | } 134 | 135 | org.everit.json.schema.Schema loadSchema(String name) throws IOException { 136 | try (InputStream inputStream = this.getClass().getResourceAsStream(name)) { 137 | JSONObject rawSchema = new JSONObject(new JSONTokener(inputStream)); 138 | return SchemaLoader.builder() 139 | .draftV7Support() 140 | .addFormatValidator(new DateFormatValidator()) 141 | .addFormatValidator(new TimeFormatValidator()) 142 | .addFormatValidator(new DateTimeFormatValidator()) 143 | .schemaJson(rawSchema) 144 | .build() 145 | .load() 146 | .build(); 147 | } 148 | } 149 | 150 | @Test 151 | public void productSchema() throws IOException { 152 | org.everit.json.schema.Schema jsonSchema = loadSchema("SchemaConverterTest/product.schema.json"); 153 | Schema expected = SchemaBuilder.struct() 154 | .name("Product") 155 | .doc("A product from Acme's catalog") 156 | .field("price", SchemaBuilder.float64().doc("The price of the product").build()) 157 | .field("productId", SchemaBuilder.int64().doc("The unique identifier for a product").build()) 158 | .field("productName", SchemaBuilder.string().doc("Name of the product").build()) 159 | .build(); 160 | assertJsonSchema(expected, jsonSchema); 161 | } 162 | 163 | @Test 164 | public void wikiMediaRecentChangeSchema() throws IOException { 165 | org.everit.json.schema.Schema jsonSchema = loadSchema("SchemaConverterTest/wikimedia.recentchange.schema.json"); 166 | Schema propertiesLength = SchemaBuilder.struct() 167 | .name("properties.length") 168 | .optional() 169 | .doc("Length of old and new change") 170 | .field("new", SchemaBuilder.int64().doc("(rc_new_len)").optional().build()) 171 | .field("old", SchemaBuilder.int64().doc("(rc_old_len)").optional().build()) 172 | .build(); 173 | Schema propertiesMeta = SchemaBuilder.struct() 174 | .name("properties.meta") 175 | .field("domain", SchemaBuilder.string().optional().doc("Domain the event or entity pertains to").build()) 176 | .field("dt", Timestamp.builder().doc("Event datetime, in ISO-8601 format").build()) 177 | .field("id", SchemaBuilder.string().doc("Unique ID of this event").build()) 178 | .field("request_id", SchemaBuilder.string().optional().doc("Unique ID of the request that caused the event").build()) 179 | .field("stream", SchemaBuilder.string().doc("Name of the stream/queue/dataset that this event belongs in").build()) 180 | .field("uri", SchemaBuilder.string().optional().doc("Unique URI identifying the event or entity").build()) 181 | .build(); 182 | Schema propertiesRevision = SchemaBuilder.struct() 183 | .name("properties.revision") 184 | .optional() 185 | .doc("Old and new revision IDs") 186 | .field("new", SchemaBuilder.int64().doc("(rc_last_oldid)").optional().build()) 187 | .field("old", SchemaBuilder.int64().doc("(rc_this_oldid)").optional().build()) 188 | .build(); 189 | 190 | 191 | Schema expected = SchemaBuilder.struct() 192 | .name("mediawiki.recentchange") 193 | .doc("Represents a MW RecentChange event. https://www.mediawiki.org/wiki/Manual:RCFeed\n") 194 | .field("bot", SchemaBuilder.bool().optional().doc("(rc_bot)").build()) 195 | .field("comment", SchemaBuilder.string().optional().doc("(rc_comment)").build()) 196 | .field("id", SchemaBuilder.int64().optional().doc("ID of the recentchange event (rcid).").build()) 197 | .field("length", propertiesLength) 198 | .field("log_action", SchemaBuilder.string().optional().doc("(rc_log_action)").build()) 199 | .field("log_action_comment", SchemaBuilder.string().optional().build()) 200 | .field("log_id", SchemaBuilder.int64().optional().doc("(rc_log_id)").build()) 201 | .field("log_type", SchemaBuilder.string().optional().doc("(rc_log_type)").build()) 202 | .field("meta", propertiesMeta) 203 | .field("minor", SchemaBuilder.bool().optional().doc("(rc_minor).").build()) 204 | .field("namespace", SchemaBuilder.int64().optional().doc("ID of relevant namespace of affected page (rc_namespace, page_namespace). This is -1 (\"Special\") for log events.\n").build()) 205 | .field("parsedcomment", SchemaBuilder.string().optional().doc("The rc_comment parsed into simple HTML. Optional").build()) 206 | .field("patrolled", SchemaBuilder.bool().optional().doc("(rc_patrolled). This property only exists if patrolling is supported for this event (based on $wgUseRCPatrol, $wgUseNPPatrol).\n").build()) 207 | .field("revision", propertiesRevision) 208 | .field("server_name", SchemaBuilder.string().optional().doc("$wgServerName").build()) 209 | .field("server_script_path", SchemaBuilder.string().optional().doc("$wgScriptPath").build()) 210 | .field("server_url", SchemaBuilder.string().optional().doc("$wgCanonicalServer").build()) 211 | .field("timestamp", SchemaBuilder.int64().optional().doc("Unix timestamp (derived from rc_timestamp).").build()) 212 | .field("title", SchemaBuilder.string().optional().doc("Full page name, from Title::getPrefixedText.").build()) 213 | .field("type", SchemaBuilder.string().optional().doc("Type of recentchange event (rc_type). One of \"edit\", \"new\", \"log\", \"categorize\", or \"external\". (See Manual:Recentchanges table#rc_type)\n").build()) 214 | .field("user", SchemaBuilder.string().optional().doc("(rc_user_text)").build()) 215 | .field("wiki", SchemaBuilder.string().optional().doc("wfWikiID ($wgDBprefix, $wgDBname)").build()) 216 | 217 | .build(); 218 | 219 | 220 | assertJsonSchema(expected, jsonSchema); 221 | } 222 | 223 | @Test 224 | public void nested() throws IOException { 225 | 226 | org.everit.json.schema.Schema jsonSchema = loadSchema("SchemaConverterTest/nested.schema.json"); 227 | Schema addressSchema = SchemaBuilder.struct() 228 | .name("Address") 229 | .optional() 230 | .field("city", SchemaBuilder.string().build()) 231 | .field("state", SchemaBuilder.string().build()) 232 | .field("street_address", SchemaBuilder.string().build()) 233 | .build(); 234 | Schema expected = SchemaBuilder.struct() 235 | .name("Customer") 236 | .field("billing_address", addressSchema) 237 | .field("shipping_address", addressSchema) 238 | .build(); 239 | assertJsonSchema(expected, jsonSchema); 240 | } 241 | 242 | @Test 243 | public void array() { 244 | JSONObject rawSchema = new JSONObject() 245 | .put("type", "array") 246 | .put("items", new JSONObject().put("type", "number")); 247 | org.everit.json.schema.Schema jsonSchema = TestUtils.jsonSchema(rawSchema); 248 | assertJsonSchema(SchemaBuilder.array(Schema.FLOAT64_SCHEMA).build(), jsonSchema); 249 | } 250 | 251 | @TestFactory 252 | public Stream stringFormats() { 253 | Map formats = new LinkedHashMap<>(); 254 | formats.put("email", "test@example.com"); 255 | formats.put("idn-email", "test@example.com"); 256 | formats.put("hostname", "example.com"); 257 | formats.put("idn-hostname", "example.com"); 258 | formats.put("ipv4", "127.0.0.1"); 259 | formats.put("ipv6", "::1"); 260 | formats.put("uri", "http://example.com"); 261 | formats.put("uri-reference", "http://example.com"); 262 | formats.put("iri", "http://example.com"); 263 | formats.put("iri-reference", "http://example.com"); 264 | formats.put("uri-template", "http://example.com/~{username}/"); 265 | 266 | return formats.entrySet() 267 | .stream() 268 | .map(e -> dynamicTest(e.getKey(), () -> { 269 | JSONObject rawSchema = new JSONObject() 270 | .put("type", "string") 271 | .put("format", e.getKey()); 272 | log.trace("schema = '{}'", rawSchema); 273 | org.everit.json.schema.Schema jsonSchema = TestUtils.jsonSchema(rawSchema); 274 | assertJsonSchema(Schema.STRING_SCHEMA, jsonSchema); 275 | })); 276 | } 277 | 278 | } 279 | -------------------------------------------------------------------------------- /src/test/java/com/github/jcustenborder/kafka/connect/json/FromJsonTest.java: -------------------------------------------------------------------------------- 1 | package com.github.jcustenborder.kafka.connect.json; 2 | 3 | import com.github.jcustenborder.kafka.connect.utils.SinkRecordHelper; 4 | import com.google.common.collect.ImmutableMap; 5 | import com.google.common.io.ByteStreams; 6 | import org.apache.kafka.connect.data.Schema; 7 | import org.apache.kafka.connect.data.SchemaAndValue; 8 | import org.apache.kafka.connect.data.Struct; 9 | import org.apache.kafka.connect.errors.DataException; 10 | import org.apache.kafka.connect.sink.SinkRecord; 11 | import org.junit.jupiter.api.BeforeEach; 12 | import org.junit.jupiter.api.Test; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import java.io.File; 17 | import java.io.IOException; 18 | import java.time.LocalDateTime; 19 | import java.time.ZoneId; 20 | import java.time.ZonedDateTime; 21 | import java.time.format.DateTimeFormatter; 22 | import java.util.Map; 23 | 24 | import static org.junit.jupiter.api.Assertions.assertNotNull; 25 | import static org.junit.jupiter.api.Assertions.assertThrows; 26 | import static org.junit.jupiter.api.Assertions.assertTrue; 27 | 28 | public class FromJsonTest { 29 | private static final Logger log = LoggerFactory.getLogger(FromJsonTest.class); 30 | FromJson transform; 31 | 32 | @BeforeEach 33 | public void beforeEach() { 34 | this.transform = new FromJson.Value<>(); 35 | } 36 | 37 | @Test 38 | public void basic() throws IOException { 39 | byte[] input = ByteStreams.toByteArray(this.getClass().getResourceAsStream( 40 | "basic.data.json" 41 | )); 42 | File schemaFile = new File("src/test/resources/com/github/jcustenborder/kafka/connect/json/basic.schema.json"); 43 | Map settings = ImmutableMap.of( 44 | JsonConfig.SCHEMA_URL_CONF, schemaFile.toURI().toString() 45 | ); 46 | this.transform.configure(settings); 47 | SinkRecord inputRecord = SinkRecordHelper.write("foo", new SchemaAndValue(Schema.STRING_SCHEMA, "foo"), new SchemaAndValue(Schema.BYTES_SCHEMA, input)); 48 | SinkRecord transformedRecord = this.transform.apply(inputRecord); 49 | assertNotNull(transformedRecord); 50 | assertNotNull(transformedRecord.value()); 51 | assertTrue(transformedRecord.value() instanceof Struct); 52 | Struct actual = (Struct) transformedRecord.value(); 53 | log.info("actual = '{}'", actual); 54 | } 55 | 56 | @Test 57 | public void customdate() throws IOException { 58 | byte[] input = ByteStreams.toByteArray(this.getClass().getResourceAsStream( 59 | "customdate.data.json" 60 | )); 61 | File schemaFile = new File("src/test/resources/com/github/jcustenborder/kafka/connect/json/customdate.schema.json"); 62 | Map settings = ImmutableMap.of( 63 | JsonConfig.SCHEMA_URL_CONF, schemaFile.toURI().toString() 64 | ); 65 | this.transform.configure(settings); 66 | SinkRecord inputRecord = SinkRecordHelper.write("foo", new SchemaAndValue(Schema.STRING_SCHEMA, "foo"), new SchemaAndValue(Schema.BYTES_SCHEMA, input)); 67 | SinkRecord transformedRecord = this.transform.apply(inputRecord); 68 | assertNotNull(transformedRecord); 69 | assertNotNull(transformedRecord.value()); 70 | assertTrue(transformedRecord.value() instanceof Struct); 71 | Struct actual = (Struct) transformedRecord.value(); 72 | log.info("actual = '{}'", actual); 73 | } 74 | 75 | @Test 76 | public void validate() throws IOException { 77 | byte[] input = ByteStreams.toByteArray(this.getClass().getResourceAsStream( 78 | "basic.data.json" 79 | )); 80 | File schemaFile = new File("src/test/resources/com/github/jcustenborder/kafka/connect/json/geo.schema.json"); 81 | Map settings = ImmutableMap.of( 82 | JsonConfig.SCHEMA_URL_CONF, schemaFile.toURI().toString(), 83 | JsonConfig.VALIDATE_JSON_ENABLED_CONF, "true" 84 | ); 85 | this.transform.configure(settings); 86 | SinkRecord inputRecord = SinkRecordHelper.write("foo", new SchemaAndValue(Schema.STRING_SCHEMA, "foo"), new SchemaAndValue(Schema.BYTES_SCHEMA, input)); 87 | DataException exception = assertThrows(DataException.class, () -> { 88 | SinkRecord transformedRecord = this.transform.apply(inputRecord); 89 | }); 90 | 91 | assertTrue(exception.getMessage().contains("required key [latitude] not found")); 92 | assertTrue(exception.getMessage().contains("required key [longitude] not found")); 93 | } 94 | @Test 95 | public void wikiMediaRecentChange() throws IOException { 96 | byte[] input = ByteStreams.toByteArray(this.getClass().getResourceAsStream( 97 | "wikimedia.recentchange.data.json" 98 | )); 99 | File schemaFile = new File("src/test/resources/com/github/jcustenborder/kafka/connect/json/wikimedia.recentchange.schema.json"); 100 | Map settings = ImmutableMap.of( 101 | JsonConfig.SCHEMA_URL_CONF, schemaFile.toURI().toString(), 102 | JsonConfig.VALIDATE_JSON_ENABLED_CONF, "true", 103 | JsonConfig.EXCLUDE_LOCATIONS_CONF, "#/properties/log_params" 104 | ); 105 | this.transform.configure(settings); 106 | SinkRecord inputRecord = SinkRecordHelper.write("foo", new SchemaAndValue(Schema.STRING_SCHEMA, "foo"), new SchemaAndValue(Schema.BYTES_SCHEMA, input)); 107 | assertNotNull(inputRecord); 108 | } 109 | 110 | @Test 111 | public void foo() { 112 | String timestamp = "2020-01-07 04:47:05.0000000"; 113 | DateTimeFormatter dateFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSSS") 114 | .withZone(ZoneId.of("UTC")); 115 | log.info(dateFormat.format(LocalDateTime.now())); 116 | ZonedDateTime dateTime = ZonedDateTime.parse(timestamp, dateFormat); 117 | } 118 | 119 | } 120 | -------------------------------------------------------------------------------- /src/test/java/com/github/jcustenborder/kafka/connect/json/JsonSchemaConverterTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import com.google.common.base.Charsets; 19 | import com.google.common.collect.ImmutableList; 20 | import com.google.common.collect.ImmutableMap; 21 | import org.apache.kafka.common.header.Header; 22 | import org.apache.kafka.common.header.Headers; 23 | import org.apache.kafka.common.header.internals.RecordHeaders; 24 | import org.apache.kafka.connect.data.Decimal; 25 | import org.apache.kafka.connect.data.Schema; 26 | import org.apache.kafka.connect.data.SchemaAndValue; 27 | import org.apache.kafka.connect.data.SchemaBuilder; 28 | import org.apache.kafka.connect.data.Struct; 29 | import org.junit.jupiter.api.BeforeEach; 30 | import org.junit.jupiter.api.DynamicTest; 31 | import org.junit.jupiter.api.Test; 32 | import org.junit.jupiter.api.TestFactory; 33 | import org.slf4j.Logger; 34 | import org.slf4j.LoggerFactory; 35 | 36 | import java.io.IOException; 37 | import java.math.BigDecimal; 38 | import java.time.LocalDate; 39 | import java.time.LocalTime; 40 | import java.time.ZoneOffset; 41 | import java.util.Date; 42 | import java.util.LinkedHashMap; 43 | import java.util.Map; 44 | import java.util.stream.IntStream; 45 | import java.util.stream.Stream; 46 | 47 | import static com.github.jcustenborder.kafka.connect.utils.AssertSchema.assertSchema; 48 | import static com.github.jcustenborder.kafka.connect.utils.AssertStruct.assertStruct; 49 | import static org.junit.jupiter.api.Assertions.assertArrayEquals; 50 | import static org.junit.jupiter.api.Assertions.assertEquals; 51 | import static org.junit.jupiter.api.Assertions.assertNotNull; 52 | import static org.junit.jupiter.api.Assertions.assertNull; 53 | import static org.junit.jupiter.api.Assertions.assertTrue; 54 | import static org.junit.jupiter.api.DynamicTest.dynamicTest; 55 | 56 | public class JsonSchemaConverterTest { 57 | private static final Logger log = LoggerFactory.getLogger(JsonSchemaConverterTest.class); 58 | JsonSchemaConverter converter; 59 | 60 | @BeforeEach 61 | public void beforeEach() { 62 | this.converter = new JsonSchemaConverter(); 63 | } 64 | 65 | @Test 66 | public void nulls() { 67 | this.converter.configure( 68 | ImmutableMap.of(), 69 | false 70 | ); 71 | SchemaAndValue expected = SchemaAndValue.NULL; 72 | Headers headers = new RecordHeaders(); 73 | byte[] buffer = this.converter.fromConnectData("topic", headers, expected.schema(), expected.value()); 74 | assertNull(buffer, "buffer should be null."); 75 | } 76 | 77 | @Test 78 | public void roundTripString() { 79 | this.converter.configure( 80 | ImmutableMap.of(), 81 | false 82 | ); 83 | SchemaAndValue expected = new SchemaAndValue( 84 | Schema.STRING_SCHEMA, 85 | "This is a test" 86 | ); 87 | Headers headers = new RecordHeaders(); 88 | byte[] buffer = this.converter.fromConnectData("topic", headers, expected.schema(), expected.value()); 89 | assertNotNull(buffer, "buffer should not be null."); 90 | assertTrue(buffer.length > 0, "buffer should be longer than zero."); 91 | Header schemaHeader = headers.lastHeader(this.converter.jsonSchemaHeader); 92 | assertNotNull(schemaHeader, "schemaHeader should not be null."); 93 | SchemaAndValue actual = this.converter.toConnectData("topic", headers, buffer); 94 | assertNotNull(actual, "actual should not be null."); 95 | assertSchema(expected.schema(), actual.schema()); 96 | assertEquals(expected.value(), actual.value()); 97 | } 98 | 99 | @TestFactory 100 | public Stream roundtrip() { 101 | Map tests = new LinkedHashMap<>(); 102 | Schema arraySchema = SchemaBuilder.array(Schema.STRING_SCHEMA).build(); 103 | tests.put( 104 | new SchemaAndValue(arraySchema, ImmutableList.of("one", "two", "three")), 105 | new SchemaAndValue(arraySchema, ImmutableList.of("one", "two", "three")) 106 | ); 107 | tests.put( 108 | new SchemaAndValue(Schema.STRING_SCHEMA, "This is a test"), 109 | new SchemaAndValue(Schema.STRING_SCHEMA, "This is a test") 110 | ); 111 | tests.put( 112 | new SchemaAndValue(Schema.BYTES_SCHEMA, "This is a test".getBytes(Charsets.UTF_8)), 113 | new SchemaAndValue(Schema.BYTES_SCHEMA, "This is a test".getBytes(Charsets.UTF_8)) 114 | ); 115 | tests.put( 116 | new SchemaAndValue(Schema.BOOLEAN_SCHEMA, true), 117 | new SchemaAndValue(Schema.BOOLEAN_SCHEMA, true) 118 | ); 119 | tests.put( 120 | new SchemaAndValue(Schema.INT8_SCHEMA, Byte.MAX_VALUE), 121 | new SchemaAndValue(Schema.INT64_SCHEMA, (long) Byte.MAX_VALUE) 122 | ); 123 | tests.put( 124 | new SchemaAndValue(Schema.INT16_SCHEMA, Short.MAX_VALUE), 125 | new SchemaAndValue(Schema.INT64_SCHEMA, (long) Short.MAX_VALUE) 126 | ); 127 | tests.put( 128 | new SchemaAndValue(Schema.INT32_SCHEMA, Integer.MAX_VALUE), 129 | new SchemaAndValue(Schema.INT64_SCHEMA, (long) Integer.MAX_VALUE) 130 | ); 131 | tests.put( 132 | new SchemaAndValue(Schema.INT64_SCHEMA, Long.MAX_VALUE), 133 | new SchemaAndValue(Schema.INT64_SCHEMA, Long.MAX_VALUE) 134 | ); 135 | tests.put( 136 | new SchemaAndValue(Schema.FLOAT32_SCHEMA, Float.MAX_VALUE), 137 | new SchemaAndValue(Schema.FLOAT64_SCHEMA, (double) Float.MAX_VALUE) 138 | ); 139 | tests.put( 140 | new SchemaAndValue(Schema.FLOAT64_SCHEMA, Double.MAX_VALUE), 141 | new SchemaAndValue(Schema.FLOAT64_SCHEMA, Double.MAX_VALUE) 142 | ); 143 | Date date = Date.from(LocalDate.of(2020, 02, 02).atTime(LocalTime.MIDNIGHT).toInstant(ZoneOffset.UTC)); 144 | tests.put( 145 | new SchemaAndValue(org.apache.kafka.connect.data.Date.SCHEMA, date), 146 | new SchemaAndValue(org.apache.kafka.connect.data.Date.SCHEMA, date) 147 | ); 148 | 149 | Date time = date.from(LocalTime.MIDNIGHT.atDate(LocalDate.of(1970, 1, 1)).toInstant(ZoneOffset.UTC)); 150 | tests.put( 151 | new SchemaAndValue(org.apache.kafka.connect.data.Time.SCHEMA, time), 152 | new SchemaAndValue(org.apache.kafka.connect.data.Time.SCHEMA, time) 153 | ); 154 | 155 | Date timestamp = new Date(1583363608123L); 156 | tests.put( 157 | new SchemaAndValue(org.apache.kafka.connect.data.Timestamp.SCHEMA, timestamp), 158 | new SchemaAndValue(org.apache.kafka.connect.data.Timestamp.SCHEMA, timestamp) 159 | ); 160 | 161 | IntStream.range(0, 30) 162 | .forEach(scale -> { 163 | BigDecimal input = BigDecimal.valueOf(Long.MAX_VALUE, scale); 164 | SchemaAndValue schemaAndValue = new SchemaAndValue(Decimal.schema(scale), input); 165 | tests.put(schemaAndValue, schemaAndValue); 166 | }); 167 | return tests.entrySet().stream() 168 | .map(p -> dynamicTest(p.getKey().schema().toString(), () -> { 169 | assertRoundTrip(p.getKey(), p.getValue()); 170 | })); 171 | } 172 | 173 | 174 | void assertRoundTrip(SchemaAndValue input, SchemaAndValue expected) { 175 | this.converter.configure( 176 | ImmutableMap.of(), 177 | false 178 | ); 179 | Headers headers = new RecordHeaders(); 180 | byte[] buffer = this.converter.fromConnectData("topic", headers, input.schema(), input.value()); 181 | log.trace(new String(buffer, Charsets.UTF_8)); 182 | assertNotNull(buffer, "buffer should not be null."); 183 | assertTrue(buffer.length > 0, "buffer should be longer than zero."); 184 | Header schemaHeader = headers.lastHeader(this.converter.jsonSchemaHeader); 185 | assertNotNull(schemaHeader, "schemaHeader should not be null."); 186 | SchemaAndValue actual = this.converter.toConnectData("topic", headers, buffer); 187 | assertNotNull(actual, "actual should not be null."); 188 | assertSchema(expected.schema(), actual.schema()); 189 | 190 | if (Decimal.LOGICAL_NAME.equals(expected.schema().name())) { 191 | assertEquals(expected.value(), actual.value()); 192 | } else { 193 | switch (expected.schema().type()) { 194 | case BYTES: 195 | assertArrayEquals((byte[]) expected.value(), (byte[]) actual.value()); 196 | break; 197 | case STRUCT: 198 | assertStruct((Struct) expected.value(), (Struct) actual.value()); 199 | break; 200 | default: 201 | assertEquals(expected.value(), actual.value()); 202 | break; 203 | } 204 | } 205 | } 206 | 207 | @Test 208 | public void nested() throws IOException { 209 | this.converter.configure( 210 | ImmutableMap.of(), 211 | false 212 | ); 213 | Schema addressSchema = SchemaBuilder.struct() 214 | .name("Address") 215 | .optional() 216 | .field("city", SchemaBuilder.string().build()) 217 | .field("state", SchemaBuilder.string().build()) 218 | .field("street_address", SchemaBuilder.string().build()) 219 | .build(); 220 | Schema customer = SchemaBuilder.struct() 221 | .name("Customer") 222 | .field("billing_address", addressSchema) 223 | .field("shipping_address", addressSchema) 224 | .build(); 225 | Struct billingAddress = new Struct(addressSchema) 226 | .put("city", "Austin") 227 | .put("state", "TX") 228 | .put("street_address", "123 Main St"); 229 | Struct shippingAddress = new Struct(addressSchema) 230 | .put("city", "Dallas") 231 | .put("state", "TX") 232 | .put("street_address", "321 Something St"); 233 | Struct struct = new Struct(customer) 234 | .put("billing_address", billingAddress) 235 | .put("shipping_address", shippingAddress); 236 | SchemaAndValue expected = new SchemaAndValue(customer, struct); 237 | Headers headers = new RecordHeaders(); 238 | byte[] buffer = this.converter.fromConnectData("topic", headers, expected.schema(), expected.value()); 239 | log.trace(new String(buffer, Charsets.UTF_8)); 240 | assertNotNull(buffer, "buffer should not be null."); 241 | assertTrue(buffer.length > 0, "buffer should be longer than zero."); 242 | Header schemaHeader = headers.lastHeader(this.converter.jsonSchemaHeader); 243 | assertNotNull(schemaHeader, "schemaHeader should not be null."); 244 | SchemaAndValue actual = this.converter.toConnectData("topic", headers, buffer); 245 | assertNotNull(actual, "actual should not be null."); 246 | assertSchema(expected.schema(), actual.schema()); 247 | assertEquals(expected.value(), actual.value()); 248 | } 249 | 250 | } 251 | -------------------------------------------------------------------------------- /src/test/java/com/github/jcustenborder/kafka/connect/json/TestUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.github.jcustenborder.kafka.connect.json; 17 | 18 | import org.everit.json.schema.internal.DateFormatValidator; 19 | import org.everit.json.schema.internal.DateTimeFormatValidator; 20 | import org.everit.json.schema.internal.TimeFormatValidator; 21 | import org.everit.json.schema.loader.SchemaLoader; 22 | import org.json.JSONObject; 23 | 24 | public class TestUtils { 25 | public static org.everit.json.schema.Schema jsonSchema(JSONObject rawSchema) { 26 | return SchemaLoader.builder() 27 | .draftV7Support() 28 | .addFormatValidator(new DateFormatValidator()) 29 | .addFormatValidator(new TimeFormatValidator()) 30 | .addFormatValidator(new DateTimeFormatValidator()) 31 | .schemaJson(rawSchema) 32 | .build() 33 | .load() 34 | .build(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/test/resources/com/github/jcustenborder/kafka/connect/json/FromJson/inline.json: -------------------------------------------------------------------------------- 1 | { 2 | "title" : "Inline Config", 3 | "input" : { 4 | "topic" : "foo", 5 | "kafkaPartition" : 1, 6 | "keySchema" : { 7 | "type" : "STRING", 8 | "isOptional" : false 9 | }, 10 | "key" : "foo", 11 | "valueSchema" : { 12 | "type" : "BYTES", 13 | "isOptional" : false 14 | }, 15 | "value" : "ewogICJmaXJzdE5hbWUiOiAiSm9obiIsCiAgImxhc3ROYW1lIjogIkRvZSIsCiAgImFnZSI6IDIxCn0=", 16 | "timestamp" : 1530286549123, 17 | "timestampType" : "CREATE_TIME", 18 | "offset" : 91283741, 19 | "headers" : [ ] 20 | }, 21 | "description" : "This example takes an input value that is a byte array and reads this value based on the supplied schema to a Kafka Connect value. The result is data that is based on the schema", 22 | "name" : "Inline Config", 23 | "config" : { 24 | "json.schema.location" : "Inline", 25 | "json.schema.inline" : "{\n \"$id\": \"https://example.com/person.schema.json\",\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Person\",\n \"type\": \"object\",\n \"properties\": {\n \"firstName\": {\n \"type\": \"string\",\n \"description\": \"The person's first name.\"\n },\n \"lastName\": {\n \"type\": \"string\",\n \"description\": \"The person's last name.\"\n },\n \"age\": {\n \"description\": \"Age in years which must be equal to or greater than zero.\",\n \"type\": \"integer\",\n \"minimum\": 0\n }\n }\n}" 26 | }, 27 | "childClass" : "Value" 28 | } -------------------------------------------------------------------------------- /src/test/resources/com/github/jcustenborder/kafka/connect/json/SchemaConverterTest/nested.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "title": "Customer", 4 | "definitions": { 5 | "address": { 6 | "title": "Address", 7 | "type": "object", 8 | "properties": { 9 | "street_address": { "type": "string" }, 10 | "city": { "type": "string" }, 11 | "state": { "type": "string" } 12 | }, 13 | "required": ["street_address", "city", "state"] 14 | } 15 | }, 16 | 17 | "type": "object", 18 | 19 | "properties": { 20 | "billing_address": { "$ref": "#/definitions/address" }, 21 | "shipping_address": { "$ref": "#/definitions/address" } 22 | } 23 | } -------------------------------------------------------------------------------- /src/test/resources/com/github/jcustenborder/kafka/connect/json/SchemaConverterTest/product.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "$id": "http://example.com/product.schema.json", 4 | "title": "Product", 5 | "description": "A product from Acme's catalog", 6 | "type": "object", 7 | "properties": { 8 | "price": { 9 | "description": "The price of the product", 10 | "type": "number", 11 | "exclusiveMinimum": 0 12 | }, 13 | "productId": { 14 | "description": "The unique identifier for a product", 15 | "type": "integer" 16 | }, 17 | "productName": { 18 | "description": "Name of the product", 19 | "type": "string" 20 | } 21 | }, 22 | "required": [ "productId", "productName", "price" ] 23 | } -------------------------------------------------------------------------------- /src/test/resources/com/github/jcustenborder/kafka/connect/json/SchemaConverterTest/wikimedia.recentchange.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "mediawiki/recentchange", 3 | "description": "Represents a MW RecentChange event. https://www.mediawiki.org/wiki/Manual:RCFeed\n", 4 | "$id": "/mediawiki/recentchange/1.0.0", 5 | "$schema": "https://json-schema.org/draft-07/schema#", 6 | "type": "object", 7 | "additionalProperties": true, 8 | "required": [ 9 | "$schema", 10 | "meta" 11 | ], 12 | "properties": { 13 | "$schema": { 14 | "type": "string", 15 | "description": "A URI identifying the JSONSchema for this event. This should match an schema's $id in a schema repository. E.g. /schema_name/1.0.0\n" 16 | }, 17 | "meta": { 18 | "type": "object", 19 | "required": [ 20 | "id", 21 | "dt", 22 | "stream" 23 | ], 24 | "properties": { 25 | "uri": { 26 | "type": "string", 27 | "format": "uri-reference", 28 | "maxLength": 8192, 29 | "description": "Unique URI identifying the event or entity" 30 | }, 31 | "request_id": { 32 | "type": "string", 33 | "description": "Unique ID of the request that caused the event" 34 | }, 35 | "id": { 36 | "type": "string", 37 | "pattern": "^[a-fA-F0-9]{8}(-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}$", 38 | "maxLength": 36, 39 | "description": "Unique ID of this event" 40 | }, 41 | "dt": { 42 | "type": "string", 43 | "format": "date-time", 44 | "maxLength": 128, 45 | "description": "Event datetime, in ISO-8601 format" 46 | }, 47 | "domain": { 48 | "type": "string", 49 | "description": "Domain the event or entity pertains to", 50 | "minLength": 1 51 | }, 52 | "stream": { 53 | "type": "string", 54 | "description": "Name of the stream/queue/dataset that this event belongs in", 55 | "minLength": 1 56 | } 57 | } 58 | }, 59 | "id": { 60 | "description": "ID of the recentchange event (rcid).", 61 | "type": [ 62 | "integer", 63 | "null" 64 | ] 65 | }, 66 | "type": { 67 | "description": "Type of recentchange event (rc_type). One of \"edit\", \"new\", \"log\", \"categorize\", or \"external\". (See Manual:Recentchanges table#rc_type)\n", 68 | "type": "string" 69 | }, 70 | "title": { 71 | "description": "Full page name, from Title::getPrefixedText.", 72 | "type": "string" 73 | }, 74 | "namespace": { 75 | "description": "ID of relevant namespace of affected page (rc_namespace, page_namespace). This is -1 (\"Special\") for log events.\n", 76 | "type": "integer" 77 | }, 78 | "comment": { 79 | "description": "(rc_comment)", 80 | "type": "string" 81 | }, 82 | "parsedcomment": { 83 | "description": "The rc_comment parsed into simple HTML. Optional", 84 | "type": "string" 85 | }, 86 | "timestamp": { 87 | "description": "Unix timestamp (derived from rc_timestamp).", 88 | "type": "integer" 89 | }, 90 | "user": { 91 | "description": "(rc_user_text)", 92 | "type": "string" 93 | }, 94 | "bot": { 95 | "description": "(rc_bot)", 96 | "type": "boolean" 97 | }, 98 | "server_url": { 99 | "description": "$wgCanonicalServer", 100 | "type": "string" 101 | }, 102 | "server_name": { 103 | "description": "$wgServerName", 104 | "type": "string" 105 | }, 106 | "server_script_path": { 107 | "description": "$wgScriptPath", 108 | "type": "string" 109 | }, 110 | "wiki": { 111 | "description": "wfWikiID ($wgDBprefix, $wgDBname)", 112 | "type": "string" 113 | }, 114 | "minor": { 115 | "description": "(rc_minor).", 116 | "type": "boolean" 117 | }, 118 | "patrolled": { 119 | "description": "(rc_patrolled). This property only exists if patrolling is supported for this event (based on $wgUseRCPatrol, $wgUseNPPatrol).\n", 120 | "type": "boolean" 121 | }, 122 | "length": { 123 | "description": "Length of old and new change", 124 | "type": "object", 125 | "properties": { 126 | "old": { 127 | "description": "(rc_old_len)", 128 | "type": [ 129 | "integer", 130 | "null" 131 | ] 132 | }, 133 | "new": { 134 | "description": "(rc_new_len)", 135 | "type": [ 136 | "integer", 137 | "null" 138 | ] 139 | } 140 | } 141 | }, 142 | "revision": { 143 | "description": "Old and new revision IDs", 144 | "type": "object", 145 | "properties": { 146 | "new": { 147 | "description": "(rc_last_oldid)", 148 | "type": [ 149 | "integer", 150 | "null" 151 | ] 152 | }, 153 | "old": { 154 | "description": "(rc_this_oldid)", 155 | "type": [ 156 | "integer", 157 | "null" 158 | ] 159 | } 160 | } 161 | }, 162 | "log_id": { 163 | "description": "(rc_log_id)", 164 | "type": [ 165 | "integer", 166 | "null" 167 | ] 168 | }, 169 | "log_type": { 170 | "description": "(rc_log_type)", 171 | "type": [ 172 | "string", 173 | "null" 174 | ] 175 | }, 176 | "log_action": { 177 | "description": "(rc_log_action)", 178 | "type": "string" 179 | }, 180 | "log_params": { 181 | "description": "Property only exists if event has rc_params.", 182 | "type": [ 183 | "array", 184 | "object", 185 | "string" 186 | ], 187 | "additionalProperties": true 188 | }, 189 | "log_action_comment": { 190 | "type": [ 191 | "string", 192 | "null" 193 | ] 194 | } 195 | } 196 | } -------------------------------------------------------------------------------- /src/test/resources/com/github/jcustenborder/kafka/connect/json/basic.data.json: -------------------------------------------------------------------------------- 1 | { 2 | "firstName": "John", 3 | "lastName": "Doe", 4 | "age": 21 5 | } -------------------------------------------------------------------------------- /src/test/resources/com/github/jcustenborder/kafka/connect/json/basic.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$id": "https://example.com/person.schema.json", 3 | "$schema": "http://json-schema.org/draft-07/schema#", 4 | "title": "Person", 5 | "type": "object", 6 | "properties": { 7 | "firstName": { 8 | "type": "string", 9 | "description": "The person's first name." 10 | }, 11 | "lastName": { 12 | "type": "string", 13 | "description": "The person's last name." 14 | }, 15 | "age": { 16 | "description": "Age in years which must be equal to or greater than zero.", 17 | "type": "integer", 18 | "minimum": 0 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /src/test/resources/com/github/jcustenborder/kafka/connect/json/customdate.data.json: -------------------------------------------------------------------------------- 1 | { 2 | "Timestamp": "2020-01-07 04:47:05.0000000" 3 | } -------------------------------------------------------------------------------- /src/test/resources/com/github/jcustenborder/kafka/connect/json/customdate.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$id": "https://example.com/person.schema.json", 3 | "$schema": "http://json-schema.org/draft-07/schema#", 4 | "title": "Person", 5 | "type": "object", 6 | "properties": { 7 | "Timestamp": { 8 | "type": "string", 9 | "description": "Timestamp", 10 | "format": "custom-timestamp", 11 | "dateTimeFormat": "yyyy-MM-dd HH:mm:ss.SSSSSSS" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /src/test/resources/com/github/jcustenborder/kafka/connect/json/geo.data.json: -------------------------------------------------------------------------------- 1 | { 2 | "latitude": 48.858093, 3 | "longitude": 2.294694 4 | } -------------------------------------------------------------------------------- /src/test/resources/com/github/jcustenborder/kafka/connect/json/geo.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$id": "https://example.com/geographical-location.schema.json", 3 | "$schema": "http://json-schema.org/draft-07/schema#", 4 | "title": "Longitude and Latitude Values", 5 | "description": "A geographical coordinate.", 6 | "required": [ 7 | "latitude", 8 | "longitude" 9 | ], 10 | "type": "object", 11 | "properties": { 12 | "latitude": { 13 | "type": "number", 14 | "minimum": -90, 15 | "maximum": 90 16 | }, 17 | "longitude": { 18 | "type": "number", 19 | "minimum": -180, 20 | "maximum": 180 21 | } 22 | } 23 | } -------------------------------------------------------------------------------- /src/test/resources/com/github/jcustenborder/kafka/connect/json/wikimedia.recentchange.data.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "/mediawiki/recentchange/1.0.0", 3 | "meta": { 4 | "uri": "https://sl.wikipedia.org/wiki/Iztrebek", 5 | "request_id": "61a2da40-3fea-450f-8404-8ed5e98e784a", 6 | "id": "5fe0b22e-1ed1-4acd-9e06-07a39ef800e7", 7 | "dt": "2020-05-14T16:24:27Z", 8 | "domain": "sl.wikipedia.org", 9 | "stream": "mediawiki.recentchange", 10 | "topic": "eqiad.mediawiki.recentchange", 11 | "partition": 0, 12 | "offset": 2403979241 13 | }, 14 | "id": 21345744, 15 | "type": "edit", 16 | "namespace": 0, 17 | "title": "Iztrebek", 18 | "comment": "vrnitev sprememb uporabnika [[Special:Contributions/176.76.241.31|176.76.241.31]] ([[User talk:176.76.241.31|pogovor]]) na zadnje urejanje uporabnika [[User:Samuele2002|Samuele2002]]", 19 | "timestamp": 1589473467, 20 | "user": "Upwinxp", 21 | "bot": false, 22 | "minor": true, 23 | "length": { 24 | "old": 561, 25 | "new": 1061 26 | }, 27 | "revision": { 28 | "old": 5320868, 29 | "new": 5320869 30 | }, 31 | "server_url": "https://sl.wikipedia.org", 32 | "server_name": "sl.wikipedia.org", 33 | "server_script_path": "/w", 34 | "wiki": "slwiki", 35 | "parsedcomment": "vrnitev sprememb uporabnika 176.76.241.31 (pogovor) na zadnje urejanje uporabnika Samuele2002" 36 | } -------------------------------------------------------------------------------- /src/test/resources/com/github/jcustenborder/kafka/connect/json/wikimedia.recentchange.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "mediawiki/recentchange", 3 | "description": "Represents a MW RecentChange event. https://www.mediawiki.org/wiki/Manual:RCFeed\n", 4 | "$id": "/mediawiki/recentchange/1.0.0", 5 | "$schema": "https://json-schema.org/draft-07/schema#", 6 | "type": "object", 7 | "additionalProperties": true, 8 | "required": [ 9 | "$schema", 10 | "meta" 11 | ], 12 | "properties": { 13 | "$schema": { 14 | "type": "string", 15 | "description": "A URI identifying the JSONSchema for this event. This should match an schema's $id in a schema repository. E.g. /schema_name/1.0.0\n" 16 | }, 17 | "meta": { 18 | "type": "object", 19 | "required": [ 20 | "id", 21 | "dt", 22 | "stream" 23 | ], 24 | "properties": { 25 | "uri": { 26 | "type": "string", 27 | "format": "uri-reference", 28 | "maxLength": 8192, 29 | "description": "Unique URI identifying the event or entity" 30 | }, 31 | "request_id": { 32 | "type": "string", 33 | "description": "Unique ID of the request that caused the event" 34 | }, 35 | "id": { 36 | "type": "string", 37 | "pattern": "^[a-fA-F0-9]{8}(-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}$", 38 | "maxLength": 36, 39 | "description": "Unique ID of this event" 40 | }, 41 | "dt": { 42 | "type": "string", 43 | "format": "date-time", 44 | "maxLength": 128, 45 | "description": "Event datetime, in ISO-8601 format" 46 | }, 47 | "domain": { 48 | "type": "string", 49 | "description": "Domain the event or entity pertains to", 50 | "minLength": 1 51 | }, 52 | "stream": { 53 | "type": "string", 54 | "description": "Name of the stream/queue/dataset that this event belongs in", 55 | "minLength": 1 56 | } 57 | } 58 | }, 59 | "id": { 60 | "description": "ID of the recentchange event (rcid).", 61 | "type": [ 62 | "integer", 63 | "null" 64 | ] 65 | }, 66 | "type": { 67 | "description": "Type of recentchange event (rc_type). One of \"edit\", \"new\", \"log\", \"categorize\", or \"external\". (See Manual:Recentchanges table#rc_type)\n", 68 | "type": "string" 69 | }, 70 | "title": { 71 | "description": "Full page name, from Title::getPrefixedText.", 72 | "type": "string" 73 | }, 74 | "namespace": { 75 | "description": "ID of relevant namespace of affected page (rc_namespace, page_namespace). This is -1 (\"Special\") for log events.\n", 76 | "type": "integer" 77 | }, 78 | "comment": { 79 | "description": "(rc_comment)", 80 | "type": "string" 81 | }, 82 | "parsedcomment": { 83 | "description": "The rc_comment parsed into simple HTML. Optional", 84 | "type": "string" 85 | }, 86 | "timestamp": { 87 | "description": "Unix timestamp (derived from rc_timestamp).", 88 | "type": "integer" 89 | }, 90 | "user": { 91 | "description": "(rc_user_text)", 92 | "type": "string" 93 | }, 94 | "bot": { 95 | "description": "(rc_bot)", 96 | "type": "boolean" 97 | }, 98 | "server_url": { 99 | "description": "$wgCanonicalServer", 100 | "type": "string" 101 | }, 102 | "server_name": { 103 | "description": "$wgServerName", 104 | "type": "string" 105 | }, 106 | "server_script_path": { 107 | "description": "$wgScriptPath", 108 | "type": "string" 109 | }, 110 | "wiki": { 111 | "description": "wfWikiID ($wgDBprefix, $wgDBname)", 112 | "type": "string" 113 | }, 114 | "minor": { 115 | "description": "(rc_minor).", 116 | "type": "boolean" 117 | }, 118 | "patrolled": { 119 | "description": "(rc_patrolled). This property only exists if patrolling is supported for this event (based on $wgUseRCPatrol, $wgUseNPPatrol).\n", 120 | "type": "boolean" 121 | }, 122 | "length": { 123 | "description": "Length of old and new change", 124 | "type": "object", 125 | "properties": { 126 | "old": { 127 | "description": "(rc_old_len)", 128 | "type": [ 129 | "integer", 130 | "null" 131 | ] 132 | }, 133 | "new": { 134 | "description": "(rc_new_len)", 135 | "type": [ 136 | "integer", 137 | "null" 138 | ] 139 | } 140 | } 141 | }, 142 | "revision": { 143 | "description": "Old and new revision IDs", 144 | "type": "object", 145 | "properties": { 146 | "new": { 147 | "description": "(rc_last_oldid)", 148 | "type": [ 149 | "integer", 150 | "null" 151 | ] 152 | }, 153 | "old": { 154 | "description": "(rc_this_oldid)", 155 | "type": [ 156 | "integer", 157 | "null" 158 | ] 159 | } 160 | } 161 | }, 162 | "log_id": { 163 | "description": "(rc_log_id)", 164 | "type": [ 165 | "integer", 166 | "null" 167 | ] 168 | }, 169 | "log_type": { 170 | "description": "(rc_log_type)", 171 | "type": [ 172 | "string", 173 | "null" 174 | ] 175 | }, 176 | "log_action": { 177 | "description": "(rc_log_action)", 178 | "type": "string" 179 | }, 180 | "log_params": { 181 | "description": "Property only exists if event has rc_params.", 182 | "type": [ 183 | "array", 184 | "object", 185 | "string" 186 | ], 187 | "additionalProperties": true 188 | }, 189 | "log_action_comment": { 190 | "type": [ 191 | "string", 192 | "null" 193 | ] 194 | } 195 | } 196 | } -------------------------------------------------------------------------------- /src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | --------------------------------------------------------------------------------