├── .gitignore
├── Jenkinsfile
├── LICENSE
├── README.md
├── bin
└── debug.sh
├── config
└── connect-avro-docker.properties
├── docker-compose.yml
├── pom.xml
└── src
├── main
└── java
│ └── com
│ └── github
│ └── jcustenborder
│ └── kafka
│ └── connect
│ └── json
│ ├── CustomTimestampFormatValidator.java
│ ├── DecimalFormatValidator.java
│ ├── FromConnectConversionKey.java
│ ├── FromConnectSchemaConverter.java
│ ├── FromConnectState.java
│ ├── FromConnectVisitor.java
│ ├── FromJson.java
│ ├── FromJsonConfig.java
│ ├── FromJsonConversionKey.java
│ ├── FromJsonSchemaConverter.java
│ ├── FromJsonSchemaConverterFactory.java
│ ├── FromJsonState.java
│ ├── FromJsonVisitor.java
│ ├── JacksonFactory.java
│ ├── JsonConfig.java
│ ├── JsonSchemaConverter.java
│ ├── JsonSchemaConverterConfig.java
│ ├── Utils.java
│ └── package-info.java
└── test
├── java
└── com
│ └── github
│ └── jcustenborder
│ └── kafka
│ └── connect
│ └── json
│ ├── DocumentationTest.java
│ ├── FromConnectSchemaConverterTest.java
│ ├── FromJsonSchemaConverterTest.java
│ ├── FromJsonTest.java
│ ├── JsonSchemaConverterTest.java
│ └── TestUtils.java
└── resources
├── com
└── github
│ └── jcustenborder
│ └── kafka
│ └── connect
│ └── json
│ ├── FromJson
│ └── inline.json
│ ├── SchemaConverterTest
│ ├── nested.schema.json
│ ├── product.schema.json
│ └── wikimedia.recentchange.schema.json
│ ├── basic.data.json
│ ├── basic.schema.json
│ ├── customdate.data.json
│ ├── customdate.schema.json
│ ├── geo.data.json
│ ├── geo.schema.json
│ ├── wikimedia.recentchange.data.json
│ └── wikimedia.recentchange.schema.json
└── logback.xml
/.gitignore:
--------------------------------------------------------------------------------
1 | # Compiled class file
2 | *.class
3 |
4 | # Log file
5 | *.log
6 |
7 | # BlueJ files
8 | *.ctxt
9 |
10 | # Mobile Tools for Java (J2ME)
11 | .mtj.tmp/
12 |
13 | # Package Files #
14 | *.jar
15 | *.war
16 | *.nar
17 | *.ear
18 | *.zip
19 | *.tar.gz
20 | *.rar
21 |
22 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
23 | hs_err_pid*
24 | target
25 |
--------------------------------------------------------------------------------
/Jenkinsfile:
--------------------------------------------------------------------------------
1 | #!groovy
2 | @Library('jenkins-pipeline') import com.github.jcustenborder.jenkins.pipeline.KafkaConnectPipeline
3 |
4 | def pipe = new KafkaConnectPipeline()
5 | pipe.execute()
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright [yyyy] [name of copyright owner]
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Introduction
2 | [Documentation](https://jcustenborder.github.io/kafka-connect-documentation/projects/kafka-connect-json-schema)
3 |
4 | Installation through the [Confluent Hub Client](https://docs.confluent.io/current/connect/managing/confluent-hub/client.html)
5 |
6 | This plugin is used to add additional JSON parsing functionality to Kafka Connect.
7 |
8 | ## [From Json transformation](https://jcustenborder.github.io/kafka-connect-documentation/projects/kafka-connect-json-schema/transformations/FromJson.html)
9 |
10 | The FromJson will read JSON data that is in string on byte form and parse the data to a connect structure based on the JSON schema provided.
11 |
12 | # Development
13 |
14 | ## Building the source
15 |
16 | ```bash
17 | mvn clean package
18 | ```
--------------------------------------------------------------------------------
/bin/debug.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | #
3 | # Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
4 | #
5 | # Licensed under the Apache License, Version 2.0 (the "License");
6 | # you may not use this file except in compliance with the License.
7 | # You may obtain a copy of the License at
8 | #
9 | # http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # Unless required by applicable law or agreed to in writing, software
12 | # distributed under the License is distributed on an "AS IS" BASIS,
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | # See the License for the specific language governing permissions and
15 | # limitations under the License.
16 | #
17 |
18 | : ${SUSPEND:='n'}
19 |
20 | set -e
21 |
22 | mvn clean package
23 | export KAFKA_DEBUG='y'
24 | connect-standalone config/connect-avro-docker.properties config/sink.properties
--------------------------------------------------------------------------------
/config/connect-avro-docker.properties:
--------------------------------------------------------------------------------
1 | #
2 | # Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 | #
16 |
17 | bootstrap.servers=kafka:9092
18 | key.converter=io.confluent.connect.avro.AvroConverter
19 | key.converter.schema.registry.url=http://schema-registry:8081
20 | value.converter=io.confluent.connect.avro.AvroConverter
21 | value.converter.schema.registry.url=http://schema-registry:8081
22 | internal.key.converter=org.apache.kafka.connect.json.JsonConverter
23 | internal.value.converter=org.apache.kafka.connect.json.JsonConverter
24 | internal.key.converter.schemas.enable=false
25 | internal.value.converter.schemas.enable=false
26 | offset.storage.file.filename=/tmp/connect.offsets
27 | plugin.path=target/kafka-connect-target/usr/share/kafka-connect
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | #
2 | # Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 | #
16 |
17 | version: "2"
18 | services:
19 | zookeeper:
20 | image: confluentinc/cp-zookeeper:5.4.0
21 | ports:
22 | - "2181:2181"
23 | environment:
24 | ZOOKEEPER_CLIENT_PORT: 2181
25 | kafka:
26 | image: confluentinc/cp-kafka:5.4.0
27 | depends_on:
28 | - zookeeper
29 | ports:
30 | - "9092:9092"
31 | environment:
32 | KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
33 | KAFKA_ADVERTISED_LISTENERS: "plaintext://kafka:9092"
34 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
35 | schema-registry:
36 | image: confluentinc/cp-schema-registry:5.4.0
37 | depends_on:
38 | - kafka
39 | - zookeeper
40 | ports:
41 | - "8081:8081"
42 | environment:
43 | SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: "zookeeper:2181"
44 | SCHEMA_REGISTRY_HOST_NAME: schema-registry
45 |
--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
19 |
23 | 4.0.0
24 |
25 | com.github.jcustenborder.kafka.connect
26 | kafka-connect-parent
27 | 2.4.0
28 |
29 | kafka-connect-json-schema
30 | 0.2-SNAPSHOT
31 | kafka-connect-json-schema
32 | This project provides a mechanism to marshall data from JSON to a Kafka Connect struct based on a user provided JSON schema. This is accomplished by using the FromJson transformation which will convert data as a transformation. The JsonSchemaConverter transformation can be used to read and write data as Json.
33 | https://github.com/jcustenborder/kafka-connect-json-schema
34 | 2020
35 |
36 |
37 | The Apache License, Version 2.0
38 | https://www.apache.org/licenses/LICENSE-2.0
39 | repo
40 |
41 |
42 |
43 |
44 | jcustenborder
45 | Jeremy Custenborder
46 | https://github.com/jcustenborder
47 |
48 | Committer
49 |
50 |
51 |
52 |
53 | scm:git:https://github.com/jcustenborder/kafka-connect-json-schema.git
54 | scm:git:git@github.com:jcustenborder/kafka-connect-json-schema.git
55 |
56 | https://github.com/jcustenborder/kafka-connect-json-schema
57 |
58 |
59 | github
60 | https://github.com/jcustenborder/kafka-connect-json-schema/issues
61 |
62 |
63 |
64 | com.github.everit-org.json-schema
65 | org.everit.json.schema
66 | 1.12.1
67 |
68 |
69 |
70 |
71 | jitpack.io
72 | https://jitpack.io
73 |
74 |
75 |
76 |
77 |
78 | io.confluent
79 | kafka-connect-maven-plugin
80 |
81 | true
82 | https://jcustenborder.github.io/kafka-connect-documentation/
83 |
84 | transform
85 | converter
86 |
87 |
88 | Transform
89 | JSON
90 |
91 | Kafka Connect JSON Schema Transformations
92 | ${pom.issueManagement.url}
93 | Support provided through community involvement.
94 |
95 | org.reflections:reflections
96 |
97 |
98 |
99 |
100 |
101 |
102 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/CustomTimestampFormatValidator.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import org.everit.json.schema.FormatValidator;
19 |
20 | import java.util.Optional;
21 |
22 | public class CustomTimestampFormatValidator implements FormatValidator {
23 | @Override
24 | public Optional validate(String s) {
25 | return Optional.empty();
26 | }
27 |
28 | @Override
29 | public String formatName() {
30 | return "custom-timestamp";
31 | }
32 | }
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/DecimalFormatValidator.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import org.everit.json.schema.FormatValidator;
19 |
20 | import java.util.Optional;
21 |
22 | class DecimalFormatValidator implements FormatValidator {
23 | @Override
24 | public Optional validate(String s) {
25 | return Optional.empty();
26 | }
27 |
28 | @Override
29 | public String formatName() {
30 | return "decimal";
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/FromConnectConversionKey.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import com.google.common.base.MoreObjects;
19 | import com.google.common.base.Objects;
20 | import org.apache.kafka.connect.data.Decimal;
21 | import org.apache.kafka.connect.data.Schema;
22 |
23 | public class FromConnectConversionKey {
24 | public final Schema.Type type;
25 | public final String schemaName;
26 | public final Integer scale;
27 |
28 | private FromConnectConversionKey(Schema.Type type, String schemaName, Integer scale) {
29 | this.type = type;
30 | this.schemaName = schemaName;
31 | this.scale = scale;
32 | }
33 |
34 | public static FromConnectConversionKey of(Schema schema) {
35 | Integer scale;
36 | if (Decimal.LOGICAL_NAME.equals(schema.name())) {
37 | String scaleText = schema.parameters().get(Decimal.SCALE_FIELD);
38 | scale = Integer.parseInt(scaleText);
39 | } else {
40 | scale = null;
41 | }
42 | return of(schema.type(), schema.name(), scale);
43 | }
44 |
45 | public static FromConnectConversionKey of(Schema.Type type) {
46 | return new FromConnectConversionKey(type, null, null);
47 | }
48 |
49 | public static FromConnectConversionKey of(Schema.Type type, String schemaName) {
50 | return new FromConnectConversionKey(type, schemaName, null);
51 | }
52 |
53 | public static FromConnectConversionKey of(Schema.Type type, String schemaName, Integer scale) {
54 | return new FromConnectConversionKey(type, schemaName, scale);
55 | }
56 |
57 | @Override
58 | public boolean equals(Object o) {
59 | if (this == o) return true;
60 | if (o == null || getClass() != o.getClass()) return false;
61 | FromConnectConversionKey that = (FromConnectConversionKey) o;
62 | return type == that.type &&
63 | Objects.equal(schemaName, that.schemaName) &&
64 | Objects.equal(scale, that.scale);
65 | }
66 |
67 | @Override
68 | public int hashCode() {
69 | return Objects.hashCode(type, schemaName, scale);
70 | }
71 |
72 | @Override
73 | public String toString() {
74 | return MoreObjects.toStringHelper(this)
75 | .omitNullValues()
76 | .add("type", type)
77 | .add("schemaName", schemaName)
78 | .add("scale", scale)
79 | .toString();
80 | }
81 | }
82 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/FromConnectSchemaConverter.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import com.google.common.base.Charsets;
19 | import com.google.common.base.MoreObjects;
20 | import com.google.common.base.Strings;
21 | import com.google.common.collect.ImmutableMap;
22 | import org.apache.kafka.common.header.Header;
23 | import org.apache.kafka.common.header.internals.RecordHeader;
24 | import org.apache.kafka.connect.data.Date;
25 | import org.apache.kafka.connect.data.Decimal;
26 | import org.apache.kafka.connect.data.Schema;
27 | import org.apache.kafka.connect.data.Schema.Type;
28 | import org.apache.kafka.connect.data.Time;
29 | import org.apache.kafka.connect.data.Timestamp;
30 | import org.json.JSONObject;
31 | import org.slf4j.Logger;
32 | import org.slf4j.LoggerFactory;
33 |
34 | import java.util.ArrayList;
35 | import java.util.LinkedHashMap;
36 | import java.util.List;
37 | import java.util.Map;
38 | import java.util.stream.IntStream;
39 | import java.util.stream.Stream;
40 |
41 | public class FromConnectSchemaConverter {
42 | static final Map> PRIMITIVE_TYPES;
43 | static final Map PRIMITIVE_VISITORS;
44 | private static final Logger log = LoggerFactory.getLogger(FromConnectSchemaConverter.class);
45 |
46 | private static void addType(
47 | Map primitiveVisitors,
48 | Map> primitiveTypes,
49 | FromConnectConversionKey key,
50 | FromConnectVisitor visitor,
51 | ImmutableMap properties) {
52 |
53 | primitiveTypes.put(key, properties);
54 | primitiveVisitors.put(key, visitor);
55 | }
56 |
57 | static {
58 | Map visitors = new LinkedHashMap<>();
59 | Map> types = new LinkedHashMap<>();
60 | addType(
61 | visitors, types,
62 | FromConnectConversionKey.of(Type.BOOLEAN),
63 | new FromConnectVisitor.BooleanVisitor(),
64 | ImmutableMap.of("type", "boolean")
65 | );
66 | addType(
67 | visitors, types,
68 | FromConnectConversionKey.of(Type.BYTES),
69 | new FromConnectVisitor.BytesVisitor(),
70 | ImmutableMap.of("type", "string", "contentEncoding", "base64")
71 | );
72 | addType(
73 | visitors, types,
74 | FromConnectConversionKey.of(Type.FLOAT32),
75 | new FromConnectVisitor.FloatVisitor(),
76 | ImmutableMap.of("type", "number")
77 | );
78 | addType(
79 | visitors, types,
80 | FromConnectConversionKey.of(Type.FLOAT64),
81 | new FromConnectVisitor.FloatVisitor(),
82 | ImmutableMap.of("type", "number")
83 | );
84 | Stream.of(Type.INT8, Type.INT16, Type.INT32, Type.INT64)
85 | .forEach(type -> {
86 | addType(
87 | visitors, types,
88 | FromConnectConversionKey.of(type),
89 | new FromConnectVisitor.IntegerVisitor(),
90 | ImmutableMap.of("type", "integer")
91 | );
92 | });
93 | IntStream.range(0, 100)
94 | .forEach(scale -> {
95 | Schema decimalSchema = Decimal.schema(scale);
96 | addType(
97 | visitors, types,
98 | FromConnectConversionKey.of(decimalSchema),
99 | new FromConnectVisitor.DecimalVisitor(scale),
100 | ImmutableMap.of(
101 | "type", "string",
102 | "format", "decimal",
103 | "scale", Integer.toString(scale)
104 | )
105 | );
106 | });
107 |
108 | visitors.put(FromConnectConversionKey.of(Type.STRING), new FromConnectVisitor.StringVisitor());
109 | types.put(FromConnectConversionKey.of(Type.STRING), ImmutableMap.of("type", "string"));
110 |
111 | visitors.put(FromConnectConversionKey.of(Date.SCHEMA), new FromConnectVisitor.DateVisitor());
112 | types.put(FromConnectConversionKey.of(Date.SCHEMA), ImmutableMap.of("type", "string", "format", "date"));
113 |
114 | visitors.put(FromConnectConversionKey.of(Time.SCHEMA), new FromConnectVisitor.TimeVisitor());
115 | types.put(FromConnectConversionKey.of(Time.SCHEMA), ImmutableMap.of("type", "string", "format", "time"));
116 |
117 | visitors.put(FromConnectConversionKey.of(Timestamp.SCHEMA), new FromConnectVisitor.DateTimeVisitor());
118 | types.put(FromConnectConversionKey.of(Timestamp.SCHEMA), ImmutableMap.of("type", "string", "format", "date-time"));
119 | PRIMITIVE_TYPES = ImmutableMap.copyOf(types);
120 | PRIMITIVE_VISITORS = ImmutableMap.copyOf(visitors);
121 | }
122 |
123 | public static FromConnectState toJsonSchema(org.apache.kafka.connect.data.Schema schema, String headerName) {
124 | Map definitions = new LinkedHashMap<>();
125 | List visitors = new ArrayList<>();
126 | JSONObject result = toJsonSchema(schema, definitions, visitors);
127 | result.put("$schema", "http://json-schema.org/draft-07/schema#");
128 | if (!definitions.isEmpty()) {
129 | //definitions
130 | JSONObject definitionsObject = new JSONObject();
131 | definitions.forEach((definitionName, definition) -> {
132 | definitionsObject.put(definition.name(), definition.jsonSchema());
133 | });
134 | result.put("definitions", definitionsObject);
135 | }
136 |
137 | Header header = new RecordHeader(
138 | headerName,
139 | result.toString().getBytes(Charsets.UTF_8)
140 | );
141 |
142 | FromConnectVisitor visitor = visitors.get(0);
143 | return FromConnectState.of(header, visitor);
144 | }
145 |
146 | private static JSONObject toJsonSchema(org.apache.kafka.connect.data.Schema schema, Map definitions, List visitors) {
147 | JSONObject result = new JSONObject();
148 | if (!Strings.isNullOrEmpty(schema.doc())) {
149 | result.put("description", schema.doc());
150 | }
151 | FromConnectConversionKey key = FromConnectConversionKey.of(schema);
152 | log.trace("toJsonSchema() - Checking for '{}'", key);
153 | Map primitiveType = PRIMITIVE_TYPES.get(key);
154 | if (null != primitiveType) {
155 | primitiveType.forEach(result::put);
156 | FromConnectVisitor visitor = PRIMITIVE_VISITORS.get(key);
157 | visitors.add(visitor);
158 | return result;
159 | }
160 |
161 | if (!Strings.isNullOrEmpty(schema.name())) {
162 | result.put("title", schema.name());
163 | }
164 |
165 |
166 | if (Type.ARRAY == schema.type()) {
167 | result.put("type", "array");
168 | FromConnectVisitor elementVisitor;
169 | if (Type.STRUCT == schema.valueSchema().type()) {
170 | Definition definition = definitions.computeIfAbsent(schema.valueSchema(), s -> {
171 | List childVisitors = new ArrayList<>();
172 | JSONObject fieldJsonSchema = toJsonSchema(schema.valueSchema(), definitions, childVisitors);
173 | String definitionName = schema.valueSchema().name().toLowerCase();
174 | return Definition.of(fieldJsonSchema, definitionName, childVisitors);
175 | });
176 | result.put("items", definition.ref());
177 | elementVisitor = definition.visitors().get(0);
178 | } else {
179 | List childVisitors = new ArrayList<>();
180 | JSONObject arrayValueSchema = toJsonSchema(schema.valueSchema(), definitions, childVisitors);
181 | elementVisitor = childVisitors.get(0);
182 | result.put("items", arrayValueSchema);
183 | }
184 | visitors.add(new FromConnectVisitor.ArrayVisitor(elementVisitor));
185 | }
186 | if (Type.STRUCT == schema.type()) {
187 | List requiredFields = new ArrayList<>(schema.fields().size());
188 | Map properties = new LinkedHashMap<>(schema.fields().size());
189 | Map structVisitors = new LinkedHashMap<>();
190 | schema.fields().forEach(field -> {
191 | log.trace("toJsonSchema() - field:{} type:{}", field.name(), field.schema().type());
192 | List childVisitors = new ArrayList<>();
193 | if (!field.schema().isOptional()) {
194 | requiredFields.add(field.name());
195 | }
196 | if (Type.STRUCT == field.schema().type()) {
197 | Definition definition = definitions.computeIfAbsent(field.schema(), s -> {
198 | List definitionVisitors = new ArrayList<>();
199 | JSONObject fieldJsonSchema = toJsonSchema(field.schema(), definitions, definitionVisitors);
200 | String definitionName = field.schema().name().toLowerCase();
201 | return Definition.of(fieldJsonSchema, definitionName, definitionVisitors);
202 | });
203 | childVisitors.addAll(definition.visitors());
204 | properties.put(field.name(), definition.ref());
205 | } else {
206 | JSONObject fieldJsonSchema = toJsonSchema(field.schema(), definitions, childVisitors);
207 | properties.put(field.name(), fieldJsonSchema);
208 | }
209 | FromConnectVisitor fieldVisitor = childVisitors.get(0);
210 | structVisitors.put(field.name(), fieldVisitor);
211 | });
212 | result.put("properties", properties);
213 | result.put("required", requiredFields);
214 | result.put("type", "object");
215 | visitors.add(new FromConnectVisitor.StructVisitor(structVisitors));
216 | }
217 |
218 |
219 | log.trace("toJsonSchema() - '{}' is not primitive.", schema.type());
220 |
221 | return result;
222 | }
223 |
224 | static class Definition {
225 | private final JSONObject jsonSchema;
226 | private final String name;
227 | private final List visitors;
228 |
229 |
230 | private Definition(JSONObject jsonSchema, String name, List visitors) {
231 | this.jsonSchema = jsonSchema;
232 | this.name = name;
233 | this.visitors = visitors;
234 | }
235 |
236 | public static Definition of(JSONObject jsonSchema, String ref, List visitors) {
237 | return new Definition(jsonSchema, ref, visitors);
238 | }
239 |
240 | public JSONObject jsonSchema() {
241 | return this.jsonSchema;
242 | }
243 |
244 | public String name() {
245 | return this.name;
246 | }
247 |
248 | public JSONObject ref() {
249 | return new JSONObject()
250 | .put("$ref", String.format("#/definitions/%s", this.name));
251 | }
252 |
253 | public List visitors() {
254 | return this.visitors;
255 | }
256 |
257 | @Override
258 | public String toString() {
259 | return MoreObjects.toStringHelper(this)
260 | .add("jsonSchema", jsonSchema)
261 | .add("name", name)
262 | .toString();
263 | }
264 | }
265 |
266 |
267 | }
268 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/FromConnectState.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import org.apache.kafka.common.header.Header;
19 |
20 | class FromConnectState {
21 | final Header header;
22 | final FromConnectVisitor visitor;
23 |
24 |
25 | private FromConnectState(Header header, FromConnectVisitor visitor) {
26 | this.header = header;
27 | this.visitor = visitor;
28 | }
29 |
30 | public static FromConnectState of(Header header, FromConnectVisitor visitor) {
31 | return new FromConnectState(header, visitor);
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/FromConnectVisitor.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import com.fasterxml.jackson.core.JsonGenerator;
19 | import com.google.common.io.BaseEncoding;
20 | import org.apache.kafka.connect.data.Struct;
21 |
22 | import java.io.IOException;
23 | import java.math.BigDecimal;
24 | import java.text.DecimalFormat;
25 | import java.util.Date;
26 | import java.util.List;
27 | import java.util.Map;
28 |
29 | public abstract class FromConnectVisitor {
30 | public abstract void doVisit(JsonGenerator jsonGenerator, T value) throws IOException;
31 |
32 | public static class StringVisitor extends FromConnectVisitor {
33 | @Override
34 | public void doVisit(JsonGenerator jsonGenerator, String value) throws IOException {
35 | jsonGenerator.writeString(value);
36 | }
37 | }
38 |
39 | public static class StructVisitor extends FromConnectVisitor {
40 | final Map visitors;
41 |
42 | public StructVisitor(Map visitors) {
43 | this.visitors = visitors;
44 | }
45 |
46 | @Override
47 | public void doVisit(JsonGenerator jsonGenerator, Struct value) throws IOException {
48 | jsonGenerator.writeStartObject();
49 | for (Map.Entry e : this.visitors.entrySet()) {
50 | final String fieldName = e.getKey();
51 | final FromConnectVisitor visitor = e.getValue();
52 | final Object fieldValue = value.get(fieldName);
53 | jsonGenerator.writeFieldName(fieldName);
54 | visitor.doVisit(jsonGenerator, fieldValue);
55 | }
56 | jsonGenerator.writeEndObject();
57 | }
58 | }
59 |
60 | public static class BooleanVisitor extends FromConnectVisitor {
61 | @Override
62 | public void doVisit(JsonGenerator jsonGenerator, Boolean value) throws IOException {
63 | jsonGenerator.writeBoolean(value);
64 | }
65 | }
66 |
67 | public static class BytesVisitor extends FromConnectVisitor {
68 | @Override
69 | public void doVisit(JsonGenerator jsonGenerator, byte[] value) throws IOException {
70 | jsonGenerator.writeString(
71 | BaseEncoding.base64().encode(value)
72 | );
73 | }
74 | }
75 |
76 | public static class FloatVisitor extends FromConnectVisitor {
77 |
78 | @Override
79 | public void doVisit(JsonGenerator jsonGenerator, Number value) throws IOException {
80 | jsonGenerator.writeNumber(value.doubleValue());
81 | }
82 | }
83 |
84 | public static class DateTimeVisitor extends FromConnectVisitor {
85 | @Override
86 | public void doVisit(JsonGenerator jsonGenerator, Date value) throws IOException {
87 | jsonGenerator.writeString(
88 | Utils.TIMESTAMP_FORMATTER.format(
89 | value.toInstant()
90 | )
91 | );
92 | }
93 | }
94 |
95 | public static class DateVisitor extends FromConnectVisitor {
96 | @Override
97 | public void doVisit(JsonGenerator jsonGenerator, Date value) throws IOException {
98 | jsonGenerator.writeString(
99 | Utils.DATE_FORMATTER.format(
100 | value.toInstant()
101 | )
102 | );
103 | }
104 | }
105 |
106 | public static class TimeVisitor extends FromConnectVisitor {
107 | @Override
108 | public void doVisit(JsonGenerator jsonGenerator, Date value) throws IOException {
109 | jsonGenerator.writeString(
110 | Utils.TIME_FORMATTER.format(
111 | value.toInstant()
112 | )
113 | );
114 | }
115 | }
116 |
117 | public static class IntegerVisitor extends FromConnectVisitor {
118 |
119 |
120 | @Override
121 | public void doVisit(JsonGenerator jsonGenerator, Number value) throws IOException {
122 | jsonGenerator.writeNumber(value.longValue());
123 | }
124 | }
125 |
126 | public static class ArrayVisitor extends FromConnectVisitor {
127 | final FromConnectVisitor elementVisitor;
128 |
129 | public ArrayVisitor(FromConnectVisitor elementVisitor) {
130 | this.elementVisitor = elementVisitor;
131 | }
132 |
133 | @Override
134 | public void doVisit(JsonGenerator jsonGenerator, List value) throws IOException {
135 | jsonGenerator.writeStartArray();
136 | for (Object o : value) {
137 | this.elementVisitor.doVisit(jsonGenerator, o);
138 | }
139 | jsonGenerator.writeEndArray();
140 | }
141 | }
142 |
143 | public static class DecimalVisitor extends FromConnectVisitor {
144 | DecimalFormat decimalFormat;
145 |
146 | public DecimalVisitor(int scale) {
147 | this.decimalFormat = new DecimalFormat("#");
148 | this.decimalFormat.setMaximumFractionDigits(scale);
149 | }
150 |
151 | @Override
152 | public void doVisit(JsonGenerator jsonGenerator, BigDecimal value) throws IOException {
153 | jsonGenerator.writeString(
154 | this.decimalFormat.format(value)
155 | );
156 | }
157 | }
158 | }
159 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/FromJson.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import com.fasterxml.jackson.databind.JsonNode;
19 | import com.fasterxml.jackson.databind.ObjectMapper;
20 | import com.github.jcustenborder.kafka.connect.utils.config.Description;
21 | import com.github.jcustenborder.kafka.connect.utils.config.DocumentationTip;
22 | import com.github.jcustenborder.kafka.connect.utils.config.Title;
23 | import com.github.jcustenborder.kafka.connect.utils.transformation.BaseKeyValueTransformation;
24 | import org.apache.kafka.common.config.ConfigDef;
25 | import org.apache.kafka.common.config.ConfigException;
26 | import org.apache.kafka.connect.connector.ConnectRecord;
27 | import org.apache.kafka.connect.data.Schema;
28 | import org.apache.kafka.connect.data.SchemaAndValue;
29 | import org.apache.kafka.connect.errors.DataException;
30 | import org.everit.json.schema.ValidationException;
31 | import org.json.JSONObject;
32 | import org.slf4j.Logger;
33 | import org.slf4j.LoggerFactory;
34 |
35 | import java.io.ByteArrayInputStream;
36 | import java.io.IOException;
37 | import java.io.InputStream;
38 | import java.io.Reader;
39 | import java.io.StringReader;
40 | import java.util.Map;
41 |
42 | @Title("From Json transformation")
43 | @Description("The FromJson will read JSON data that is in string on byte form and parse the data to " +
44 | "a connect structure based on the JSON schema provided.")
45 | @DocumentationTip("This transformation expects data to be in either String or Byte format. You are " +
46 | "most likely going to use the ByteArrayConverter or the StringConverter.")
47 | public class FromJson> extends BaseKeyValueTransformation {
48 | private static final Logger log = LoggerFactory.getLogger(FromJson.class);
49 | FromJsonConfig config;
50 |
51 | protected FromJson(boolean isKey) {
52 | super(isKey);
53 | }
54 |
55 | @Override
56 | public ConfigDef config() {
57 | return FromJsonConfig.config();
58 | }
59 |
60 | @Override
61 | public void close() {
62 |
63 | }
64 |
65 | SchemaAndValue processJsonNode(R record, Schema inputSchema, JsonNode node) {
66 | Object result = this.fromJsonState.visitor.visit(node);
67 | return new SchemaAndValue(this.fromJsonState.schema, result);
68 | }
69 |
70 |
71 | void validateJson(JSONObject jsonObject) {
72 | try {
73 | this.fromJsonState.jsonSchema.validate(jsonObject);
74 | } catch (ValidationException ex) {
75 | StringBuilder builder = new StringBuilder();
76 | builder.append(
77 | String.format(
78 | "Could not validate JSON. Found %s violations(s).",
79 | ex.getViolationCount()
80 | )
81 | );
82 | for (ValidationException message : ex.getCausingExceptions()) {
83 | log.error("Validation exception", message);
84 | builder.append("\n");
85 | builder.append(message.getMessage());
86 | }
87 | throw new DataException(
88 | builder.toString(),
89 | ex
90 | );
91 | }
92 | }
93 |
94 |
95 | @Override
96 | protected SchemaAndValue processBytes(R record, Schema inputSchema, byte[] input) {
97 | try {
98 | if (this.config.validateJson) {
99 | try (InputStream inputStream = new ByteArrayInputStream(input)) {
100 | JSONObject jsonObject = Utils.loadObject(inputStream);
101 | validateJson(jsonObject);
102 | }
103 | }
104 | JsonNode node = this.objectMapper.readValue(input, JsonNode.class);
105 | return processJsonNode(record, inputSchema, node);
106 | } catch (IOException e) {
107 | throw new DataException(e);
108 | }
109 | }
110 |
111 | @Override
112 | protected SchemaAndValue processString(R record, Schema inputSchema, String input) {
113 | try {
114 | if (this.config.validateJson) {
115 | try (Reader reader = new StringReader(input)) {
116 | JSONObject jsonObject = Utils.loadObject(reader);
117 | validateJson(jsonObject);
118 | }
119 | }
120 | JsonNode node = this.objectMapper.readValue(input, JsonNode.class);
121 | return processJsonNode(record, inputSchema, node);
122 | } catch (IOException e) {
123 | throw new DataException(e);
124 | }
125 | }
126 |
127 | FromJsonState fromJsonState;
128 | FromJsonSchemaConverterFactory fromJsonSchemaConverterFactory;
129 | ObjectMapper objectMapper;
130 |
131 | @Override
132 | public void configure(Map map) {
133 | this.config = new FromJsonConfig(map);
134 | this.fromJsonSchemaConverterFactory = new FromJsonSchemaConverterFactory(config);
135 |
136 | org.everit.json.schema.Schema schema;
137 | if (JsonConfig.SchemaLocation.Url == this.config.schemaLocation) {
138 | try {
139 | try (InputStream inputStream = this.config.schemaUrl.openStream()) {
140 | schema = Utils.loadSchema(inputStream);
141 | }
142 | } catch (IOException e) {
143 | ConfigException exception = new ConfigException(JsonConfig.SCHEMA_URL_CONF, this.config.schemaUrl, "exception while loading schema");
144 | exception.initCause(e);
145 | throw exception;
146 | }
147 | } else if (JsonConfig.SchemaLocation.Inline == this.config.schemaLocation) {
148 | schema = Utils.loadSchema(this.config.schemaText);
149 | } else {
150 | throw new ConfigException(
151 | JsonConfig.SCHEMA_LOCATION_CONF,
152 | this.config.schemaLocation.toString(),
153 | "Location is not supported"
154 | );
155 | }
156 |
157 | this.fromJsonState = this.fromJsonSchemaConverterFactory.fromJSON(schema);
158 | this.objectMapper = JacksonFactory.create();
159 | }
160 |
161 | public static class Key> extends FromJson {
162 | public Key() {
163 | super(true);
164 | }
165 | }
166 |
167 | public static class Value> extends FromJson {
168 | public Value() {
169 | super(false);
170 | }
171 | }
172 | }
173 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/FromJsonConfig.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import org.apache.kafka.common.config.ConfigDef;
19 |
20 | import java.util.Map;
21 |
22 | class FromJsonConfig extends JsonConfig {
23 |
24 |
25 | public FromJsonConfig(Map, ?> originals) {
26 | super(config(), originals);
27 | }
28 |
29 | public static ConfigDef config() {
30 | return JsonConfig.config();
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/FromJsonConversionKey.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import com.google.common.base.MoreObjects;
19 | import com.google.common.base.Objects;
20 | import org.everit.json.schema.NumberSchema;
21 | import org.everit.json.schema.Schema;
22 | import org.everit.json.schema.StringSchema;
23 | import org.slf4j.Logger;
24 | import org.slf4j.LoggerFactory;
25 |
26 | class FromJsonConversionKey {
27 | static final String UNNAMED_FORMAT = "unnamed-format";
28 | private static final Logger log = LoggerFactory.getLogger(FromJsonConversionKey.class);
29 | final Class extends org.everit.json.schema.Schema> schemaClass;
30 | final String format;
31 | final Boolean requiresInteger;
32 | final String contentEncoding;
33 |
34 | private FromJsonConversionKey(Class extends Schema> schemaClass, String format, Boolean requiresInteger, String contentEncoding) {
35 | this.schemaClass = schemaClass;
36 | this.format = format;
37 | this.requiresInteger = requiresInteger;
38 | this.contentEncoding = contentEncoding;
39 | }
40 |
41 | public static FromJsonConversionKey of(org.everit.json.schema.Schema jsonSchema) {
42 | String format;
43 | Boolean requiresInteger;
44 | String contentEncoding;
45 | if (jsonSchema instanceof StringSchema) {
46 | StringSchema stringSchema = (StringSchema) jsonSchema;
47 | format = UNNAMED_FORMAT.equals(stringSchema.getFormatValidator().formatName()) ? null : stringSchema.getFormatValidator().formatName();
48 | contentEncoding = (String) stringSchema.getUnprocessedProperties().get("contentEncoding");
49 | requiresInteger = null;
50 | log.trace("jsonSchema = '{}' format = '{}'", jsonSchema, format);
51 | } else if (jsonSchema instanceof NumberSchema) {
52 | NumberSchema numberSchema = (NumberSchema) jsonSchema;
53 | requiresInteger = numberSchema.requiresInteger();
54 | format = null;
55 | contentEncoding = null;
56 | } else {
57 | format = null;
58 | requiresInteger = null;
59 | contentEncoding = null;
60 | }
61 |
62 | return new FromJsonConversionKey(jsonSchema.getClass(), format, requiresInteger, contentEncoding);
63 | }
64 |
65 | public static Builder from(Class extends org.everit.json.schema.Schema> schemaClass) {
66 | return new Builder().schemaClass(schemaClass);
67 | }
68 |
69 | @Override
70 | public boolean equals(Object o) {
71 | if (this == o) return true;
72 | if (o == null || getClass() != o.getClass()) return false;
73 | FromJsonConversionKey that = (FromJsonConversionKey) o;
74 | return Objects.equal(schemaClass, that.schemaClass) &&
75 | Objects.equal(format, that.format) &&
76 | Objects.equal(requiresInteger, that.requiresInteger) &&
77 | Objects.equal(contentEncoding, that.contentEncoding);
78 | }
79 |
80 | @Override
81 | public int hashCode() {
82 | return Objects.hashCode(schemaClass, format, requiresInteger, contentEncoding);
83 | }
84 |
85 | // public static ConversionKey of(Class extends org.everit.json.schema.Schema> schemaClass) {
86 | // return new ConversionKey(schemaClass, null, null, contentMediaType);
87 | // }
88 | //
89 | // public static ConversionKey of(Class extends org.everit.json.schema.Schema> schemaClass, String format) {
90 | // return new ConversionKey(schemaClass, format, null, contentMediaType);
91 | // }
92 | //
93 | // public static ConversionKey of(Class extends org.everit.json.schema.Schema> schemaClass, Boolean requiesInteger) {
94 | // return new ConversionKey(schemaClass, null, requiesInteger, contentMediaType);
95 | // }
96 |
97 | @Override
98 | public String toString() {
99 | return MoreObjects.toStringHelper(this)
100 | .add("schemaClass", schemaClass)
101 | .add("format", format)
102 | .add("requiresInteger", requiresInteger)
103 | .add("contentEncoding", contentEncoding)
104 | .toString();
105 | }
106 |
107 | static class Builder {
108 | Class extends org.everit.json.schema.Schema> schemaClass;
109 | String format;
110 | Boolean requiresInteger;
111 | String contentEncoding;
112 | private Builder() {
113 | }
114 |
115 | public Builder schemaClass(Class extends Schema> schemaClass) {
116 | this.schemaClass = schemaClass;
117 | return this;
118 | }
119 |
120 | public Builder format(String format) {
121 | this.format = format;
122 | return this;
123 | }
124 |
125 | public Builder requiresInteger(Boolean requiresInteger) {
126 | this.requiresInteger = requiresInteger;
127 | return this;
128 | }
129 |
130 | public Builder contentEncoding(String contentMediaType) {
131 | this.contentEncoding = contentMediaType;
132 | return this;
133 | }
134 |
135 | public FromJsonConversionKey build() {
136 | return new FromJsonConversionKey(this.schemaClass, this.format, this.requiresInteger, this.contentEncoding);
137 | }
138 | }
139 |
140 | }
141 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/FromJsonSchemaConverter.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import com.fasterxml.jackson.databind.JsonNode;
19 | import com.fasterxml.jackson.databind.node.ArrayNode;
20 | import com.fasterxml.jackson.databind.node.BooleanNode;
21 | import com.fasterxml.jackson.databind.node.NumericNode;
22 | import com.fasterxml.jackson.databind.node.ObjectNode;
23 | import com.fasterxml.jackson.databind.node.TextNode;
24 | import com.google.common.base.Preconditions;
25 | import com.google.common.collect.ImmutableSet;
26 | import org.apache.kafka.connect.data.Date;
27 | import org.apache.kafka.connect.data.Decimal;
28 | import org.apache.kafka.connect.data.Schema;
29 | import org.apache.kafka.connect.data.SchemaBuilder;
30 | import org.apache.kafka.connect.data.Struct;
31 | import org.apache.kafka.connect.data.Time;
32 | import org.apache.kafka.connect.data.Timestamp;
33 | import org.everit.json.schema.ArraySchema;
34 | import org.everit.json.schema.BooleanSchema;
35 | import org.everit.json.schema.NumberSchema;
36 | import org.everit.json.schema.ObjectSchema;
37 | import org.everit.json.schema.StringSchema;
38 | import org.slf4j.Logger;
39 | import org.slf4j.LoggerFactory;
40 |
41 | import java.util.List;
42 | import java.util.Map;
43 | import java.util.Set;
44 |
45 | public abstract class FromJsonSchemaConverter {
46 | private static final Logger log = LoggerFactory.getLogger(FromJsonSchemaConverter.class);
47 | protected final FromJsonSchemaConverterFactory factory;
48 | protected final JsonConfig config;
49 |
50 | protected FromJsonSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) {
51 | this.factory = factory;
52 | this.config = config;
53 | }
54 |
55 | protected abstract SchemaBuilder schemaBuilder(T schema);
56 |
57 | protected abstract FromJsonConversionKey key();
58 |
59 | protected abstract FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors);
60 |
61 | protected abstract void fromJSON(SchemaBuilder builder, T jsonSchema, Map visitors);
62 |
63 | static class BooleanSchemaConverter extends FromJsonSchemaConverter {
64 | BooleanSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) {
65 | super(factory, config);
66 | }
67 |
68 | @Override
69 | protected SchemaBuilder schemaBuilder(BooleanSchema schema) {
70 | return SchemaBuilder.bool();
71 | }
72 |
73 | @Override
74 | protected FromJsonConversionKey key() {
75 | return FromJsonConversionKey.from(BooleanSchema.class).build();
76 | }
77 |
78 | @Override
79 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) {
80 | return new FromJsonVisitor.BooleanVisitor(connectSchema);
81 | }
82 |
83 | @Override
84 | protected void fromJSON(SchemaBuilder builder, BooleanSchema jsonSchema, Map visitors) {
85 |
86 | }
87 | }
88 |
89 | static class ObjectSchemaConverter extends FromJsonSchemaConverter {
90 |
91 | ObjectSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) {
92 | super(factory, config);
93 | }
94 |
95 | @Override
96 | protected SchemaBuilder schemaBuilder(ObjectSchema schema) {
97 | return SchemaBuilder.struct();
98 | }
99 |
100 | @Override
101 | protected FromJsonConversionKey key() {
102 | return FromJsonConversionKey.from(ObjectSchema.class).build();
103 | }
104 |
105 | @Override
106 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) {
107 | return new FromJsonVisitor.StructVisitor(connectSchema, visitors);
108 | }
109 |
110 | static final Set EXCLUDE_PROPERTIES = ImmutableSet.of("$schema");
111 |
112 | @Override
113 | protected void fromJSON(SchemaBuilder builder, ObjectSchema jsonSchema, Map visitors) {
114 | Set requiredProperties = ImmutableSet.copyOf(jsonSchema.getRequiredProperties());
115 | jsonSchema.getPropertySchemas()
116 | .entrySet()
117 | .stream()
118 | .filter(e -> {
119 | boolean result = !EXCLUDE_PROPERTIES.contains(e.getKey());
120 | log.trace("fromJson() - filtering '{}' result = '{}'", e.getKey(), result);
121 | return result;
122 | })
123 | .filter(e -> {
124 | String schemaLocation = e.getValue().getSchemaLocation();
125 | boolean result = !this.config.excludeLocations.contains(schemaLocation);
126 | log.trace("fromJson() - filtering '{}' location='{}' result = '{}'", e.getKey(), e.getValue().getSchemaLocation(), result);
127 | return result;
128 | })
129 | .sorted(Map.Entry.comparingByKey())
130 | .forEach(e -> {
131 | final String propertyName = e.getKey();
132 | final org.everit.json.schema.Schema propertyJsonSchema = e.getValue();
133 | final boolean isOptional = !requiredProperties.contains(propertyName);
134 | log.trace("fromJson() - Processing property '{}' '{}'", propertyName, propertyJsonSchema);
135 | FromJsonState state = this.factory.fromJSON(propertyJsonSchema, isOptional);
136 | builder.field(propertyName, state.schema);
137 | visitors.put(propertyName, state.visitor);
138 | });
139 | }
140 | }
141 |
142 | static class IntegerSchemaConverter extends FromJsonSchemaConverter {
143 |
144 | IntegerSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) {
145 | super(factory, config);
146 | }
147 |
148 | @Override
149 | protected SchemaBuilder schemaBuilder(NumberSchema schema) {
150 | return SchemaBuilder.int64();
151 | }
152 |
153 | @Override
154 | protected FromJsonConversionKey key() {
155 | return FromJsonConversionKey.from(NumberSchema.class)
156 | .requiresInteger(true)
157 | .build();
158 | }
159 |
160 | @Override
161 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) {
162 | return new FromJsonVisitor.IntegerVisitor(connectSchema);
163 | }
164 |
165 | @Override
166 | protected void fromJSON(SchemaBuilder builder, NumberSchema jsonSchema, Map visitors) {
167 | log.trace("fromJson() - Processing '{}'", jsonSchema);
168 | }
169 | }
170 |
171 | static class FloatSchemaConverter extends FromJsonSchemaConverter {
172 |
173 | FloatSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) {
174 | super(factory, config);
175 | }
176 |
177 | @Override
178 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) {
179 | return new FromJsonVisitor.FloatVisitor(connectSchema);
180 | }
181 |
182 | @Override
183 | protected SchemaBuilder schemaBuilder(NumberSchema schema) {
184 | return SchemaBuilder.float64();
185 | }
186 |
187 | @Override
188 | protected FromJsonConversionKey key() {
189 | return FromJsonConversionKey.from(NumberSchema.class)
190 | .requiresInteger(false)
191 | .build();
192 | }
193 |
194 | @Override
195 | protected void fromJSON(SchemaBuilder builder, NumberSchema jsonSchema, Map visitors) {
196 | log.trace("fromJson() - Processing '{}'", jsonSchema);
197 | }
198 | }
199 |
200 | static class StringSchemaConverter extends FromJsonSchemaConverter {
201 |
202 | StringSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) {
203 | super(factory, config);
204 | }
205 |
206 | @Override
207 | protected SchemaBuilder schemaBuilder(StringSchema schema) {
208 | return SchemaBuilder.string();
209 | }
210 |
211 | @Override
212 | protected FromJsonConversionKey key() {
213 | return FromJsonConversionKey.from(StringSchema.class).build();
214 | }
215 |
216 | @Override
217 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) {
218 | return new FromJsonVisitor.StringVisitor(connectSchema);
219 | }
220 |
221 | @Override
222 | protected void fromJSON(SchemaBuilder builder, StringSchema jsonSchema, Map visitors) {
223 | log.trace("fromJson() - Processing '{}'", jsonSchema);
224 | }
225 | }
226 |
227 | static class DateSchemaConverter extends FromJsonSchemaConverter {
228 |
229 | DateSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) {
230 | super(factory, config);
231 | }
232 |
233 | @Override
234 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) {
235 | return new FromJsonVisitor.DateVisitor(connectSchema);
236 | }
237 |
238 | @Override
239 | protected SchemaBuilder schemaBuilder(StringSchema schema) {
240 | return Date.builder();
241 | }
242 |
243 | @Override
244 | protected FromJsonConversionKey key() {
245 | return FromJsonConversionKey.from(StringSchema.class)
246 | .format("date")
247 | .build();
248 | }
249 |
250 | @Override
251 | protected void fromJSON(SchemaBuilder builder, StringSchema jsonSchema, Map visitors) {
252 | log.trace("fromJson() - Processing '{}'", jsonSchema);
253 | }
254 | }
255 |
256 | static class TimeSchemaConverter extends FromJsonSchemaConverter {
257 |
258 | TimeSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) {
259 | super(factory, config);
260 | }
261 |
262 | @Override
263 | protected SchemaBuilder schemaBuilder(StringSchema schema) {
264 | return Time.builder();
265 | }
266 |
267 | @Override
268 | protected FromJsonConversionKey key() {
269 | return FromJsonConversionKey.from(StringSchema.class)
270 | .format("time")
271 | .build();
272 | }
273 |
274 | @Override
275 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) {
276 | return new FromJsonVisitor.TimeVisitor(connectSchema);
277 | }
278 |
279 | @Override
280 | protected void fromJSON(SchemaBuilder builder, StringSchema jsonSchema, Map visitors) {
281 | log.trace("fromJson() - Processing '{}'", jsonSchema);
282 | }
283 | }
284 |
285 | static class DateTimeSchemaConverter extends FromJsonSchemaConverter {
286 |
287 | DateTimeSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) {
288 | super(factory, config);
289 | }
290 |
291 | @Override
292 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) {
293 | return new FromJsonVisitor.DateTimeVisitor(connectSchema);
294 | }
295 |
296 | @Override
297 | protected SchemaBuilder schemaBuilder(StringSchema schema) {
298 | return Timestamp.builder();
299 | }
300 |
301 | @Override
302 | protected FromJsonConversionKey key() {
303 | return FromJsonConversionKey.from(StringSchema.class)
304 | .format("date-time")
305 | .build();
306 | }
307 |
308 | @Override
309 | protected void fromJSON(SchemaBuilder builder, StringSchema jsonSchema, Map visitors) {
310 | log.trace("fromJson() - Processing '{}'", jsonSchema);
311 | }
312 | }
313 |
314 | static class BytesSchemaConverter extends FromJsonSchemaConverter {
315 |
316 | BytesSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) {
317 | super(factory, config);
318 | }
319 |
320 | @Override
321 | protected SchemaBuilder schemaBuilder(StringSchema schema) {
322 | return SchemaBuilder.bytes();
323 | }
324 |
325 | @Override
326 | protected FromJsonConversionKey key() {
327 | return FromJsonConversionKey.from(StringSchema.class)
328 | .contentEncoding("base64")
329 | .build();
330 | }
331 |
332 | @Override
333 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) {
334 | return new FromJsonVisitor.BytesVisitor(connectSchema);
335 | }
336 |
337 | @Override
338 | protected void fromJSON(SchemaBuilder builder, StringSchema jsonSchema, Map visitors) {
339 |
340 | }
341 | }
342 |
343 | static class DecimalSchemaConverter extends FromJsonSchemaConverter {
344 | public DecimalSchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) {
345 | super(factory, config);
346 | }
347 |
348 | @Override
349 | protected SchemaBuilder schemaBuilder(StringSchema schema) {
350 | int scale = Utils.scale(schema);
351 | return Decimal.builder(scale);
352 | }
353 |
354 | @Override
355 | protected FromJsonConversionKey key() {
356 | return FromJsonConversionKey.from(StringSchema.class)
357 | .format("decimal")
358 | .build();
359 | }
360 |
361 | @Override
362 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) {
363 | int scale = Utils.scale(connectSchema);
364 | return new FromJsonVisitor.DecimalVisitor(connectSchema, scale);
365 | }
366 |
367 | @Override
368 | protected void fromJSON(SchemaBuilder builder, StringSchema jsonSchema, Map visitors) {
369 |
370 | }
371 | }
372 |
373 | static class ArraySchemaConverter extends FromJsonSchemaConverter {
374 |
375 | ArraySchemaConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) {
376 | super(factory, config);
377 | }
378 |
379 | @Override
380 | protected SchemaBuilder schemaBuilder(ArraySchema schema) {
381 | FromJsonState state = this.factory.fromJSON(schema.getAllItemSchema());
382 | return SchemaBuilder.array(state.schema);
383 | }
384 |
385 | @Override
386 | protected FromJsonConversionKey key() {
387 | return FromJsonConversionKey.from(ArraySchema.class).build();
388 | }
389 |
390 | @Override
391 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) {
392 | FromJsonVisitor visitor = visitors.get("item");
393 | return new FromJsonVisitor.ArrayVisitor(connectSchema, visitor);
394 | }
395 |
396 | @Override
397 | protected void fromJSON(SchemaBuilder builder, ArraySchema jsonSchema, Map visitors) {
398 | FromJsonState state = this.factory.fromJSON(jsonSchema.getAllItemSchema());
399 | visitors.put("item", state.visitor);
400 | }
401 | }
402 |
403 | static class CustomTimestampConverter extends FromJsonSchemaConverter {
404 |
405 | CustomTimestampConverter(FromJsonSchemaConverterFactory factory, JsonConfig config) {
406 | super(factory, config);
407 | }
408 |
409 | @Override
410 | protected FromJsonVisitor jsonVisitor(Schema connectSchema, Map visitors) {
411 | return new FromJsonVisitor.CustomDateVisitor(connectSchema);
412 | }
413 |
414 | @Override
415 | protected SchemaBuilder schemaBuilder(StringSchema schema) {
416 | Object dateTimeFormat = schema.getUnprocessedProperties().get("dateTimeFormat");
417 | Preconditions.checkNotNull(dateTimeFormat, "dateTimeFormat cannot be null");
418 | return Timestamp.builder()
419 | .parameter("dateFormat", dateTimeFormat.toString());
420 | }
421 |
422 | @Override
423 | protected FromJsonConversionKey key() {
424 | return FromJsonConversionKey.from(StringSchema.class)
425 | .format("custom-timestamp")
426 | .build();
427 | }
428 |
429 | @Override
430 | protected void fromJSON(SchemaBuilder builder, StringSchema jsonSchema, Map visitors) {
431 | log.trace("fromJson() - Processing '{}'", jsonSchema);
432 | }
433 | }
434 | }
435 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/FromJsonSchemaConverterFactory.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import com.fasterxml.jackson.databind.JsonNode;
19 | import com.google.common.base.Joiner;
20 | import com.google.common.base.Strings;
21 | import org.apache.kafka.connect.data.SchemaBuilder;
22 | import org.everit.json.schema.CombinedSchema;
23 | import org.everit.json.schema.NullSchema;
24 | import org.everit.json.schema.ObjectSchema;
25 | import org.everit.json.schema.ReferenceSchema;
26 | import org.everit.json.schema.Schema;
27 | import org.everit.json.schema.StringSchema;
28 | import org.slf4j.Logger;
29 | import org.slf4j.LoggerFactory;
30 |
31 | import java.util.Collections;
32 | import java.util.LinkedHashMap;
33 | import java.util.List;
34 | import java.util.Map;
35 | import java.util.stream.Collectors;
36 | import java.util.stream.Stream;
37 |
38 | public class FromJsonSchemaConverterFactory {
39 | private static final Logger log = LoggerFactory.getLogger(FromJsonSchemaConverterFactory.class);
40 | private final Map<
41 | FromJsonConversionKey,
42 | FromJsonSchemaConverter extends Schema, ? extends JsonNode, ?>
43 | > lookup;
44 | private final JsonConfig config;
45 | private final FromJsonConversionKey genericStringKey;
46 |
47 | public FromJsonSchemaConverterFactory(JsonConfig config) {
48 | this.config = config;
49 | FromJsonSchemaConverter.StringSchemaConverter stringSchemaConverter = new FromJsonSchemaConverter.StringSchemaConverter(this, config);
50 | genericStringKey = stringSchemaConverter.key();
51 | lookup = Stream.of(
52 | stringSchemaConverter,
53 | new FromJsonSchemaConverter.ObjectSchemaConverter(this, config),
54 | new FromJsonSchemaConverter.IntegerSchemaConverter(this, config),
55 | new FromJsonSchemaConverter.BooleanSchemaConverter(this, config),
56 | new FromJsonSchemaConverter.TimeSchemaConverter(this, config),
57 | new FromJsonSchemaConverter.DateSchemaConverter(this, config),
58 | new FromJsonSchemaConverter.DateTimeSchemaConverter(this, config),
59 | new FromJsonSchemaConverter.FloatSchemaConverter(this, config),
60 | new FromJsonSchemaConverter.ArraySchemaConverter(this, config),
61 | new FromJsonSchemaConverter.BytesSchemaConverter(this, config),
62 | new FromJsonSchemaConverter.DecimalSchemaConverter(this, config),
63 | new FromJsonSchemaConverter.CustomTimestampConverter(this, config)
64 | ).collect(Collectors.toMap(FromJsonSchemaConverter::key, c -> c));
65 | }
66 |
67 | public FromJsonState fromJSON(org.everit.json.schema.Schema jsonSchema) {
68 | return fromJSON(jsonSchema, false);
69 | }
70 |
71 | public FromJsonState fromJSON(org.everit.json.schema.Schema jsonSchema, boolean isOptional) {
72 | final String description;
73 |
74 | if (jsonSchema instanceof ReferenceSchema) {
75 | ReferenceSchema referenceSchema = (ReferenceSchema) jsonSchema;
76 | jsonSchema = referenceSchema.getReferredSchema();
77 | description = jsonSchema.getDescription();
78 | } else if (jsonSchema instanceof CombinedSchema) {
79 | CombinedSchema combinedSchema = (CombinedSchema) jsonSchema;
80 | description = combinedSchema.getDescription();
81 | List nonNullSubSchemas = combinedSchema
82 | .getSubschemas()
83 | .stream()
84 | .filter(s -> !(s instanceof NullSchema))
85 | .collect(Collectors.toList());
86 | if (1 != nonNullSubSchemas.size()) {
87 | throw new UnsupportedOperationException(
88 | String.format(
89 | "More than one choice for non null schemas. Schema location %s: %s",
90 | jsonSchema.getSchemaLocation(),
91 | Joiner.on(", ").join(nonNullSubSchemas)
92 | )
93 | );
94 | }
95 | jsonSchema = nonNullSubSchemas.get(0);
96 | } else {
97 | description = jsonSchema.getDescription();
98 | }
99 | FromJsonConversionKey key = FromJsonConversionKey.of(jsonSchema);
100 |
101 | FromJsonSchemaConverter converter = lookup.get(key);
102 |
103 | if (null == converter && jsonSchema instanceof StringSchema) {
104 | log.trace("fromJSON() - falling back to string passthrough for {}", jsonSchema);
105 | converter = lookup.get(genericStringKey);
106 | }
107 |
108 | if (null == converter) {
109 | throw new UnsupportedOperationException(
110 | String.format("Schema type is not supported. %s:%s", jsonSchema.getClass().getName(), jsonSchema)
111 | );
112 | }
113 |
114 | SchemaBuilder builder = converter.schemaBuilder(jsonSchema);
115 | if (jsonSchema instanceof ObjectSchema) {
116 | ObjectSchema objectSchema = (ObjectSchema) jsonSchema;
117 | String schemaName = schemaName(objectSchema);
118 | builder.name(schemaName);
119 | }
120 |
121 | if (!Strings.isNullOrEmpty(description)) {
122 | builder.doc(description);
123 | }
124 | if (isOptional) {
125 | builder.optional();
126 | }
127 | Map visitors = new LinkedHashMap<>();
128 | converter.fromJSON(builder, jsonSchema, visitors);
129 | org.apache.kafka.connect.data.Schema schema = builder.build();
130 | FromJsonVisitor visitor = converter.jsonVisitor(schema, visitors);
131 | return FromJsonState.of(jsonSchema, schema, visitor);
132 | }
133 |
134 | private List clean(String text) {
135 | List result;
136 |
137 | if (Strings.isNullOrEmpty(text)) {
138 | result = Collections.EMPTY_LIST;
139 | } else {
140 | result = Stream.of(text.split("[#\\\\/\\.]+"))
141 | .filter(p -> !Strings.isNullOrEmpty(p))
142 | .collect(Collectors.toList());
143 | }
144 |
145 | return result;
146 | }
147 |
148 | private String schemaName(ObjectSchema objectSchema) {
149 | final List parts;
150 |
151 | if (!Strings.isNullOrEmpty(objectSchema.getTitle())) {
152 | parts = clean(objectSchema.getTitle());
153 | } else if (!Strings.isNullOrEmpty(objectSchema.getSchemaLocation())) {
154 | parts = clean(objectSchema.getSchemaLocation());
155 | } else {
156 | parts = Collections.EMPTY_LIST;
157 | }
158 |
159 | return parts.isEmpty() ? null : Joiner.on('.').join(parts);
160 | }
161 |
162 | private String cleanName(String title) {
163 | String result = title.replaceAll("[#\\\\/]+", ".");
164 | return result;
165 | }
166 |
167 |
168 | }
169 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/FromJsonState.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 |
19 | class FromJsonState {
20 | public final org.everit.json.schema.Schema jsonSchema;
21 | public final org.apache.kafka.connect.data.Schema schema;
22 | public final FromJsonVisitor visitor;
23 |
24 | FromJsonState(org.everit.json.schema.Schema jsonSchema, org.apache.kafka.connect.data.Schema schema, FromJsonVisitor visitor) {
25 | this.jsonSchema = jsonSchema;
26 | this.schema = schema;
27 | this.visitor = visitor;
28 | }
29 |
30 | public static FromJsonState of(org.everit.json.schema.Schema jsonSchema, org.apache.kafka.connect.data.Schema schema, FromJsonVisitor visitor) {
31 | return new FromJsonState(jsonSchema, schema, visitor);
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/FromJsonVisitor.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import com.fasterxml.jackson.databind.JsonNode;
19 | import com.fasterxml.jackson.databind.node.ArrayNode;
20 | import com.fasterxml.jackson.databind.node.BooleanNode;
21 | import com.fasterxml.jackson.databind.node.NumericNode;
22 | import com.fasterxml.jackson.databind.node.ObjectNode;
23 | import com.fasterxml.jackson.databind.node.TextNode;
24 | import com.google.common.io.BaseEncoding;
25 | import org.apache.kafka.connect.data.Schema;
26 | import org.apache.kafka.connect.data.Struct;
27 | import org.apache.kafka.connect.errors.DataException;
28 | import org.slf4j.Logger;
29 | import org.slf4j.LoggerFactory;
30 |
31 | import java.text.DecimalFormat;
32 | import java.text.ParseException;
33 | import java.time.Instant;
34 | import java.time.LocalDate;
35 | import java.time.LocalDateTime;
36 | import java.time.LocalTime;
37 | import java.time.ZoneOffset;
38 | import java.time.format.DateTimeFormatter;
39 | import java.util.ArrayList;
40 | import java.util.Date;
41 | import java.util.List;
42 | import java.util.Map;
43 |
44 | public abstract class FromJsonVisitor {
45 | protected final Schema schema;
46 |
47 | protected FromJsonVisitor(Schema schema) {
48 | this.schema = schema;
49 | }
50 |
51 | public V visit(T node) {
52 | V result;
53 |
54 | if (null == node || node.isNull()) {
55 | result = null;
56 | } else {
57 | result = doVisit(node);
58 | }
59 |
60 | return result;
61 | }
62 |
63 | protected abstract V doVisit(T node);
64 |
65 | public static class StringVisitor extends FromJsonVisitor {
66 | public StringVisitor(Schema schema) {
67 | super(schema);
68 | }
69 |
70 | @Override
71 | public String doVisit(TextNode node) {
72 | return node.textValue();
73 | }
74 | }
75 |
76 | public static class BooleanVisitor extends FromJsonVisitor {
77 | public BooleanVisitor(Schema schema) {
78 | super(schema);
79 | }
80 |
81 | @Override
82 | protected Boolean doVisit(BooleanNode node) {
83 | return node.booleanValue();
84 | }
85 | }
86 |
87 | public static class StructVisitor extends FromJsonVisitor {
88 | private final Map visitors;
89 |
90 | public StructVisitor(Schema schema, Map visitors) {
91 | super(schema);
92 | this.visitors = visitors;
93 | }
94 |
95 | @Override
96 | protected Struct doVisit(ObjectNode node) {
97 | Struct result = new Struct(this.schema);
98 | visitors.forEach((fieldName, visitor) -> {
99 | try {
100 | JsonNode rawValue = node.get(fieldName);
101 | Object convertedValue = visitor.visit(rawValue);
102 | result.put(fieldName, convertedValue);
103 | } catch (Exception ex) {
104 | throw new IllegalStateException(
105 | String.format("Exception thrown while reading %s:%s", this.schema.name(), fieldName),
106 | ex
107 | );
108 | }
109 | });
110 |
111 | return result;
112 | }
113 | }
114 |
115 | public static class IntegerVisitor extends FromJsonVisitor {
116 | public IntegerVisitor(Schema schema) {
117 | super(schema);
118 | }
119 |
120 | @Override
121 | protected Number doVisit(NumericNode node) {
122 | return node.longValue();
123 | }
124 | }
125 |
126 | public static class FloatVisitor extends FromJsonVisitor {
127 | public FloatVisitor(Schema schema) {
128 | super(schema);
129 | }
130 |
131 | @Override
132 | protected Number doVisit(NumericNode node) {
133 | return node.doubleValue();
134 | }
135 | }
136 |
137 | public static class DateTimeVisitor extends FromJsonVisitor {
138 | private static final Logger log = LoggerFactory.getLogger(DateTimeVisitor.class);
139 |
140 | public DateTimeVisitor(Schema schema) {
141 | super(schema);
142 | }
143 |
144 | @Override
145 | protected Date doVisit(TextNode node) {
146 | log.trace(node.asText());
147 | LocalDateTime localDateTime = LocalDateTime.parse(node.asText(), Utils.TIMESTAMP_FORMATTER);
148 | Instant instant = localDateTime.toInstant(ZoneOffset.UTC);
149 | return Date.from(instant);
150 | }
151 | }
152 |
153 | public static class DateVisitor extends FromJsonVisitor {
154 | private static final Logger log = LoggerFactory.getLogger(DateTimeVisitor.class);
155 |
156 | public DateVisitor(Schema schema) {
157 | super(schema);
158 | }
159 |
160 | @Override
161 | protected Date doVisit(TextNode node) {
162 | log.trace(node.asText());
163 | LocalDate localDateTime = LocalDate.parse(node.asText(), Utils.DATE_FORMATTER);
164 | Instant instant = localDateTime.atStartOfDay().toInstant(ZoneOffset.UTC);
165 | return Date.from(instant);
166 | }
167 | }
168 |
169 | public static class TimeVisitor extends FromJsonVisitor {
170 | private static final Logger log = LoggerFactory.getLogger(DateTimeVisitor.class);
171 |
172 | public TimeVisitor(Schema schema) {
173 | super(schema);
174 | }
175 |
176 | @Override
177 | protected Date doVisit(TextNode node) {
178 | log.trace(node.asText());
179 | LocalTime localDateTime = LocalTime.parse(node.asText(), Utils.TIME_FORMATTER);
180 | Instant instant = LocalDate.ofEpochDay(0).atTime(localDateTime).toInstant(ZoneOffset.UTC);
181 | return Date.from(instant);
182 | }
183 | }
184 |
185 | public static class DecimalVisitor extends FromJsonVisitor {
186 | final int scale;
187 | final DecimalFormat decimalFormat;
188 |
189 | protected DecimalVisitor(Schema schema, int scale) {
190 | super(schema);
191 | this.scale = scale;
192 | this.decimalFormat = new DecimalFormat("#");
193 | this.decimalFormat.setParseBigDecimal(true);
194 | this.decimalFormat.setMinimumFractionDigits(scale);
195 | }
196 |
197 | @Override
198 | protected Number doVisit(TextNode node) {
199 | try {
200 | return this.decimalFormat.parse(node.asText());
201 | } catch (ParseException e) {
202 | throw new DataException(e);
203 | }
204 | }
205 | }
206 |
207 | public static class ArrayVisitor extends FromJsonVisitor {
208 | final FromJsonVisitor itemVisitor;
209 |
210 | public ArrayVisitor(Schema schema, FromJsonVisitor itemVisitor) {
211 | super(schema);
212 | this.itemVisitor = itemVisitor;
213 | }
214 |
215 | @Override
216 | protected List doVisit(ArrayNode node) {
217 | List result = new ArrayList();
218 | for (JsonNode jsonNode : node) {
219 | Object value = itemVisitor.visit(jsonNode);
220 | result.add(value);
221 | }
222 | return result;
223 | }
224 | }
225 |
226 | public static class BytesVisitor extends FromJsonVisitor {
227 | public BytesVisitor(Schema schema) {
228 | super(schema);
229 | }
230 |
231 | @Override
232 | protected byte[] doVisit(TextNode node) {
233 | return BaseEncoding.base64().decode(node.textValue());
234 | }
235 | }
236 |
237 | public static class CustomDateVisitor extends FromJsonVisitor {
238 | private static final Logger log = LoggerFactory.getLogger(DateTimeVisitor.class);
239 |
240 | final DateTimeFormatter dateTimeFormatter;
241 |
242 | public CustomDateVisitor(Schema schema) {
243 | super(schema);
244 | String pattern = schema.parameters().get("dateFormat");
245 | this.dateTimeFormatter = DateTimeFormatter.ofPattern(pattern);
246 | }
247 |
248 | @Override
249 | protected Date doVisit(TextNode node) {
250 | log.trace(node.asText());
251 | LocalDateTime localDateTime = LocalDateTime.parse(node.asText(), this.dateTimeFormatter);
252 | Instant instant = localDateTime.toInstant(ZoneOffset.UTC);
253 | return Date.from(instant);
254 | }
255 | }
256 | }
257 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/JacksonFactory.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import com.fasterxml.jackson.core.JsonGenerator;
19 | import com.fasterxml.jackson.databind.JsonSerializer;
20 | import com.fasterxml.jackson.databind.ObjectMapper;
21 | import com.fasterxml.jackson.databind.SerializerProvider;
22 | import com.fasterxml.jackson.databind.module.SimpleModule;
23 |
24 | import java.io.IOException;
25 | import java.text.DecimalFormat;
26 |
27 | public class JacksonFactory {
28 |
29 | public static ObjectMapper create() {
30 | ObjectMapper objectMapper = new ObjectMapper();
31 | return objectMapper;
32 | }
33 |
34 | static class SerializationModule extends SimpleModule {
35 | public SerializationModule() {
36 | addSerializer(double.class, new DoubleSerializer());
37 | }
38 | }
39 |
40 | static class DoubleSerializer extends JsonSerializer {
41 | final DecimalFormat decimalFormat;
42 |
43 | public DoubleSerializer() {
44 | this.decimalFormat = new DecimalFormat("#");
45 | // this.df.setMaximumFractionDigits(8);
46 | }
47 |
48 |
49 | @Override
50 | public void serialize(Double aDouble, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException {
51 | jsonGenerator.writeRaw(this.decimalFormat.format(aDouble));
52 | }
53 | }
54 |
55 |
56 | }
57 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/JsonConfig.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import com.github.jcustenborder.kafka.connect.utils.config.ConfigKeyBuilder;
19 | import com.github.jcustenborder.kafka.connect.utils.config.ConfigUtils;
20 | import com.github.jcustenborder.kafka.connect.utils.config.Description;
21 | import com.github.jcustenborder.kafka.connect.utils.config.recommenders.Recommenders;
22 | import com.github.jcustenborder.kafka.connect.utils.config.validators.Validators;
23 | import org.apache.kafka.common.config.AbstractConfig;
24 | import org.apache.kafka.common.config.ConfigDef;
25 |
26 | import java.net.URL;
27 | import java.util.Collections;
28 | import java.util.Map;
29 | import java.util.Set;
30 |
31 | class JsonConfig extends AbstractConfig {
32 | public static final String SCHEMA_URL_CONF = "json.schema.url";
33 | public static final String SCHEMA_INLINE_CONF = "json.schema.inline";
34 | public static final String SCHEMA_LOCATION_CONF = "json.schema.location";
35 | public static final String VALIDATE_JSON_ENABLED_CONF = "json.schema.validation.enabled";
36 | public static final String EXCLUDE_LOCATIONS_CONF = "json.exclude.locations";
37 | static final String SCHEMA_URL_DOC = "Url to retrieve the schema from. Urls can be anything that is " +
38 | "supported by URL.openStream(). For example the local filesystem file:///schemas/something.json. " +
39 | "A web address https://www.schemas.com/something.json";
40 | static final String SCHEMA_INLINE_DOC = "The JSON schema to use as an escaped string.";
41 | static final String SCHEMA_LOCATION_DOC = "Location to retrieve the schema from. " +
42 | ConfigUtils.enumDescription(SchemaLocation.class);
43 | static final String VALIDATE_JSON_ENABLED_DOC = "Flag to determine if the JSON should be validated " +
44 | "against the schema.";
45 | static final String EXCLUDE_LOCATIONS_DOC = "Location(s) in the schema to exclude. This is primarily " +
46 | "because connect cannot support those locations. For example types that would require a union type.";
47 |
48 | public JsonConfig(ConfigDef definition, Map, ?> originals) {
49 | super(definition, originals);
50 | this.schemaUrl = ConfigUtils.url(this, SCHEMA_URL_CONF);
51 | this.schemaLocation = ConfigUtils.getEnum(SchemaLocation.class, this, SCHEMA_LOCATION_CONF);
52 | this.schemaText = getString(SCHEMA_INLINE_CONF);
53 | this.validateJson = getBoolean(VALIDATE_JSON_ENABLED_CONF);
54 | this.excludeLocations = ConfigUtils.getSet(this, EXCLUDE_LOCATIONS_CONF);
55 |
56 | }
57 |
58 |
59 | public enum SchemaLocation {
60 | @Description("Loads the schema from the url specified in `" + SCHEMA_URL_CONF + "`.")
61 | Url,
62 | @Description("Loads the schema from `" + SCHEMA_INLINE_CONF + "` as an inline string.")
63 | Inline
64 | }
65 |
66 | public final URL schemaUrl;
67 | public final SchemaLocation schemaLocation;
68 | public final String schemaText;
69 | public final boolean validateJson;
70 | public final Set excludeLocations;
71 |
72 | public static ConfigDef config() {
73 | return new ConfigDef().define(
74 | ConfigKeyBuilder.of(SCHEMA_URL_CONF, ConfigDef.Type.STRING)
75 | .documentation(SCHEMA_URL_DOC)
76 | .validator(Validators.validUrl())
77 | .importance(ConfigDef.Importance.HIGH)
78 | .recommender(Recommenders.visibleIf(SCHEMA_LOCATION_CONF, SchemaLocation.Url.toString()))
79 | .defaultValue("File:///doesNotExist")
80 | .build()
81 | ).define(
82 | ConfigKeyBuilder.of(SCHEMA_LOCATION_CONF, ConfigDef.Type.STRING)
83 | .documentation(SCHEMA_LOCATION_DOC)
84 | .validator(Validators.validEnum(SchemaLocation.class))
85 | .recommender(Recommenders.enumValues(SchemaLocation.class))
86 | .importance(ConfigDef.Importance.HIGH)
87 | .defaultValue(SchemaLocation.Url.toString())
88 | .build()
89 | ).define(
90 | ConfigKeyBuilder.of(VALIDATE_JSON_ENABLED_CONF, ConfigDef.Type.BOOLEAN)
91 | .documentation(VALIDATE_JSON_ENABLED_DOC)
92 | .importance(ConfigDef.Importance.MEDIUM)
93 | .defaultValue(false)
94 | .build()
95 | ).define(
96 | ConfigKeyBuilder.of(SCHEMA_INLINE_CONF, ConfigDef.Type.STRING)
97 | .documentation(SCHEMA_INLINE_DOC)
98 | .recommender(Recommenders.visibleIf(SCHEMA_LOCATION_CONF, SchemaLocation.Inline.toString()))
99 | .importance(ConfigDef.Importance.HIGH)
100 | .defaultValue("")
101 | .build()
102 | ).define(
103 | ConfigKeyBuilder.of(EXCLUDE_LOCATIONS_CONF, ConfigDef.Type.LIST)
104 | .documentation(EXCLUDE_LOCATIONS_DOC)
105 | .defaultValue(Collections.EMPTY_LIST)
106 | .importance(ConfigDef.Importance.LOW)
107 | .build()
108 | );
109 | }
110 | }
111 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/JsonSchemaConverter.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import com.fasterxml.jackson.core.JsonGenerator;
19 | import com.fasterxml.jackson.databind.JsonNode;
20 | import com.fasterxml.jackson.databind.ObjectMapper;
21 | import com.google.common.base.Charsets;
22 | import com.google.common.hash.Hashing;
23 | import com.google.common.io.ByteStreams;
24 | import org.apache.kafka.common.config.ConfigException;
25 | import org.apache.kafka.common.errors.SerializationException;
26 | import org.apache.kafka.common.header.Header;
27 | import org.apache.kafka.common.header.Headers;
28 | import org.apache.kafka.common.header.internals.RecordHeader;
29 | import org.apache.kafka.connect.data.Schema;
30 | import org.apache.kafka.connect.data.SchemaAndValue;
31 | import org.apache.kafka.connect.errors.DataException;
32 | import org.apache.kafka.connect.storage.Converter;
33 |
34 | import java.io.ByteArrayOutputStream;
35 | import java.io.IOException;
36 | import java.io.InputStream;
37 | import java.nio.charset.Charset;
38 | import java.util.HashMap;
39 | import java.util.Map;
40 |
41 | public class JsonSchemaConverter implements Converter {
42 | private static final String KEY_HEADER = "json.key.schema";
43 | private static final String VALUE_HEADER = "json.value.schema";
44 | JsonSchemaConverterConfig config;
45 | String jsonSchemaHeader;
46 | Charset encodingCharset;
47 | ObjectMapper objectMapper;
48 | FromJsonSchemaConverterFactory fromJsonSchemaConverterFactory;
49 | Map fromConnectStateLookup = new HashMap<>();
50 | Map toConnectStateLookup = new HashMap<>();
51 | Header fallbackHeader;
52 |
53 | @Override
54 | public void configure(Map settings, boolean isKey) {
55 | this.config = new JsonSchemaConverterConfig(settings);
56 | this.jsonSchemaHeader = isKey ? KEY_HEADER : VALUE_HEADER;
57 | this.encodingCharset = Charsets.UTF_8;
58 | this.objectMapper = JacksonFactory.create();
59 | this.fromJsonSchemaConverterFactory = new FromJsonSchemaConverterFactory(config);
60 |
61 | if (this.config.insertSchema) {
62 | byte[] headerValue;
63 | if (JsonConfig.SchemaLocation.Url == this.config.schemaLocation) {
64 | try {
65 | try (InputStream inputStream = this.config.schemaUrl.openStream()) {
66 | headerValue = ByteStreams.toByteArray(inputStream);
67 | }
68 | } catch (IOException e) {
69 | ConfigException exception = new ConfigException(JsonConfig.SCHEMA_URL_CONF, this.config.schemaUrl, "exception while loading schema");
70 | exception.initCause(e);
71 | throw exception;
72 | }
73 | } else if (JsonConfig.SchemaLocation.Inline == this.config.schemaLocation) {
74 | headerValue = this.jsonSchemaHeader.getBytes(Charsets.UTF_8);
75 | } else {
76 | throw new ConfigException(
77 | JsonConfig.SCHEMA_LOCATION_CONF,
78 | this.config.schemaLocation.toString(),
79 | "Location is not supported"
80 | );
81 | }
82 | this.fallbackHeader = new RecordHeader(this.jsonSchemaHeader, headerValue);
83 | } else {
84 | fallbackHeader = null;
85 | }
86 | }
87 |
88 | @Override
89 | public byte[] fromConnectData(String s, Schema schema, Object o) {
90 | throw new UnsupportedOperationException(
91 | "This converter requires Kafka 2.4.0 or higher with header support."
92 | );
93 | }
94 |
95 | @Override
96 | public byte[] fromConnectData(String topic, Headers headers, Schema schema, Object value) {
97 | if (null == value) {
98 | return null;
99 | }
100 |
101 |
102 | try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
103 | try (JsonGenerator jsonGenerator = objectMapper.getFactory().createGenerator(outputStream)) {
104 | FromConnectState fromConnectState = fromConnectStateLookup.computeIfAbsent(schema, s -> FromConnectSchemaConverter.toJsonSchema(schema, jsonSchemaHeader));
105 | headers.add(fromConnectState.header);
106 | fromConnectState.visitor.doVisit(jsonGenerator, value);
107 | }
108 | return outputStream.toByteArray();
109 | } catch (IOException ex) {
110 | throw new SerializationException(ex);
111 | }
112 | }
113 |
114 | @Override
115 | public SchemaAndValue toConnectData(String s, byte[] bytes) {
116 | throw new UnsupportedOperationException(
117 | "This converter requires Kafka 2.4.0 or higher with header support."
118 | );
119 | }
120 |
121 | Header schemaHeader(Headers headers) {
122 | Header schemaHeader = headers.lastHeader(this.jsonSchemaHeader);
123 | if (null == schemaHeader) {
124 | schemaHeader = this.fallbackHeader;
125 | }
126 | return schemaHeader;
127 | }
128 |
129 | @Override
130 | public SchemaAndValue toConnectData(String topic, Headers headers, byte[] value) {
131 | if (null == value) {
132 | return SchemaAndValue.NULL;
133 | }
134 |
135 | final Header schemaHeader = schemaHeader(headers);
136 |
137 | if (null == schemaHeader) {
138 | throw new DataException(
139 | String.format(
140 | "Record does not have '{}' header and '%s' is not enabled.",
141 | this.jsonSchemaHeader,
142 | JsonSchemaConverterConfig.INSERT_SCHEMA_ENABLED_CONF
143 | )
144 | );
145 | }
146 |
147 | String hash = Hashing.goodFastHash(32)
148 | .hashBytes(schemaHeader.value())
149 | .toString();
150 | FromJsonState state = this.toConnectStateLookup.computeIfAbsent(hash, h -> {
151 | org.everit.json.schema.Schema schema = Utils.loadSchema(schemaHeader);
152 | return this.fromJsonSchemaConverterFactory.fromJSON(schema);
153 | });
154 |
155 | JsonNode jsonNode;
156 | try {
157 | jsonNode = this.objectMapper.readValue(value, JsonNode.class);
158 | } catch (IOException ex) {
159 | throw new SerializationException(ex);
160 | }
161 | Object result = state.visitor.visit(jsonNode);
162 | return new SchemaAndValue(state.schema, result);
163 | }
164 | }
165 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/JsonSchemaConverterConfig.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import com.github.jcustenborder.kafka.connect.utils.config.ConfigKeyBuilder;
19 | import org.apache.kafka.common.config.ConfigDef;
20 |
21 | import java.util.Map;
22 |
23 | class JsonSchemaConverterConfig extends JsonConfig {
24 | public final boolean insertSchema;
25 |
26 | public final static String INSERT_SCHEMA_ENABLED_CONF = "json.insert.schema.enabled";
27 | final static String INSERT_SCHEMA_ENABLED_DOC = "Flag to determine if the schema specified should be " +
28 | "used if there is no schema header found. This allows a connector to consume a topic " +
29 | "that does not have schema headers and apply an external header.";
30 |
31 | public JsonSchemaConverterConfig(Map, ?> originals) {
32 | super(config(), originals);
33 | this.insertSchema = getBoolean(INSERT_SCHEMA_ENABLED_CONF);
34 | }
35 |
36 | public static ConfigDef config() {
37 | return JsonConfig.config()
38 | .define(
39 | ConfigKeyBuilder.of(INSERT_SCHEMA_ENABLED_CONF, ConfigDef.Type.BOOLEAN)
40 | .documentation(INSERT_SCHEMA_ENABLED_DOC)
41 | .importance(ConfigDef.Importance.HIGH)
42 | .defaultValue(false)
43 | .build()
44 | );
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/Utils.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import org.apache.kafka.common.header.Header;
19 | import org.apache.kafka.connect.data.Decimal;
20 | import org.apache.kafka.connect.errors.DataException;
21 | import org.everit.json.schema.Schema;
22 | import org.everit.json.schema.StringSchema;
23 | import org.everit.json.schema.internal.DateFormatValidator;
24 | import org.everit.json.schema.internal.DateTimeFormatValidator;
25 | import org.everit.json.schema.internal.TimeFormatValidator;
26 | import org.everit.json.schema.loader.SchemaLoader;
27 | import org.json.JSONObject;
28 | import org.json.JSONTokener;
29 |
30 | import java.io.ByteArrayInputStream;
31 | import java.io.IOException;
32 | import java.io.InputStream;
33 | import java.io.Reader;
34 | import java.io.StringReader;
35 | import java.time.ZoneId;
36 | import java.time.format.DateTimeFormatter;
37 |
38 | public class Utils {
39 |
40 | static final ZoneId ZONE_ID = ZoneId.of("UTC");
41 | public static final DateTimeFormatter TIMESTAMP_FORMATTER = DateTimeFormatter.ISO_INSTANT
42 | .withZone(ZONE_ID);
43 | public static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ISO_LOCAL_DATE
44 | .withZone(ZONE_ID);
45 | public static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ISO_LOCAL_TIME
46 | .withZone(ZONE_ID);
47 |
48 |
49 | public static JSONObject loadObject(Header header) {
50 | try (InputStream inputStream = new ByteArrayInputStream(header.value())) {
51 | return loadObject(inputStream);
52 | } catch (IOException ex) {
53 | throw new DataException("Could not load schema", ex);
54 | }
55 | }
56 |
57 | public static JSONObject loadObject(InputStream inputStream) {
58 | return new JSONObject(new JSONTokener(inputStream));
59 | }
60 |
61 | public static Schema loadSchema(InputStream inputStream) {
62 | JSONObject rawSchema = loadObject(inputStream);
63 | return loadSchema(rawSchema);
64 | }
65 |
66 | public static org.everit.json.schema.Schema loadSchema(JSONObject rawSchema) {
67 | return SchemaLoader.builder()
68 | .draftV7Support()
69 | .addFormatValidator(new DateFormatValidator())
70 | .addFormatValidator(new TimeFormatValidator())
71 | .addFormatValidator(new DateTimeFormatValidator())
72 | .addFormatValidator(new DecimalFormatValidator())
73 | .addFormatValidator(new CustomTimestampFormatValidator())
74 | .schemaJson(rawSchema)
75 | .build()
76 | .load()
77 | .build();
78 | }
79 |
80 | public static org.everit.json.schema.Schema loadSchema(Header header) {
81 | JSONObject rawSchema = loadObject(header);
82 | return loadSchema(rawSchema);
83 | }
84 |
85 |
86 | public static int scale(StringSchema schema) {
87 | String scale = schema.getUnprocessedProperties().get("scale").toString();
88 | return scale(scale);
89 | }
90 |
91 | private static int scale(String scale) {
92 | return Integer.parseInt(scale);
93 | }
94 |
95 | public static int scale(org.apache.kafka.connect.data.Schema connectSchema) {
96 | String scale = connectSchema.parameters().get(Decimal.SCALE_FIELD);
97 | return scale(scale);
98 | }
99 |
100 | public static JSONObject loadObject(Reader reader) {
101 | return new JSONObject(new JSONTokener(reader));
102 | }
103 |
104 | public static Schema loadSchema(String schemaText) {
105 | try (Reader reader = new StringReader(schemaText)) {
106 | JSONObject rawSchema = loadObject(reader);
107 | return loadSchema(rawSchema);
108 | } catch (IOException ex) {
109 | throw new DataException("Could not load schema", ex);
110 | }
111 | }
112 |
113 |
114 | }
115 |
--------------------------------------------------------------------------------
/src/main/java/com/github/jcustenborder/kafka/connect/json/package-info.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | @Introduction("This plugin is used to add additional JSON parsing functionality to Kafka Connect.")
17 | @Title("Json Schema")
18 | @PluginOwner("jcustenborder")
19 | @PluginName("kafka-connect-json-schema")
20 | package com.github.jcustenborder.kafka.connect.json;
21 |
22 | import com.github.jcustenborder.kafka.connect.utils.config.Introduction;
23 | import com.github.jcustenborder.kafka.connect.utils.config.PluginName;
24 | import com.github.jcustenborder.kafka.connect.utils.config.PluginOwner;
25 | import com.github.jcustenborder.kafka.connect.utils.config.Title;
--------------------------------------------------------------------------------
/src/test/java/com/github/jcustenborder/kafka/connect/json/DocumentationTest.java:
--------------------------------------------------------------------------------
1 | package com.github.jcustenborder.kafka.connect.json;
2 |
3 | import com.github.jcustenborder.kafka.connect.utils.BaseDocumentationTest;
4 |
5 | public class DocumentationTest extends BaseDocumentationTest {
6 |
7 | }
8 |
--------------------------------------------------------------------------------
/src/test/java/com/github/jcustenborder/kafka/connect/json/FromConnectSchemaConverterTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import com.google.common.base.Strings;
19 | import org.apache.kafka.connect.data.Schema;
20 | import org.apache.kafka.connect.data.SchemaBuilder;
21 | import org.json.JSONObject;
22 | import org.junit.jupiter.api.DynamicTest;
23 | import org.junit.jupiter.api.Test;
24 | import org.junit.jupiter.api.TestFactory;
25 | import org.slf4j.Logger;
26 | import org.slf4j.LoggerFactory;
27 |
28 | import java.util.stream.Stream;
29 |
30 | import static org.junit.jupiter.api.Assertions.assertEquals;
31 | import static org.junit.jupiter.api.Assertions.assertNotNull;
32 | import static org.junit.jupiter.api.DynamicTest.dynamicTest;
33 |
34 | public class FromConnectSchemaConverterTest {
35 | private static final Logger log = LoggerFactory.getLogger(FromConnectSchemaConverterTest.class);
36 |
37 | @Test
38 | public void array() {
39 | Schema addressSchema = SchemaBuilder.struct()
40 | .name("Address")
41 | .optional()
42 | .field("city", SchemaBuilder.string().build())
43 | .field("state", SchemaBuilder.string().build())
44 | .field("street_address", SchemaBuilder.string().build())
45 | .build();
46 | Schema arraySchema = SchemaBuilder.array(addressSchema);
47 | Schema expected = SchemaBuilder.struct()
48 | .name("Person")
49 | .field("previous_addresses", arraySchema)
50 | .build();
51 | FromConnectState state = FromConnectSchemaConverter.toJsonSchema(expected, "foo");
52 | assertNotNull(state, "rawJsonSchema should not be null.");
53 | JSONObject rawJsonSchema = Utils.loadObject(state.header);
54 | log.trace("rawJsonSchema = {}", rawJsonSchema.toString(2));
55 | org.everit.json.schema.Schema jsonSchema = TestUtils.jsonSchema(rawJsonSchema);
56 | assertNotNull(jsonSchema);
57 | }
58 |
59 | @Test
60 | public void struct() {
61 | Schema addressSchema = SchemaBuilder.struct()
62 | .name("Address")
63 | .doc("An object to store an address.")
64 | .optional()
65 | .field("city", SchemaBuilder.string().doc("city of the address.").build())
66 | .field("state", SchemaBuilder.string().doc("state of the address.").build())
67 | .field("street_address", SchemaBuilder.string().doc("street address of the address.").build())
68 | .build();
69 | Schema expected = SchemaBuilder.struct()
70 | .name("Customer")
71 | .field("first_name", SchemaBuilder.string().doc("First name of the customer").build())
72 | .field("billing_address", addressSchema)
73 | .field("shipping_address", addressSchema)
74 | .build();
75 |
76 | FromConnectState state = FromConnectSchemaConverter.toJsonSchema(expected, "foo");
77 | assertNotNull(state, "rawJsonSchema should not be null.");
78 | JSONObject rawJsonSchema = Utils.loadObject(state.header);
79 | log.trace("rawJsonSchema = {}", rawJsonSchema.toString(2));
80 | org.everit.json.schema.Schema jsonSchema = TestUtils.jsonSchema(rawJsonSchema);
81 | assertNotNull(jsonSchema);
82 | }
83 |
84 | @TestFactory
85 | public Stream primitives() {
86 | return FromConnectSchemaConverter.PRIMITIVE_TYPES.entrySet()
87 | .stream()
88 | .filter(e-> Strings.isNullOrEmpty(e.getKey().schemaName))
89 | .map(e -> dynamicTest(e.getKey().toString(), () -> {
90 | String description = String.format("This schema represents a %s", e.getKey());
91 | Schema expected = SchemaBuilder.type(e.getKey().type)
92 | .doc(description)
93 | .build();
94 | FromConnectState state = FromConnectSchemaConverter.toJsonSchema(expected, "foo");
95 | assertNotNull(state, "rawJsonSchema should not be null.");
96 | JSONObject rawJsonSchema = Utils.loadObject(state.header);
97 | assertNotNull(rawJsonSchema, "rawJsonSchema should not be null.");
98 | log.trace("rawJsonSchema = {}", rawJsonSchema.toString(2));
99 | e.getValue().forEach((propertyName, expectedValue) -> {
100 | Object propertyValue = rawJsonSchema.get(propertyName);
101 | assertEquals(expectedValue, propertyValue);
102 | });
103 | assertEquals(description, rawJsonSchema.getString("description"));
104 | org.everit.json.schema.Schema jsonSchema = TestUtils.jsonSchema(rawJsonSchema);
105 |
106 | }));
107 | }
108 |
109 |
110 | }
111 |
--------------------------------------------------------------------------------
/src/test/java/com/github/jcustenborder/kafka/connect/json/FromJsonSchemaConverterTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import com.google.common.collect.ImmutableMap;
19 | import org.apache.kafka.connect.data.Date;
20 | import org.apache.kafka.connect.data.Schema;
21 | import org.apache.kafka.connect.data.SchemaBuilder;
22 | import org.apache.kafka.connect.data.Time;
23 | import org.apache.kafka.connect.data.Timestamp;
24 | import org.everit.json.schema.internal.DateFormatValidator;
25 | import org.everit.json.schema.internal.DateTimeFormatValidator;
26 | import org.everit.json.schema.internal.TimeFormatValidator;
27 | import org.everit.json.schema.loader.SchemaLoader;
28 | import org.json.JSONObject;
29 | import org.json.JSONTokener;
30 | import org.junit.jupiter.api.BeforeEach;
31 | import org.junit.jupiter.api.DynamicTest;
32 | import org.junit.jupiter.api.Test;
33 | import org.junit.jupiter.api.TestFactory;
34 | import org.slf4j.Logger;
35 | import org.slf4j.LoggerFactory;
36 |
37 | import java.io.IOException;
38 | import java.io.InputStream;
39 | import java.util.LinkedHashMap;
40 | import java.util.Map;
41 | import java.util.stream.Stream;
42 |
43 | import static com.github.jcustenborder.kafka.connect.utils.AssertSchema.assertSchema;
44 | import static org.junit.jupiter.api.DynamicTest.dynamicTest;
45 |
46 | public class FromJsonSchemaConverterTest {
47 | private static final Logger log = LoggerFactory.getLogger(FromJsonSchemaConverterTest.class);
48 |
49 | JsonConfig config;
50 | FromJsonSchemaConverterFactory factory;
51 |
52 | @BeforeEach
53 | public void before() {
54 | config = new FromJsonConfig(ImmutableMap.of(
55 | FromJsonConfig.SCHEMA_INLINE_CONF, "\"string\"",
56 | FromJsonConfig.SCHEMA_LOCATION_CONF, JsonConfig.SchemaLocation.Inline.toString(),
57 | JsonConfig.EXCLUDE_LOCATIONS_CONF, "#/properties/log_params"
58 | ));
59 | this.factory = new FromJsonSchemaConverterFactory(config);
60 | }
61 |
62 |
63 | org.everit.json.schema.Schema jsonSchema(String type) {
64 | JSONObject rawSchema = new JSONObject();
65 | rawSchema.put("type", type);
66 | return TestUtils.jsonSchema(rawSchema);
67 | }
68 |
69 | org.everit.json.schema.Schema jsonSchema(String type, String key1, String value1) {
70 | JSONObject rawSchema = new JSONObject();
71 | rawSchema.put("type", type);
72 | rawSchema.put(key1, value1);
73 | return TestUtils.jsonSchema(rawSchema);
74 | }
75 |
76 | org.everit.json.schema.Schema jsonSchema(String type, String key1, String value1, String key2, String value2) {
77 | JSONObject rawSchema = new JSONObject();
78 | rawSchema.put("type", type);
79 | rawSchema.put(key1, value1);
80 | rawSchema.put(key2, value2);
81 | return TestUtils.jsonSchema(rawSchema);
82 | }
83 |
84 |
85 | void assertJsonSchema(org.apache.kafka.connect.data.Schema expected, org.everit.json.schema.Schema input) {
86 | FromJsonState state = this.factory.fromJSON(input);
87 |
88 |
89 | log.trace("schema:\n{}", state.schema);
90 | assertSchema(expected, state.schema);
91 | }
92 |
93 | @Test
94 | public void booleanSchema() {
95 | org.everit.json.schema.Schema jsonSchema = jsonSchema("boolean");
96 | assertJsonSchema(Schema.BOOLEAN_SCHEMA, jsonSchema);
97 | }
98 |
99 | @Test
100 | public void stringSchema() {
101 | org.everit.json.schema.Schema jsonSchema = jsonSchema("string");
102 | assertJsonSchema(Schema.STRING_SCHEMA, jsonSchema);
103 | }
104 |
105 | @Test
106 | public void integerSchema() {
107 | org.everit.json.schema.Schema jsonSchema = jsonSchema("integer");
108 | assertJsonSchema(Schema.INT64_SCHEMA, jsonSchema);
109 | }
110 |
111 | @Test
112 | public void numberSchema() {
113 | org.everit.json.schema.Schema jsonSchema = jsonSchema("number");
114 | assertJsonSchema(Schema.FLOAT64_SCHEMA, jsonSchema);
115 | }
116 |
117 | @Test
118 | public void dateSchema() {
119 | org.everit.json.schema.Schema jsonSchema = jsonSchema("string", "format", "date");
120 | assertJsonSchema(Date.SCHEMA, jsonSchema);
121 | }
122 |
123 | @Test
124 | public void timeSchema() {
125 | org.everit.json.schema.Schema jsonSchema = jsonSchema("string", "format", "time");
126 | assertJsonSchema(Time.SCHEMA, jsonSchema);
127 | }
128 |
129 | @Test
130 | public void datetimeSchema() {
131 | org.everit.json.schema.Schema jsonSchema = jsonSchema("string", "format", "date-time");
132 | assertJsonSchema(Timestamp.SCHEMA, jsonSchema);
133 | }
134 |
135 | org.everit.json.schema.Schema loadSchema(String name) throws IOException {
136 | try (InputStream inputStream = this.getClass().getResourceAsStream(name)) {
137 | JSONObject rawSchema = new JSONObject(new JSONTokener(inputStream));
138 | return SchemaLoader.builder()
139 | .draftV7Support()
140 | .addFormatValidator(new DateFormatValidator())
141 | .addFormatValidator(new TimeFormatValidator())
142 | .addFormatValidator(new DateTimeFormatValidator())
143 | .schemaJson(rawSchema)
144 | .build()
145 | .load()
146 | .build();
147 | }
148 | }
149 |
150 | @Test
151 | public void productSchema() throws IOException {
152 | org.everit.json.schema.Schema jsonSchema = loadSchema("SchemaConverterTest/product.schema.json");
153 | Schema expected = SchemaBuilder.struct()
154 | .name("Product")
155 | .doc("A product from Acme's catalog")
156 | .field("price", SchemaBuilder.float64().doc("The price of the product").build())
157 | .field("productId", SchemaBuilder.int64().doc("The unique identifier for a product").build())
158 | .field("productName", SchemaBuilder.string().doc("Name of the product").build())
159 | .build();
160 | assertJsonSchema(expected, jsonSchema);
161 | }
162 |
163 | @Test
164 | public void wikiMediaRecentChangeSchema() throws IOException {
165 | org.everit.json.schema.Schema jsonSchema = loadSchema("SchemaConverterTest/wikimedia.recentchange.schema.json");
166 | Schema propertiesLength = SchemaBuilder.struct()
167 | .name("properties.length")
168 | .optional()
169 | .doc("Length of old and new change")
170 | .field("new", SchemaBuilder.int64().doc("(rc_new_len)").optional().build())
171 | .field("old", SchemaBuilder.int64().doc("(rc_old_len)").optional().build())
172 | .build();
173 | Schema propertiesMeta = SchemaBuilder.struct()
174 | .name("properties.meta")
175 | .field("domain", SchemaBuilder.string().optional().doc("Domain the event or entity pertains to").build())
176 | .field("dt", Timestamp.builder().doc("Event datetime, in ISO-8601 format").build())
177 | .field("id", SchemaBuilder.string().doc("Unique ID of this event").build())
178 | .field("request_id", SchemaBuilder.string().optional().doc("Unique ID of the request that caused the event").build())
179 | .field("stream", SchemaBuilder.string().doc("Name of the stream/queue/dataset that this event belongs in").build())
180 | .field("uri", SchemaBuilder.string().optional().doc("Unique URI identifying the event or entity").build())
181 | .build();
182 | Schema propertiesRevision = SchemaBuilder.struct()
183 | .name("properties.revision")
184 | .optional()
185 | .doc("Old and new revision IDs")
186 | .field("new", SchemaBuilder.int64().doc("(rc_last_oldid)").optional().build())
187 | .field("old", SchemaBuilder.int64().doc("(rc_this_oldid)").optional().build())
188 | .build();
189 |
190 |
191 | Schema expected = SchemaBuilder.struct()
192 | .name("mediawiki.recentchange")
193 | .doc("Represents a MW RecentChange event. https://www.mediawiki.org/wiki/Manual:RCFeed\n")
194 | .field("bot", SchemaBuilder.bool().optional().doc("(rc_bot)").build())
195 | .field("comment", SchemaBuilder.string().optional().doc("(rc_comment)").build())
196 | .field("id", SchemaBuilder.int64().optional().doc("ID of the recentchange event (rcid).").build())
197 | .field("length", propertiesLength)
198 | .field("log_action", SchemaBuilder.string().optional().doc("(rc_log_action)").build())
199 | .field("log_action_comment", SchemaBuilder.string().optional().build())
200 | .field("log_id", SchemaBuilder.int64().optional().doc("(rc_log_id)").build())
201 | .field("log_type", SchemaBuilder.string().optional().doc("(rc_log_type)").build())
202 | .field("meta", propertiesMeta)
203 | .field("minor", SchemaBuilder.bool().optional().doc("(rc_minor).").build())
204 | .field("namespace", SchemaBuilder.int64().optional().doc("ID of relevant namespace of affected page (rc_namespace, page_namespace). This is -1 (\"Special\") for log events.\n").build())
205 | .field("parsedcomment", SchemaBuilder.string().optional().doc("The rc_comment parsed into simple HTML. Optional").build())
206 | .field("patrolled", SchemaBuilder.bool().optional().doc("(rc_patrolled). This property only exists if patrolling is supported for this event (based on $wgUseRCPatrol, $wgUseNPPatrol).\n").build())
207 | .field("revision", propertiesRevision)
208 | .field("server_name", SchemaBuilder.string().optional().doc("$wgServerName").build())
209 | .field("server_script_path", SchemaBuilder.string().optional().doc("$wgScriptPath").build())
210 | .field("server_url", SchemaBuilder.string().optional().doc("$wgCanonicalServer").build())
211 | .field("timestamp", SchemaBuilder.int64().optional().doc("Unix timestamp (derived from rc_timestamp).").build())
212 | .field("title", SchemaBuilder.string().optional().doc("Full page name, from Title::getPrefixedText.").build())
213 | .field("type", SchemaBuilder.string().optional().doc("Type of recentchange event (rc_type). One of \"edit\", \"new\", \"log\", \"categorize\", or \"external\". (See Manual:Recentchanges table#rc_type)\n").build())
214 | .field("user", SchemaBuilder.string().optional().doc("(rc_user_text)").build())
215 | .field("wiki", SchemaBuilder.string().optional().doc("wfWikiID ($wgDBprefix, $wgDBname)").build())
216 |
217 | .build();
218 |
219 |
220 | assertJsonSchema(expected, jsonSchema);
221 | }
222 |
223 | @Test
224 | public void nested() throws IOException {
225 |
226 | org.everit.json.schema.Schema jsonSchema = loadSchema("SchemaConverterTest/nested.schema.json");
227 | Schema addressSchema = SchemaBuilder.struct()
228 | .name("Address")
229 | .optional()
230 | .field("city", SchemaBuilder.string().build())
231 | .field("state", SchemaBuilder.string().build())
232 | .field("street_address", SchemaBuilder.string().build())
233 | .build();
234 | Schema expected = SchemaBuilder.struct()
235 | .name("Customer")
236 | .field("billing_address", addressSchema)
237 | .field("shipping_address", addressSchema)
238 | .build();
239 | assertJsonSchema(expected, jsonSchema);
240 | }
241 |
242 | @Test
243 | public void array() {
244 | JSONObject rawSchema = new JSONObject()
245 | .put("type", "array")
246 | .put("items", new JSONObject().put("type", "number"));
247 | org.everit.json.schema.Schema jsonSchema = TestUtils.jsonSchema(rawSchema);
248 | assertJsonSchema(SchemaBuilder.array(Schema.FLOAT64_SCHEMA).build(), jsonSchema);
249 | }
250 |
251 | @TestFactory
252 | public Stream stringFormats() {
253 | Map formats = new LinkedHashMap<>();
254 | formats.put("email", "test@example.com");
255 | formats.put("idn-email", "test@example.com");
256 | formats.put("hostname", "example.com");
257 | formats.put("idn-hostname", "example.com");
258 | formats.put("ipv4", "127.0.0.1");
259 | formats.put("ipv6", "::1");
260 | formats.put("uri", "http://example.com");
261 | formats.put("uri-reference", "http://example.com");
262 | formats.put("iri", "http://example.com");
263 | formats.put("iri-reference", "http://example.com");
264 | formats.put("uri-template", "http://example.com/~{username}/");
265 |
266 | return formats.entrySet()
267 | .stream()
268 | .map(e -> dynamicTest(e.getKey(), () -> {
269 | JSONObject rawSchema = new JSONObject()
270 | .put("type", "string")
271 | .put("format", e.getKey());
272 | log.trace("schema = '{}'", rawSchema);
273 | org.everit.json.schema.Schema jsonSchema = TestUtils.jsonSchema(rawSchema);
274 | assertJsonSchema(Schema.STRING_SCHEMA, jsonSchema);
275 | }));
276 | }
277 |
278 | }
279 |
--------------------------------------------------------------------------------
/src/test/java/com/github/jcustenborder/kafka/connect/json/FromJsonTest.java:
--------------------------------------------------------------------------------
1 | package com.github.jcustenborder.kafka.connect.json;
2 |
3 | import com.github.jcustenborder.kafka.connect.utils.SinkRecordHelper;
4 | import com.google.common.collect.ImmutableMap;
5 | import com.google.common.io.ByteStreams;
6 | import org.apache.kafka.connect.data.Schema;
7 | import org.apache.kafka.connect.data.SchemaAndValue;
8 | import org.apache.kafka.connect.data.Struct;
9 | import org.apache.kafka.connect.errors.DataException;
10 | import org.apache.kafka.connect.sink.SinkRecord;
11 | import org.junit.jupiter.api.BeforeEach;
12 | import org.junit.jupiter.api.Test;
13 | import org.slf4j.Logger;
14 | import org.slf4j.LoggerFactory;
15 |
16 | import java.io.File;
17 | import java.io.IOException;
18 | import java.time.LocalDateTime;
19 | import java.time.ZoneId;
20 | import java.time.ZonedDateTime;
21 | import java.time.format.DateTimeFormatter;
22 | import java.util.Map;
23 |
24 | import static org.junit.jupiter.api.Assertions.assertNotNull;
25 | import static org.junit.jupiter.api.Assertions.assertThrows;
26 | import static org.junit.jupiter.api.Assertions.assertTrue;
27 |
28 | public class FromJsonTest {
29 | private static final Logger log = LoggerFactory.getLogger(FromJsonTest.class);
30 | FromJson transform;
31 |
32 | @BeforeEach
33 | public void beforeEach() {
34 | this.transform = new FromJson.Value<>();
35 | }
36 |
37 | @Test
38 | public void basic() throws IOException {
39 | byte[] input = ByteStreams.toByteArray(this.getClass().getResourceAsStream(
40 | "basic.data.json"
41 | ));
42 | File schemaFile = new File("src/test/resources/com/github/jcustenborder/kafka/connect/json/basic.schema.json");
43 | Map settings = ImmutableMap.of(
44 | JsonConfig.SCHEMA_URL_CONF, schemaFile.toURI().toString()
45 | );
46 | this.transform.configure(settings);
47 | SinkRecord inputRecord = SinkRecordHelper.write("foo", new SchemaAndValue(Schema.STRING_SCHEMA, "foo"), new SchemaAndValue(Schema.BYTES_SCHEMA, input));
48 | SinkRecord transformedRecord = this.transform.apply(inputRecord);
49 | assertNotNull(transformedRecord);
50 | assertNotNull(transformedRecord.value());
51 | assertTrue(transformedRecord.value() instanceof Struct);
52 | Struct actual = (Struct) transformedRecord.value();
53 | log.info("actual = '{}'", actual);
54 | }
55 |
56 | @Test
57 | public void customdate() throws IOException {
58 | byte[] input = ByteStreams.toByteArray(this.getClass().getResourceAsStream(
59 | "customdate.data.json"
60 | ));
61 | File schemaFile = new File("src/test/resources/com/github/jcustenborder/kafka/connect/json/customdate.schema.json");
62 | Map settings = ImmutableMap.of(
63 | JsonConfig.SCHEMA_URL_CONF, schemaFile.toURI().toString()
64 | );
65 | this.transform.configure(settings);
66 | SinkRecord inputRecord = SinkRecordHelper.write("foo", new SchemaAndValue(Schema.STRING_SCHEMA, "foo"), new SchemaAndValue(Schema.BYTES_SCHEMA, input));
67 | SinkRecord transformedRecord = this.transform.apply(inputRecord);
68 | assertNotNull(transformedRecord);
69 | assertNotNull(transformedRecord.value());
70 | assertTrue(transformedRecord.value() instanceof Struct);
71 | Struct actual = (Struct) transformedRecord.value();
72 | log.info("actual = '{}'", actual);
73 | }
74 |
75 | @Test
76 | public void validate() throws IOException {
77 | byte[] input = ByteStreams.toByteArray(this.getClass().getResourceAsStream(
78 | "basic.data.json"
79 | ));
80 | File schemaFile = new File("src/test/resources/com/github/jcustenborder/kafka/connect/json/geo.schema.json");
81 | Map settings = ImmutableMap.of(
82 | JsonConfig.SCHEMA_URL_CONF, schemaFile.toURI().toString(),
83 | JsonConfig.VALIDATE_JSON_ENABLED_CONF, "true"
84 | );
85 | this.transform.configure(settings);
86 | SinkRecord inputRecord = SinkRecordHelper.write("foo", new SchemaAndValue(Schema.STRING_SCHEMA, "foo"), new SchemaAndValue(Schema.BYTES_SCHEMA, input));
87 | DataException exception = assertThrows(DataException.class, () -> {
88 | SinkRecord transformedRecord = this.transform.apply(inputRecord);
89 | });
90 |
91 | assertTrue(exception.getMessage().contains("required key [latitude] not found"));
92 | assertTrue(exception.getMessage().contains("required key [longitude] not found"));
93 | }
94 | @Test
95 | public void wikiMediaRecentChange() throws IOException {
96 | byte[] input = ByteStreams.toByteArray(this.getClass().getResourceAsStream(
97 | "wikimedia.recentchange.data.json"
98 | ));
99 | File schemaFile = new File("src/test/resources/com/github/jcustenborder/kafka/connect/json/wikimedia.recentchange.schema.json");
100 | Map settings = ImmutableMap.of(
101 | JsonConfig.SCHEMA_URL_CONF, schemaFile.toURI().toString(),
102 | JsonConfig.VALIDATE_JSON_ENABLED_CONF, "true",
103 | JsonConfig.EXCLUDE_LOCATIONS_CONF, "#/properties/log_params"
104 | );
105 | this.transform.configure(settings);
106 | SinkRecord inputRecord = SinkRecordHelper.write("foo", new SchemaAndValue(Schema.STRING_SCHEMA, "foo"), new SchemaAndValue(Schema.BYTES_SCHEMA, input));
107 | assertNotNull(inputRecord);
108 | }
109 |
110 | @Test
111 | public void foo() {
112 | String timestamp = "2020-01-07 04:47:05.0000000";
113 | DateTimeFormatter dateFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSSS")
114 | .withZone(ZoneId.of("UTC"));
115 | log.info(dateFormat.format(LocalDateTime.now()));
116 | ZonedDateTime dateTime = ZonedDateTime.parse(timestamp, dateFormat);
117 | }
118 |
119 | }
120 |
--------------------------------------------------------------------------------
/src/test/java/com/github/jcustenborder/kafka/connect/json/JsonSchemaConverterTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import com.google.common.base.Charsets;
19 | import com.google.common.collect.ImmutableList;
20 | import com.google.common.collect.ImmutableMap;
21 | import org.apache.kafka.common.header.Header;
22 | import org.apache.kafka.common.header.Headers;
23 | import org.apache.kafka.common.header.internals.RecordHeaders;
24 | import org.apache.kafka.connect.data.Decimal;
25 | import org.apache.kafka.connect.data.Schema;
26 | import org.apache.kafka.connect.data.SchemaAndValue;
27 | import org.apache.kafka.connect.data.SchemaBuilder;
28 | import org.apache.kafka.connect.data.Struct;
29 | import org.junit.jupiter.api.BeforeEach;
30 | import org.junit.jupiter.api.DynamicTest;
31 | import org.junit.jupiter.api.Test;
32 | import org.junit.jupiter.api.TestFactory;
33 | import org.slf4j.Logger;
34 | import org.slf4j.LoggerFactory;
35 |
36 | import java.io.IOException;
37 | import java.math.BigDecimal;
38 | import java.time.LocalDate;
39 | import java.time.LocalTime;
40 | import java.time.ZoneOffset;
41 | import java.util.Date;
42 | import java.util.LinkedHashMap;
43 | import java.util.Map;
44 | import java.util.stream.IntStream;
45 | import java.util.stream.Stream;
46 |
47 | import static com.github.jcustenborder.kafka.connect.utils.AssertSchema.assertSchema;
48 | import static com.github.jcustenborder.kafka.connect.utils.AssertStruct.assertStruct;
49 | import static org.junit.jupiter.api.Assertions.assertArrayEquals;
50 | import static org.junit.jupiter.api.Assertions.assertEquals;
51 | import static org.junit.jupiter.api.Assertions.assertNotNull;
52 | import static org.junit.jupiter.api.Assertions.assertNull;
53 | import static org.junit.jupiter.api.Assertions.assertTrue;
54 | import static org.junit.jupiter.api.DynamicTest.dynamicTest;
55 |
56 | public class JsonSchemaConverterTest {
57 | private static final Logger log = LoggerFactory.getLogger(JsonSchemaConverterTest.class);
58 | JsonSchemaConverter converter;
59 |
60 | @BeforeEach
61 | public void beforeEach() {
62 | this.converter = new JsonSchemaConverter();
63 | }
64 |
65 | @Test
66 | public void nulls() {
67 | this.converter.configure(
68 | ImmutableMap.of(),
69 | false
70 | );
71 | SchemaAndValue expected = SchemaAndValue.NULL;
72 | Headers headers = new RecordHeaders();
73 | byte[] buffer = this.converter.fromConnectData("topic", headers, expected.schema(), expected.value());
74 | assertNull(buffer, "buffer should be null.");
75 | }
76 |
77 | @Test
78 | public void roundTripString() {
79 | this.converter.configure(
80 | ImmutableMap.of(),
81 | false
82 | );
83 | SchemaAndValue expected = new SchemaAndValue(
84 | Schema.STRING_SCHEMA,
85 | "This is a test"
86 | );
87 | Headers headers = new RecordHeaders();
88 | byte[] buffer = this.converter.fromConnectData("topic", headers, expected.schema(), expected.value());
89 | assertNotNull(buffer, "buffer should not be null.");
90 | assertTrue(buffer.length > 0, "buffer should be longer than zero.");
91 | Header schemaHeader = headers.lastHeader(this.converter.jsonSchemaHeader);
92 | assertNotNull(schemaHeader, "schemaHeader should not be null.");
93 | SchemaAndValue actual = this.converter.toConnectData("topic", headers, buffer);
94 | assertNotNull(actual, "actual should not be null.");
95 | assertSchema(expected.schema(), actual.schema());
96 | assertEquals(expected.value(), actual.value());
97 | }
98 |
99 | @TestFactory
100 | public Stream roundtrip() {
101 | Map tests = new LinkedHashMap<>();
102 | Schema arraySchema = SchemaBuilder.array(Schema.STRING_SCHEMA).build();
103 | tests.put(
104 | new SchemaAndValue(arraySchema, ImmutableList.of("one", "two", "three")),
105 | new SchemaAndValue(arraySchema, ImmutableList.of("one", "two", "three"))
106 | );
107 | tests.put(
108 | new SchemaAndValue(Schema.STRING_SCHEMA, "This is a test"),
109 | new SchemaAndValue(Schema.STRING_SCHEMA, "This is a test")
110 | );
111 | tests.put(
112 | new SchemaAndValue(Schema.BYTES_SCHEMA, "This is a test".getBytes(Charsets.UTF_8)),
113 | new SchemaAndValue(Schema.BYTES_SCHEMA, "This is a test".getBytes(Charsets.UTF_8))
114 | );
115 | tests.put(
116 | new SchemaAndValue(Schema.BOOLEAN_SCHEMA, true),
117 | new SchemaAndValue(Schema.BOOLEAN_SCHEMA, true)
118 | );
119 | tests.put(
120 | new SchemaAndValue(Schema.INT8_SCHEMA, Byte.MAX_VALUE),
121 | new SchemaAndValue(Schema.INT64_SCHEMA, (long) Byte.MAX_VALUE)
122 | );
123 | tests.put(
124 | new SchemaAndValue(Schema.INT16_SCHEMA, Short.MAX_VALUE),
125 | new SchemaAndValue(Schema.INT64_SCHEMA, (long) Short.MAX_VALUE)
126 | );
127 | tests.put(
128 | new SchemaAndValue(Schema.INT32_SCHEMA, Integer.MAX_VALUE),
129 | new SchemaAndValue(Schema.INT64_SCHEMA, (long) Integer.MAX_VALUE)
130 | );
131 | tests.put(
132 | new SchemaAndValue(Schema.INT64_SCHEMA, Long.MAX_VALUE),
133 | new SchemaAndValue(Schema.INT64_SCHEMA, Long.MAX_VALUE)
134 | );
135 | tests.put(
136 | new SchemaAndValue(Schema.FLOAT32_SCHEMA, Float.MAX_VALUE),
137 | new SchemaAndValue(Schema.FLOAT64_SCHEMA, (double) Float.MAX_VALUE)
138 | );
139 | tests.put(
140 | new SchemaAndValue(Schema.FLOAT64_SCHEMA, Double.MAX_VALUE),
141 | new SchemaAndValue(Schema.FLOAT64_SCHEMA, Double.MAX_VALUE)
142 | );
143 | Date date = Date.from(LocalDate.of(2020, 02, 02).atTime(LocalTime.MIDNIGHT).toInstant(ZoneOffset.UTC));
144 | tests.put(
145 | new SchemaAndValue(org.apache.kafka.connect.data.Date.SCHEMA, date),
146 | new SchemaAndValue(org.apache.kafka.connect.data.Date.SCHEMA, date)
147 | );
148 |
149 | Date time = date.from(LocalTime.MIDNIGHT.atDate(LocalDate.of(1970, 1, 1)).toInstant(ZoneOffset.UTC));
150 | tests.put(
151 | new SchemaAndValue(org.apache.kafka.connect.data.Time.SCHEMA, time),
152 | new SchemaAndValue(org.apache.kafka.connect.data.Time.SCHEMA, time)
153 | );
154 |
155 | Date timestamp = new Date(1583363608123L);
156 | tests.put(
157 | new SchemaAndValue(org.apache.kafka.connect.data.Timestamp.SCHEMA, timestamp),
158 | new SchemaAndValue(org.apache.kafka.connect.data.Timestamp.SCHEMA, timestamp)
159 | );
160 |
161 | IntStream.range(0, 30)
162 | .forEach(scale -> {
163 | BigDecimal input = BigDecimal.valueOf(Long.MAX_VALUE, scale);
164 | SchemaAndValue schemaAndValue = new SchemaAndValue(Decimal.schema(scale), input);
165 | tests.put(schemaAndValue, schemaAndValue);
166 | });
167 | return tests.entrySet().stream()
168 | .map(p -> dynamicTest(p.getKey().schema().toString(), () -> {
169 | assertRoundTrip(p.getKey(), p.getValue());
170 | }));
171 | }
172 |
173 |
174 | void assertRoundTrip(SchemaAndValue input, SchemaAndValue expected) {
175 | this.converter.configure(
176 | ImmutableMap.of(),
177 | false
178 | );
179 | Headers headers = new RecordHeaders();
180 | byte[] buffer = this.converter.fromConnectData("topic", headers, input.schema(), input.value());
181 | log.trace(new String(buffer, Charsets.UTF_8));
182 | assertNotNull(buffer, "buffer should not be null.");
183 | assertTrue(buffer.length > 0, "buffer should be longer than zero.");
184 | Header schemaHeader = headers.lastHeader(this.converter.jsonSchemaHeader);
185 | assertNotNull(schemaHeader, "schemaHeader should not be null.");
186 | SchemaAndValue actual = this.converter.toConnectData("topic", headers, buffer);
187 | assertNotNull(actual, "actual should not be null.");
188 | assertSchema(expected.schema(), actual.schema());
189 |
190 | if (Decimal.LOGICAL_NAME.equals(expected.schema().name())) {
191 | assertEquals(expected.value(), actual.value());
192 | } else {
193 | switch (expected.schema().type()) {
194 | case BYTES:
195 | assertArrayEquals((byte[]) expected.value(), (byte[]) actual.value());
196 | break;
197 | case STRUCT:
198 | assertStruct((Struct) expected.value(), (Struct) actual.value());
199 | break;
200 | default:
201 | assertEquals(expected.value(), actual.value());
202 | break;
203 | }
204 | }
205 | }
206 |
207 | @Test
208 | public void nested() throws IOException {
209 | this.converter.configure(
210 | ImmutableMap.of(),
211 | false
212 | );
213 | Schema addressSchema = SchemaBuilder.struct()
214 | .name("Address")
215 | .optional()
216 | .field("city", SchemaBuilder.string().build())
217 | .field("state", SchemaBuilder.string().build())
218 | .field("street_address", SchemaBuilder.string().build())
219 | .build();
220 | Schema customer = SchemaBuilder.struct()
221 | .name("Customer")
222 | .field("billing_address", addressSchema)
223 | .field("shipping_address", addressSchema)
224 | .build();
225 | Struct billingAddress = new Struct(addressSchema)
226 | .put("city", "Austin")
227 | .put("state", "TX")
228 | .put("street_address", "123 Main St");
229 | Struct shippingAddress = new Struct(addressSchema)
230 | .put("city", "Dallas")
231 | .put("state", "TX")
232 | .put("street_address", "321 Something St");
233 | Struct struct = new Struct(customer)
234 | .put("billing_address", billingAddress)
235 | .put("shipping_address", shippingAddress);
236 | SchemaAndValue expected = new SchemaAndValue(customer, struct);
237 | Headers headers = new RecordHeaders();
238 | byte[] buffer = this.converter.fromConnectData("topic", headers, expected.schema(), expected.value());
239 | log.trace(new String(buffer, Charsets.UTF_8));
240 | assertNotNull(buffer, "buffer should not be null.");
241 | assertTrue(buffer.length > 0, "buffer should be longer than zero.");
242 | Header schemaHeader = headers.lastHeader(this.converter.jsonSchemaHeader);
243 | assertNotNull(schemaHeader, "schemaHeader should not be null.");
244 | SchemaAndValue actual = this.converter.toConnectData("topic", headers, buffer);
245 | assertNotNull(actual, "actual should not be null.");
246 | assertSchema(expected.schema(), actual.schema());
247 | assertEquals(expected.value(), actual.value());
248 | }
249 |
250 | }
251 |
--------------------------------------------------------------------------------
/src/test/java/com/github/jcustenborder/kafka/connect/json/TestUtils.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright © 2020 Jeremy Custenborder (jcustenborder@gmail.com)
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.github.jcustenborder.kafka.connect.json;
17 |
18 | import org.everit.json.schema.internal.DateFormatValidator;
19 | import org.everit.json.schema.internal.DateTimeFormatValidator;
20 | import org.everit.json.schema.internal.TimeFormatValidator;
21 | import org.everit.json.schema.loader.SchemaLoader;
22 | import org.json.JSONObject;
23 |
24 | public class TestUtils {
25 | public static org.everit.json.schema.Schema jsonSchema(JSONObject rawSchema) {
26 | return SchemaLoader.builder()
27 | .draftV7Support()
28 | .addFormatValidator(new DateFormatValidator())
29 | .addFormatValidator(new TimeFormatValidator())
30 | .addFormatValidator(new DateTimeFormatValidator())
31 | .schemaJson(rawSchema)
32 | .build()
33 | .load()
34 | .build();
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/src/test/resources/com/github/jcustenborder/kafka/connect/json/FromJson/inline.json:
--------------------------------------------------------------------------------
1 | {
2 | "title" : "Inline Config",
3 | "input" : {
4 | "topic" : "foo",
5 | "kafkaPartition" : 1,
6 | "keySchema" : {
7 | "type" : "STRING",
8 | "isOptional" : false
9 | },
10 | "key" : "foo",
11 | "valueSchema" : {
12 | "type" : "BYTES",
13 | "isOptional" : false
14 | },
15 | "value" : "ewogICJmaXJzdE5hbWUiOiAiSm9obiIsCiAgImxhc3ROYW1lIjogIkRvZSIsCiAgImFnZSI6IDIxCn0=",
16 | "timestamp" : 1530286549123,
17 | "timestampType" : "CREATE_TIME",
18 | "offset" : 91283741,
19 | "headers" : [ ]
20 | },
21 | "description" : "This example takes an input value that is a byte array and reads this value based on the supplied schema to a Kafka Connect value. The result is data that is based on the schema",
22 | "name" : "Inline Config",
23 | "config" : {
24 | "json.schema.location" : "Inline",
25 | "json.schema.inline" : "{\n \"$id\": \"https://example.com/person.schema.json\",\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"title\": \"Person\",\n \"type\": \"object\",\n \"properties\": {\n \"firstName\": {\n \"type\": \"string\",\n \"description\": \"The person's first name.\"\n },\n \"lastName\": {\n \"type\": \"string\",\n \"description\": \"The person's last name.\"\n },\n \"age\": {\n \"description\": \"Age in years which must be equal to or greater than zero.\",\n \"type\": \"integer\",\n \"minimum\": 0\n }\n }\n}"
26 | },
27 | "childClass" : "Value"
28 | }
--------------------------------------------------------------------------------
/src/test/resources/com/github/jcustenborder/kafka/connect/json/SchemaConverterTest/nested.schema.json:
--------------------------------------------------------------------------------
1 | {
2 | "$schema": "http://json-schema.org/draft-07/schema#",
3 | "title": "Customer",
4 | "definitions": {
5 | "address": {
6 | "title": "Address",
7 | "type": "object",
8 | "properties": {
9 | "street_address": { "type": "string" },
10 | "city": { "type": "string" },
11 | "state": { "type": "string" }
12 | },
13 | "required": ["street_address", "city", "state"]
14 | }
15 | },
16 |
17 | "type": "object",
18 |
19 | "properties": {
20 | "billing_address": { "$ref": "#/definitions/address" },
21 | "shipping_address": { "$ref": "#/definitions/address" }
22 | }
23 | }
--------------------------------------------------------------------------------
/src/test/resources/com/github/jcustenborder/kafka/connect/json/SchemaConverterTest/product.schema.json:
--------------------------------------------------------------------------------
1 | {
2 | "$schema": "http://json-schema.org/draft-07/schema#",
3 | "$id": "http://example.com/product.schema.json",
4 | "title": "Product",
5 | "description": "A product from Acme's catalog",
6 | "type": "object",
7 | "properties": {
8 | "price": {
9 | "description": "The price of the product",
10 | "type": "number",
11 | "exclusiveMinimum": 0
12 | },
13 | "productId": {
14 | "description": "The unique identifier for a product",
15 | "type": "integer"
16 | },
17 | "productName": {
18 | "description": "Name of the product",
19 | "type": "string"
20 | }
21 | },
22 | "required": [ "productId", "productName", "price" ]
23 | }
--------------------------------------------------------------------------------
/src/test/resources/com/github/jcustenborder/kafka/connect/json/SchemaConverterTest/wikimedia.recentchange.schema.json:
--------------------------------------------------------------------------------
1 | {
2 | "title": "mediawiki/recentchange",
3 | "description": "Represents a MW RecentChange event. https://www.mediawiki.org/wiki/Manual:RCFeed\n",
4 | "$id": "/mediawiki/recentchange/1.0.0",
5 | "$schema": "https://json-schema.org/draft-07/schema#",
6 | "type": "object",
7 | "additionalProperties": true,
8 | "required": [
9 | "$schema",
10 | "meta"
11 | ],
12 | "properties": {
13 | "$schema": {
14 | "type": "string",
15 | "description": "A URI identifying the JSONSchema for this event. This should match an schema's $id in a schema repository. E.g. /schema_name/1.0.0\n"
16 | },
17 | "meta": {
18 | "type": "object",
19 | "required": [
20 | "id",
21 | "dt",
22 | "stream"
23 | ],
24 | "properties": {
25 | "uri": {
26 | "type": "string",
27 | "format": "uri-reference",
28 | "maxLength": 8192,
29 | "description": "Unique URI identifying the event or entity"
30 | },
31 | "request_id": {
32 | "type": "string",
33 | "description": "Unique ID of the request that caused the event"
34 | },
35 | "id": {
36 | "type": "string",
37 | "pattern": "^[a-fA-F0-9]{8}(-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}$",
38 | "maxLength": 36,
39 | "description": "Unique ID of this event"
40 | },
41 | "dt": {
42 | "type": "string",
43 | "format": "date-time",
44 | "maxLength": 128,
45 | "description": "Event datetime, in ISO-8601 format"
46 | },
47 | "domain": {
48 | "type": "string",
49 | "description": "Domain the event or entity pertains to",
50 | "minLength": 1
51 | },
52 | "stream": {
53 | "type": "string",
54 | "description": "Name of the stream/queue/dataset that this event belongs in",
55 | "minLength": 1
56 | }
57 | }
58 | },
59 | "id": {
60 | "description": "ID of the recentchange event (rcid).",
61 | "type": [
62 | "integer",
63 | "null"
64 | ]
65 | },
66 | "type": {
67 | "description": "Type of recentchange event (rc_type). One of \"edit\", \"new\", \"log\", \"categorize\", or \"external\". (See Manual:Recentchanges table#rc_type)\n",
68 | "type": "string"
69 | },
70 | "title": {
71 | "description": "Full page name, from Title::getPrefixedText.",
72 | "type": "string"
73 | },
74 | "namespace": {
75 | "description": "ID of relevant namespace of affected page (rc_namespace, page_namespace). This is -1 (\"Special\") for log events.\n",
76 | "type": "integer"
77 | },
78 | "comment": {
79 | "description": "(rc_comment)",
80 | "type": "string"
81 | },
82 | "parsedcomment": {
83 | "description": "The rc_comment parsed into simple HTML. Optional",
84 | "type": "string"
85 | },
86 | "timestamp": {
87 | "description": "Unix timestamp (derived from rc_timestamp).",
88 | "type": "integer"
89 | },
90 | "user": {
91 | "description": "(rc_user_text)",
92 | "type": "string"
93 | },
94 | "bot": {
95 | "description": "(rc_bot)",
96 | "type": "boolean"
97 | },
98 | "server_url": {
99 | "description": "$wgCanonicalServer",
100 | "type": "string"
101 | },
102 | "server_name": {
103 | "description": "$wgServerName",
104 | "type": "string"
105 | },
106 | "server_script_path": {
107 | "description": "$wgScriptPath",
108 | "type": "string"
109 | },
110 | "wiki": {
111 | "description": "wfWikiID ($wgDBprefix, $wgDBname)",
112 | "type": "string"
113 | },
114 | "minor": {
115 | "description": "(rc_minor).",
116 | "type": "boolean"
117 | },
118 | "patrolled": {
119 | "description": "(rc_patrolled). This property only exists if patrolling is supported for this event (based on $wgUseRCPatrol, $wgUseNPPatrol).\n",
120 | "type": "boolean"
121 | },
122 | "length": {
123 | "description": "Length of old and new change",
124 | "type": "object",
125 | "properties": {
126 | "old": {
127 | "description": "(rc_old_len)",
128 | "type": [
129 | "integer",
130 | "null"
131 | ]
132 | },
133 | "new": {
134 | "description": "(rc_new_len)",
135 | "type": [
136 | "integer",
137 | "null"
138 | ]
139 | }
140 | }
141 | },
142 | "revision": {
143 | "description": "Old and new revision IDs",
144 | "type": "object",
145 | "properties": {
146 | "new": {
147 | "description": "(rc_last_oldid)",
148 | "type": [
149 | "integer",
150 | "null"
151 | ]
152 | },
153 | "old": {
154 | "description": "(rc_this_oldid)",
155 | "type": [
156 | "integer",
157 | "null"
158 | ]
159 | }
160 | }
161 | },
162 | "log_id": {
163 | "description": "(rc_log_id)",
164 | "type": [
165 | "integer",
166 | "null"
167 | ]
168 | },
169 | "log_type": {
170 | "description": "(rc_log_type)",
171 | "type": [
172 | "string",
173 | "null"
174 | ]
175 | },
176 | "log_action": {
177 | "description": "(rc_log_action)",
178 | "type": "string"
179 | },
180 | "log_params": {
181 | "description": "Property only exists if event has rc_params.",
182 | "type": [
183 | "array",
184 | "object",
185 | "string"
186 | ],
187 | "additionalProperties": true
188 | },
189 | "log_action_comment": {
190 | "type": [
191 | "string",
192 | "null"
193 | ]
194 | }
195 | }
196 | }
--------------------------------------------------------------------------------
/src/test/resources/com/github/jcustenborder/kafka/connect/json/basic.data.json:
--------------------------------------------------------------------------------
1 | {
2 | "firstName": "John",
3 | "lastName": "Doe",
4 | "age": 21
5 | }
--------------------------------------------------------------------------------
/src/test/resources/com/github/jcustenborder/kafka/connect/json/basic.schema.json:
--------------------------------------------------------------------------------
1 | {
2 | "$id": "https://example.com/person.schema.json",
3 | "$schema": "http://json-schema.org/draft-07/schema#",
4 | "title": "Person",
5 | "type": "object",
6 | "properties": {
7 | "firstName": {
8 | "type": "string",
9 | "description": "The person's first name."
10 | },
11 | "lastName": {
12 | "type": "string",
13 | "description": "The person's last name."
14 | },
15 | "age": {
16 | "description": "Age in years which must be equal to or greater than zero.",
17 | "type": "integer",
18 | "minimum": 0
19 | }
20 | }
21 | }
--------------------------------------------------------------------------------
/src/test/resources/com/github/jcustenborder/kafka/connect/json/customdate.data.json:
--------------------------------------------------------------------------------
1 | {
2 | "Timestamp": "2020-01-07 04:47:05.0000000"
3 | }
--------------------------------------------------------------------------------
/src/test/resources/com/github/jcustenborder/kafka/connect/json/customdate.schema.json:
--------------------------------------------------------------------------------
1 | {
2 | "$id": "https://example.com/person.schema.json",
3 | "$schema": "http://json-schema.org/draft-07/schema#",
4 | "title": "Person",
5 | "type": "object",
6 | "properties": {
7 | "Timestamp": {
8 | "type": "string",
9 | "description": "Timestamp",
10 | "format": "custom-timestamp",
11 | "dateTimeFormat": "yyyy-MM-dd HH:mm:ss.SSSSSSS"
12 | }
13 | }
14 | }
--------------------------------------------------------------------------------
/src/test/resources/com/github/jcustenborder/kafka/connect/json/geo.data.json:
--------------------------------------------------------------------------------
1 | {
2 | "latitude": 48.858093,
3 | "longitude": 2.294694
4 | }
--------------------------------------------------------------------------------
/src/test/resources/com/github/jcustenborder/kafka/connect/json/geo.schema.json:
--------------------------------------------------------------------------------
1 | {
2 | "$id": "https://example.com/geographical-location.schema.json",
3 | "$schema": "http://json-schema.org/draft-07/schema#",
4 | "title": "Longitude and Latitude Values",
5 | "description": "A geographical coordinate.",
6 | "required": [
7 | "latitude",
8 | "longitude"
9 | ],
10 | "type": "object",
11 | "properties": {
12 | "latitude": {
13 | "type": "number",
14 | "minimum": -90,
15 | "maximum": 90
16 | },
17 | "longitude": {
18 | "type": "number",
19 | "minimum": -180,
20 | "maximum": 180
21 | }
22 | }
23 | }
--------------------------------------------------------------------------------
/src/test/resources/com/github/jcustenborder/kafka/connect/json/wikimedia.recentchange.data.json:
--------------------------------------------------------------------------------
1 | {
2 | "$schema": "/mediawiki/recentchange/1.0.0",
3 | "meta": {
4 | "uri": "https://sl.wikipedia.org/wiki/Iztrebek",
5 | "request_id": "61a2da40-3fea-450f-8404-8ed5e98e784a",
6 | "id": "5fe0b22e-1ed1-4acd-9e06-07a39ef800e7",
7 | "dt": "2020-05-14T16:24:27Z",
8 | "domain": "sl.wikipedia.org",
9 | "stream": "mediawiki.recentchange",
10 | "topic": "eqiad.mediawiki.recentchange",
11 | "partition": 0,
12 | "offset": 2403979241
13 | },
14 | "id": 21345744,
15 | "type": "edit",
16 | "namespace": 0,
17 | "title": "Iztrebek",
18 | "comment": "vrnitev sprememb uporabnika [[Special:Contributions/176.76.241.31|176.76.241.31]] ([[User talk:176.76.241.31|pogovor]]) na zadnje urejanje uporabnika [[User:Samuele2002|Samuele2002]]",
19 | "timestamp": 1589473467,
20 | "user": "Upwinxp",
21 | "bot": false,
22 | "minor": true,
23 | "length": {
24 | "old": 561,
25 | "new": 1061
26 | },
27 | "revision": {
28 | "old": 5320868,
29 | "new": 5320869
30 | },
31 | "server_url": "https://sl.wikipedia.org",
32 | "server_name": "sl.wikipedia.org",
33 | "server_script_path": "/w",
34 | "wiki": "slwiki",
35 | "parsedcomment": "vrnitev sprememb uporabnika 176.76.241.31 (pogovor) na zadnje urejanje uporabnika Samuele2002"
36 | }
--------------------------------------------------------------------------------
/src/test/resources/com/github/jcustenborder/kafka/connect/json/wikimedia.recentchange.schema.json:
--------------------------------------------------------------------------------
1 | {
2 | "title": "mediawiki/recentchange",
3 | "description": "Represents a MW RecentChange event. https://www.mediawiki.org/wiki/Manual:RCFeed\n",
4 | "$id": "/mediawiki/recentchange/1.0.0",
5 | "$schema": "https://json-schema.org/draft-07/schema#",
6 | "type": "object",
7 | "additionalProperties": true,
8 | "required": [
9 | "$schema",
10 | "meta"
11 | ],
12 | "properties": {
13 | "$schema": {
14 | "type": "string",
15 | "description": "A URI identifying the JSONSchema for this event. This should match an schema's $id in a schema repository. E.g. /schema_name/1.0.0\n"
16 | },
17 | "meta": {
18 | "type": "object",
19 | "required": [
20 | "id",
21 | "dt",
22 | "stream"
23 | ],
24 | "properties": {
25 | "uri": {
26 | "type": "string",
27 | "format": "uri-reference",
28 | "maxLength": 8192,
29 | "description": "Unique URI identifying the event or entity"
30 | },
31 | "request_id": {
32 | "type": "string",
33 | "description": "Unique ID of the request that caused the event"
34 | },
35 | "id": {
36 | "type": "string",
37 | "pattern": "^[a-fA-F0-9]{8}(-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}$",
38 | "maxLength": 36,
39 | "description": "Unique ID of this event"
40 | },
41 | "dt": {
42 | "type": "string",
43 | "format": "date-time",
44 | "maxLength": 128,
45 | "description": "Event datetime, in ISO-8601 format"
46 | },
47 | "domain": {
48 | "type": "string",
49 | "description": "Domain the event or entity pertains to",
50 | "minLength": 1
51 | },
52 | "stream": {
53 | "type": "string",
54 | "description": "Name of the stream/queue/dataset that this event belongs in",
55 | "minLength": 1
56 | }
57 | }
58 | },
59 | "id": {
60 | "description": "ID of the recentchange event (rcid).",
61 | "type": [
62 | "integer",
63 | "null"
64 | ]
65 | },
66 | "type": {
67 | "description": "Type of recentchange event (rc_type). One of \"edit\", \"new\", \"log\", \"categorize\", or \"external\". (See Manual:Recentchanges table#rc_type)\n",
68 | "type": "string"
69 | },
70 | "title": {
71 | "description": "Full page name, from Title::getPrefixedText.",
72 | "type": "string"
73 | },
74 | "namespace": {
75 | "description": "ID of relevant namespace of affected page (rc_namespace, page_namespace). This is -1 (\"Special\") for log events.\n",
76 | "type": "integer"
77 | },
78 | "comment": {
79 | "description": "(rc_comment)",
80 | "type": "string"
81 | },
82 | "parsedcomment": {
83 | "description": "The rc_comment parsed into simple HTML. Optional",
84 | "type": "string"
85 | },
86 | "timestamp": {
87 | "description": "Unix timestamp (derived from rc_timestamp).",
88 | "type": "integer"
89 | },
90 | "user": {
91 | "description": "(rc_user_text)",
92 | "type": "string"
93 | },
94 | "bot": {
95 | "description": "(rc_bot)",
96 | "type": "boolean"
97 | },
98 | "server_url": {
99 | "description": "$wgCanonicalServer",
100 | "type": "string"
101 | },
102 | "server_name": {
103 | "description": "$wgServerName",
104 | "type": "string"
105 | },
106 | "server_script_path": {
107 | "description": "$wgScriptPath",
108 | "type": "string"
109 | },
110 | "wiki": {
111 | "description": "wfWikiID ($wgDBprefix, $wgDBname)",
112 | "type": "string"
113 | },
114 | "minor": {
115 | "description": "(rc_minor).",
116 | "type": "boolean"
117 | },
118 | "patrolled": {
119 | "description": "(rc_patrolled). This property only exists if patrolling is supported for this event (based on $wgUseRCPatrol, $wgUseNPPatrol).\n",
120 | "type": "boolean"
121 | },
122 | "length": {
123 | "description": "Length of old and new change",
124 | "type": "object",
125 | "properties": {
126 | "old": {
127 | "description": "(rc_old_len)",
128 | "type": [
129 | "integer",
130 | "null"
131 | ]
132 | },
133 | "new": {
134 | "description": "(rc_new_len)",
135 | "type": [
136 | "integer",
137 | "null"
138 | ]
139 | }
140 | }
141 | },
142 | "revision": {
143 | "description": "Old and new revision IDs",
144 | "type": "object",
145 | "properties": {
146 | "new": {
147 | "description": "(rc_last_oldid)",
148 | "type": [
149 | "integer",
150 | "null"
151 | ]
152 | },
153 | "old": {
154 | "description": "(rc_this_oldid)",
155 | "type": [
156 | "integer",
157 | "null"
158 | ]
159 | }
160 | }
161 | },
162 | "log_id": {
163 | "description": "(rc_log_id)",
164 | "type": [
165 | "integer",
166 | "null"
167 | ]
168 | },
169 | "log_type": {
170 | "description": "(rc_log_type)",
171 | "type": [
172 | "string",
173 | "null"
174 | ]
175 | },
176 | "log_action": {
177 | "description": "(rc_log_action)",
178 | "type": "string"
179 | },
180 | "log_params": {
181 | "description": "Property only exists if event has rc_params.",
182 | "type": [
183 | "array",
184 | "object",
185 | "string"
186 | ],
187 | "additionalProperties": true
188 | },
189 | "log_action_comment": {
190 | "type": [
191 | "string",
192 | "null"
193 | ]
194 | }
195 | }
196 | }
--------------------------------------------------------------------------------
/src/test/resources/logback.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
--------------------------------------------------------------------------------