├── .editorconfig ├── .github ├── FUNDING.yml └── workflows │ ├── main.yml │ └── release.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── build.gradle.kts ├── docker-compose.yml ├── examples ├── .gitignore ├── User.avsc ├── base.yml ├── foo.json ├── foo.proto ├── with-references.avsc ├── with_references.yml └── without_compatibility.avsc ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── renovate.json ├── schema-registry-gitops.svg ├── settings.gradle.kts └── src ├── main ├── kotlin │ └── dev │ │ └── domnikl │ │ └── schemaregistrygitops │ │ ├── CLI.kt │ │ ├── Compatibility.kt │ │ ├── Configuration.kt │ │ ├── Main.kt │ │ ├── ParsedSchema.kt │ │ ├── SchemaParseException.kt │ │ ├── SchemaRegistryClient.kt │ │ ├── ServerVersionMismatchException.kt │ │ ├── State.kt │ │ ├── Subject.kt │ │ ├── cli │ │ ├── Apply.kt │ │ ├── Dump.kt │ │ └── Plan.kt │ │ └── state │ │ ├── Applier.kt │ │ ├── Diffing.kt │ │ ├── Dumper.kt │ │ └── Persistence.kt └── resources │ ├── logback.xml │ └── version.txt └── test ├── kotlin └── dev │ └── domnikl │ └── schemaregistrygitops │ ├── CLITest.kt │ ├── ConfigurationTest.kt │ ├── ParsedSchemaTest.kt │ ├── SchemaRegistryClientTest.kt │ ├── StateTest.kt │ ├── TestUtils.kt │ ├── cli │ ├── ApplyTest.kt │ ├── DumpTest.kt │ └── PlanTest.kt │ └── state │ ├── ApplierTest.kt │ ├── DiffingTest.kt │ ├── DumperTest.kt │ └── PersistenceTest.kt └── resources ├── client.properties ├── empty.yml ├── neither_schema_nor_file.yml ├── no_compatibility.yml ├── only_compatibility.yml ├── only_normalize.yml ├── referenced_file_does_not_exist.yml ├── schemas ├── avsc.diff ├── deltaA.avsc ├── deltaA.json ├── deltaA.proto ├── deltaB.avsc ├── deltaB.json ├── deltaB.proto ├── json.diff ├── key.avsc ├── proto.diff ├── with_subjects.avsc └── with_subjects_and_references.avsc ├── subject_name_is_missing.yml ├── version.txt ├── with_inline_schema.yml ├── with_subjects.yml ├── with_subjects_and_compatibility.yml └── with_subjects_and_references.yml /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | # noinspection EditorConfigKeyCorrectness 4 | [*] 5 | end_of_line = lf 6 | indent_size = 4 7 | indent_style = space 8 | insert_final_newline = true 9 | max_line_length = 140 10 | trim_trailing_whitespace = true 11 | ktlint_standard_filename = disabled 12 | ktlint_standard_function-naming = disabled 13 | ktlint_code_style = intellij_idea 14 | ktlint_function_signature_rule_force_multiline_when_parameter_count_greater_or_equal_than = 5 15 | ij_java_class_count_to_use_import_on_demand = 999 16 | ij_kotlin_allow_trailing_comma = false 17 | ij_kotlin_allow_trailing_comma_on_call_site = false 18 | ij_kotlin_line_break_after_multiline_when_entry = false 19 | ij_kotlin_name_count_to_use_star_import = 999 20 | ij_kotlin_name_count_to_use_star_import_for_members = 999 21 | ij_kotlin_packages_to_use_import_on_demand = "" 22 | 23 | # YAML Files 24 | [*.{yml,yaml}] 25 | indent_size = 2 26 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: https://paypal.me/DominikLiebler 13 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: push 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Check out the repo 10 | uses: actions/checkout@v4 11 | - name: Set up QEMU 12 | uses: docker/setup-qemu-action@v3 13 | - name: Set up Docker Buildx 14 | uses: docker/setup-buildx-action@v3 15 | - name: Validate build configuration 16 | uses: docker/build-push-action@v6 17 | with: 18 | call: check 19 | - name: Build 20 | uses: docker/build-push-action@v6 21 | with: 22 | context: . 23 | push: false 24 | tags: domnikl/schema-registry-gitops 25 | platforms: linux/amd64,linux/arm64 26 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Publish Docker image 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | jobs: 8 | push_to_registry: 9 | name: Push Docker image to Docker Hub 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Check out the repo 13 | uses: actions/checkout@v4 14 | - name: Log in to the container registry 15 | uses: docker/login-action@6d4b68b490aef8836e8fb5e50ee7b3bdfa5894f0 16 | with: 17 | username: ${{ secrets.DOCKER_USERNAME }} 18 | password: ${{ secrets.DOCKER_PASSWORD }} 19 | - name: Extract metadata (tags, labels) for Docker 20 | id: meta 21 | uses: docker/metadata-action@418e4b98bf2841bd337d0b24fe63cb36dc8afa55 22 | with: 23 | images: domnikl/schema-registry-gitops 24 | - name: Set up QEMU 25 | uses: docker/setup-qemu-action@v3 26 | - name: Set up Docker Buildx 27 | uses: docker/setup-buildx-action@v3 28 | - name: Validate build configuration 29 | uses: docker/build-push-action@v6 30 | with: 31 | call: check 32 | - name: Build 33 | uses: docker/build-push-action@v6 34 | with: 35 | context: . 36 | push: true 37 | tags: ${{ steps.meta.outputs.tags }} 38 | labels: ${{ steps.meta.outputs.labels }} 39 | platforms: linux/amd64,linux/arm64 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea/ 2 | /.gradle/ 3 | /build/ 4 | /dumped.yml 5 | /.test/ 6 | /examples/client.properties 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM amazoncorretto:17.0.15-alpine AS tester 2 | 3 | WORKDIR /home/cuser 4 | 5 | COPY ./gradle /home/cuser/gradle 6 | COPY ./gradlew ./gradle.properties ./build.gradle.kts ./settings.gradle.kts ./.editorconfig /home/cuser/ 7 | RUN ./gradlew --no-daemon build 8 | COPY ./src /home/cuser/src 9 | RUN ./gradlew --no-daemon check 10 | 11 | FROM tester AS builder 12 | RUN ./gradlew --no-daemon shadowJar 13 | 14 | FROM amazoncorretto:17.0.15-alpine AS distribution 15 | COPY --from=builder /home/cuser/build/libs/schema-registry-gitops.jar /home/cuser/schema-registry-gitops.jar 16 | 17 | WORKDIR /home/cuser 18 | 19 | ENTRYPOINT ["java", "-XX:+UseG1GC", "-XX:MaxGCPauseMillis=100", "-XX:+UseStringDeduplication", "-jar", "schema-registry-gitops.jar"] 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2020-2022 Dominik Liebler 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # schema-registry-gitops 2 | 3 | [![build](https://github.com/domnikl/schema-registry-gitops/workflows/build/badge.svg)](https://github.com/domnikl/schema-registry-gitops/actions) 4 | [![Docker Pulls](https://img.shields.io/docker/pulls/domnikl/schema-registry-gitops)](https://hub.docker.com/repository/docker/domnikl/schema-registry-gitops) 5 | 6 | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) 7 | 8 | Manages subjects, compatibility levels and schema registration for [Confluent Schema Registry](https://docs.confluent.io/platform/current/schema-registry/index.html) through applying a desired state file. 9 | 10 | ## Overview 11 | 12 | Schema Registry GitOps is an Infrastructure as Code tool that applies a desired state configured through simple YAML and 13 | Avro/Protobuf/JSON Schema files to a schema registry. That way you can keep a version control history of your 14 | schemas and use all your favorite tools to validate, review, merge and evolve schemas in your CI/CD pipeline. 15 | 16 | ![Yaml (+Avro, Protobuf, JSON) -> CI/CD -> Schema Registry](schema-registry-gitops.svg) 17 | 18 | ## Usage 19 | 20 | ``` 21 | Usage: schema-registry-gitops [-hvV] [--properties=] 22 | [-r=] [COMMAND] 23 | Manages schema registries through Infrastructure as Code 24 | -h, --help Show this help message and exit. 25 | --properties= 26 | a Java Properties file for client configuration 27 | (optional) 28 | -r, --registry= schema registry endpoint, overwrites 'schema. 29 | registry.url' from properties, can also be a 30 | list of urls separated by comma 31 | -v, --verbose enable verbose logging 32 | -V, --version Print version information and exit. 33 | Commands: 34 | apply applies the state to the given schema registry 35 | dump prints the current state 36 | plan validate and plan schema changes, can be used to see all pending 37 | changes 38 | ``` 39 | 40 | In order to get help for a specific command, try `schema-registry-gitops -h`. 41 | 42 | ## Running in Docker 43 | 44 | `schema-registry-gitops` is available through [Docker Hub](https://hub.docker.com/repository/docker/domnikl/schema-registry-gitops), so running it in a container is as easy as: 45 | 46 | ```sh 47 | docker run -v "$(pwd)/examples":/data domnikl/schema-registry-gitops plan --properties /data/client.properties /data/base.yml 48 | ``` 49 | 50 | Please keep in mind that using a tagged release may be a good idea. 51 | 52 | ## State files 53 | 54 | The desired state is managed using the following YAML schema: 55 | 56 | ```yaml 57 | # sets global compatibility level (optional) 58 | compatibility: FULL_TRANSITIVE 59 | # sets normalize flag (optional) 60 | # setting to true enables clients to not have to pass the “normalize” query parameter to have normalization occur 61 | normalize: true 62 | subjects: 63 | # a subject that links to a file for the schema definition 64 | - name: my-new-subject-referencing-a-schema-file 65 | # sets compatibility level for this subject (optional) 66 | compatibility: BACKWARD 67 | # file paths are always relative to the given (this) YAML file 68 | file: my-actual-schema.avsc 69 | # AVRO is the default type and can safely be omitted (only available for Schema Registry >= 5.5) 70 | type: AVRO 71 | # (optional) list of references for this subject 72 | # please note that these must be present in the registry before they can be referenced here 73 | references: 74 | # name including the namespace, should be the same as the `type` being used in AVRO 75 | - name: dev.domnikl.schema-registry-gitops.User 76 | # subject name this schema is registered with in the registry 77 | subject: User-value 78 | # version of the referenced schema 79 | version: 1 80 | 81 | # another example: instead of referencing a file, it is also possible 82 | # to define the schema directly here, which is Protocol Buffers here (note explicit type here) 83 | - name: my-new-inline-schema-subject 84 | schema: 'syntax = "proto3"; 85 | package com.acme; 86 | 87 | message OtherRecord { 88 | int32 an_id = 1; 89 | }' 90 | type: PROTOBUF 91 | ``` 92 | 93 | When using multiple files, the state is the merge result of those. Keep in mind that later references of the same subject names will overwrite earlier definitions. 94 | 95 | ### compatibility 96 | 97 | Supported `compatibility` values are: 98 | * `NONE` 99 | * `FORWARD` 100 | * `BACKWARD` 101 | * `FULL` 102 | * `FORWARD_TRANSITIVE` 103 | * `BACKWARD_TRANSITIVE` 104 | * `FULL_TRANSITIVE` 105 | 106 | ### file, schema 107 | 108 | Either one of `file` (recommended) or `schema` must be set. The former contains a path to a schema file while the latter can be set 109 | to a string containing the schema. 110 | 111 | ### type 112 | 113 | Supported `type` values are: 114 | 115 | * `AVRO` 116 | * `PROTOBUF` 117 | * `JSON` 118 | 119 | _Please note that `PROTOBUF` and `JSON` are only supported for Schema Registry >= 5.5, versions prior to that only support `AVRO`._ 120 | 121 | ### references 122 | 123 | [References to other schemas](https://docs.confluent.io/platform/current/schema-registry/serdes-develop/index.html#referenced-schemas) 124 | are being configured here as an optional list of references. `name`, `subject` and `version` need to be configured for 125 | this to work. Also note that referenced schemas need to be present in the schema registry by the time that 126 | `schema-registry-gitops` runs. 127 | 128 | ## Configuration .properties 129 | 130 | Configuration properties are being used to connect to the Schema Registry. The most common use case to use them 131 | instead of just supplying `--registry` is to use SSL. The example below uses client certificates authentication. 132 | 133 | ```properties 134 | schema.registry.url=https://localhost:8081 135 | security.protocol=SSL 136 | schema.registry.ssl.truststore.location=truststore.jks 137 | schema.registry.ssl.truststore.password= 138 | schema.registry.ssl.keystore.location=keystore.jks 139 | schema.registry.ssl.keystore.password= 140 | schema.registry.ssl.key.password= 141 | ``` 142 | 143 | ## Environment variables 144 | 145 | Env variables prefixed with `SCHEMA_REGISTRY_GITOPS_` can be provided for configuration and will also be forwarded 146 | to configure the schema registry client being used. This example uses the same settings above. 147 | 148 | ```sh 149 | SCHEMA_REGISTRY_GITOPS_SCHEMA_REGISTRY_URL=https://localhost:8081 150 | SCHEMA_REGISTRY_GITOPS_SECURITY_PROTOCOL=SSL 151 | SCHEMA_REGISTRY_GITOPS_SCHEMA_REGISTRY_SSL_TRUSTSTORE_LOCATION=truststore.jks 152 | SCHEMA_REGISTRY_GITOPS_SCHEMA_REGISTRY_SSL_TRUSTSTORE_PASSWORD= 153 | SCHEMA_REGISTRY_GITOPS_SCHEMA_REGISTRY_SSL_KEYSTORE_LOCATION=keystore.jks 154 | SCHEMA_REGISTRY_GITOPS_SCHEMA_REGISTRY_SSL_KEYSTORE_PASSWORD= 155 | SCHEMA_REGISTRY_GITOPS_SCHEMA_REGISTRY_SSL_KEY_PASSWORD= 156 | ``` 157 | 158 | ## Deleting subjects ⚠️ 159 | 160 | Subjects no longer listed in the State file but present in the registry will not be deleted by default. To enable full 161 | sync between the two, use either `-d` or `--enable-deletes` in `plan` and `apply` modes. 162 | 163 | ## Development 164 | 165 | Docker is used to build and test `schema-registry-gitops` for development. 166 | 167 | ```sh 168 | # test & build 169 | docker build -t domnikl/schema-registry-gitops . 170 | 171 | # run it in Docker 172 | docker run -v ./examples:/data domnikl/schema-registry-gitops plan --registry http://localhost:8081 /data/base.yml /data/with_references.yml 173 | ``` 174 | 175 | ## Acknowledgement 176 | 177 | Schema Registry GitOps was born late in 2020 while being heavily inspired by [Shawn Seymour](https://github.com/devshawn) and his excellent [kafka-gitops](https://github.com/devshawn/kafka-gitops)! Much ❤ to [Confluent](https://www.confluent.io/) for building Schema Registry and an amazing client lib, I am really just standing on the shoulders of giants here. 178 | -------------------------------------------------------------------------------- /build.gradle.kts: -------------------------------------------------------------------------------- 1 | import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar 2 | import org.jetbrains.kotlin.gradle.tasks.KotlinCompile 3 | 4 | plugins { 5 | java 6 | kotlin("jvm") version "1.9.25" 7 | kotlin("plugin.spring") version "2.1.21" 8 | id("com.github.johnrengelman.shadow") version "8.1.1" 9 | id("org.jlleitschuh.gradle.ktlint") version "12.3.0" 10 | id("org.jetbrains.gradle.plugin.idea-ext") version "1.1.10" 11 | jacoco 12 | } 13 | 14 | group = "dev.domnikl" 15 | version = "1.12.0" 16 | 17 | repositories { 18 | mavenCentral() 19 | maven { url = uri("https://packages.confluent.io/maven/") } 20 | maven { url = uri("https://jitpack.io") } 21 | } 22 | 23 | dependencies { 24 | implementation("org.jetbrains.kotlin:kotlin-reflect") 25 | implementation("org.jetbrains.kotlin:kotlin-stdlib") 26 | 27 | implementation("info.picocli:picocli:4.7.7") 28 | 29 | implementation("org.slf4j:slf4j-api:2.0.17") 30 | implementation("ch.qos.logback:logback-classic:1.5.18") 31 | implementation("ch.qos.logback:logback-core:1.5.18") 32 | 33 | implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.19.0") 34 | implementation("com.fasterxml.jackson.core:jackson-databind:2.19.0") 35 | implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.19.0") 36 | constraints { 37 | implementation("com.google.code.gson:gson:2.13.1") { 38 | because("CVE-2022-25647") 39 | } 40 | implementation("org.apache.commons:commons-compress:1.27.1") { 41 | because("CVE-2023-42503") 42 | } 43 | } 44 | 45 | implementation("io.confluent:kafka-schema-registry-client:7.9.1") 46 | implementation("io.confluent:kafka-protobuf-serializer:7.9.1") 47 | implementation("io.confluent:kafka-json-schema-serializer:7.9.1") 48 | implementation("com.github.everit-org.json-schema:org.everit.json.schema:1.14.4") 49 | 50 | implementation("io.github.java-diff-utils:java-diff-utils:4.15") 51 | 52 | testImplementation(platform("org.junit:junit-bom:5.13.1")) 53 | testImplementation("org.junit.jupiter:junit-jupiter") 54 | testImplementation("io.mockk:mockk:1.14.2") 55 | testImplementation("com.github.stefanbirkner:system-rules:1.19.0") 56 | } 57 | 58 | tasks { 59 | withType { 60 | kotlinOptions { 61 | jvmTarget = JavaVersion.VERSION_17.majorVersion 62 | } 63 | } 64 | 65 | val versionTxt = register("versionTxt") { 66 | doLast { 67 | val outputDir = File("${sourceSets.main.get().output.resourcesDir}/version.txt") 68 | 69 | if (outputDir.exists()) outputDir.writeText("${project.version}") 70 | } 71 | } 72 | 73 | withType { 74 | dependsOn(versionTxt) 75 | archiveFileName.set("schema-registry-gitops.jar") 76 | } 77 | 78 | withType { 79 | dependsOn(versionTxt) 80 | 81 | manifest { 82 | attributes["Main-Class"] = "dev.domnikl.schemaregistrygitops.MainKt" 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | services: 3 | kafka: 4 | image: confluentinc/cp-kafka:7.9.1 5 | ports: 6 | - "9092:9092" 7 | - "9101:9101" 8 | environment: 9 | KAFKA_NODE_ID: 1 10 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' 11 | KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092' 12 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 13 | KAFKA_DEFAULT_REPLICATION_FACTOR: 1 14 | KAFKA_MIN_INSYNC_REPLICAS: 1 15 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 16 | KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 17 | KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 18 | KAFKA_PROCESS_ROLES: 'broker,controller' 19 | KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093' 20 | KAFKA_LISTENERS: 'PLAINTEXT://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092' 21 | KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' 22 | KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' 23 | KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' 24 | CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk' 25 | 26 | schemaregistry: 27 | image: confluentinc/cp-schema-registry:7.9.1 28 | restart: always 29 | depends_on: 30 | - kafka 31 | environment: 32 | SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "kafka:29092" 33 | SCHEMA_REGISTRY_KAFKASTORE_TOPIC_REPLICATION_FACTOR: 1 34 | SCHEMA_REGISTRY_HOST_NAME: schemaregistry 35 | SCHEMA_REGISTRY_LISTENERS: "http://0.0.0.0:8081" 36 | ports: 37 | - 8081:8081 38 | -------------------------------------------------------------------------------- /examples/.gitignore: -------------------------------------------------------------------------------- 1 | /client.properties 2 | /keystore.jks 3 | /truststore.jks 4 | -------------------------------------------------------------------------------- /examples/User.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "User", 4 | "namespace": "dev.domnikl.schema_registry_gitops", 5 | "fields": [ 6 | { 7 | "name": "name", 8 | "type": "string" 9 | }, 10 | { 11 | "name": "role", 12 | "type": "string" 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /examples/base.yml: -------------------------------------------------------------------------------- 1 | compatibility: FULL_TRANSITIVE 2 | normalize: true 3 | subjects: 4 | - name: User-value 5 | file: User.avsc 6 | compatibility: BACKWARD 7 | 8 | - name: without-compatiblity 9 | file: without_compatibility.avsc 10 | 11 | - name: foo-proto-value 12 | file: foo.proto 13 | type: PROTOBUF 14 | 15 | - name: json-schema-value 16 | file: foo.json 17 | type: JSON 18 | -------------------------------------------------------------------------------- /examples/foo.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "f1": { 5 | "type": "string" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /examples/foo.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package com.acme; 3 | 4 | message OtherRecord { 5 | int32 an_id = 1; 6 | } 7 | -------------------------------------------------------------------------------- /examples/with-references.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "UserReference", 4 | "namespace": "dev.domnikl.schema_registry_gitops", 5 | "fields": [ 6 | { 7 | "name": "user", 8 | "type": "dev.domnikl.schema_registry_gitops.User" 9 | } 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /examples/with_references.yml: -------------------------------------------------------------------------------- 1 | subjects: 2 | - name: with-references 3 | file: with-references.avsc 4 | type: AVRO 5 | references: 6 | - name: dev.domnikl.schema_registry_gitops.User 7 | subject: User-value 8 | version: 1 9 | 10 | - name: User-value 11 | file: foo.json 12 | type: JSON 13 | compatibility: BACKWARD 14 | -------------------------------------------------------------------------------- /examples/without_compatibility.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "TestRecord", 4 | "namespace": "dev.domnikl.schema_registry_gitops", 5 | "fields": [ 6 | { 7 | "name": "hello", 8 | "type": "string" 9 | } 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | kotlin.code.style=official -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/domnikl/schema-registry-gitops/0667175bcc1bf3156cedeab85bbc87fa87fd4656/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.2-bin.zip 4 | networkTimeout=10000 5 | validateDistributionUrl=true 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | # SPDX-License-Identifier: Apache-2.0 19 | # 20 | 21 | ############################################################################## 22 | # 23 | # Gradle start up script for POSIX generated by Gradle. 24 | # 25 | # Important for running: 26 | # 27 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 28 | # noncompliant, but you have some other compliant shell such as ksh or 29 | # bash, then to run this script, type that shell name before the whole 30 | # command line, like: 31 | # 32 | # ksh Gradle 33 | # 34 | # Busybox and similar reduced shells will NOT work, because this script 35 | # requires all of these POSIX shell features: 36 | # * functions; 37 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 38 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 39 | # * compound commands having a testable exit status, especially «case»; 40 | # * various built-in commands including «command», «set», and «ulimit». 41 | # 42 | # Important for patching: 43 | # 44 | # (2) This script targets any POSIX shell, so it avoids extensions provided 45 | # by Bash, Ksh, etc; in particular arrays are avoided. 46 | # 47 | # The "traditional" practice of packing multiple parameters into a 48 | # space-separated string is a well documented source of bugs and security 49 | # problems, so this is (mostly) avoided, by progressively accumulating 50 | # options in "$@", and eventually passing that to Java. 51 | # 52 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 53 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 54 | # see the in-line comments for details. 55 | # 56 | # There are tweaks for specific operating systems such as AIX, CygWin, 57 | # Darwin, MinGW, and NonStop. 58 | # 59 | # (3) This script is generated from the Groovy template 60 | # https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 61 | # within the Gradle project. 62 | # 63 | # You can find Gradle at https://github.com/gradle/gradle/. 64 | # 65 | ############################################################################## 66 | 67 | # Attempt to set APP_HOME 68 | 69 | # Resolve links: $0 may be a link 70 | app_path=$0 71 | 72 | # Need this for daisy-chained symlinks. 73 | while 74 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 75 | [ -h "$app_path" ] 76 | do 77 | ls=$( ls -ld "$app_path" ) 78 | link=${ls#*' -> '} 79 | case $link in #( 80 | /*) app_path=$link ;; #( 81 | *) app_path=$APP_HOME$link ;; 82 | esac 83 | done 84 | 85 | # This is normally unused 86 | # shellcheck disable=SC2034 87 | APP_BASE_NAME=${0##*/} 88 | # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) 89 | APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit 90 | 91 | # Use the maximum available, or set MAX_FD != -1 to use that value. 92 | MAX_FD=maximum 93 | 94 | warn () { 95 | echo "$*" 96 | } >&2 97 | 98 | die () { 99 | echo 100 | echo "$*" 101 | echo 102 | exit 1 103 | } >&2 104 | 105 | # OS specific support (must be 'true' or 'false'). 106 | cygwin=false 107 | msys=false 108 | darwin=false 109 | nonstop=false 110 | case "$( uname )" in #( 111 | CYGWIN* ) cygwin=true ;; #( 112 | Darwin* ) darwin=true ;; #( 113 | MSYS* | MINGW* ) msys=true ;; #( 114 | NONSTOP* ) nonstop=true ;; 115 | esac 116 | 117 | CLASSPATH="\\\"\\\"" 118 | 119 | 120 | # Determine the Java command to use to start the JVM. 121 | if [ -n "$JAVA_HOME" ] ; then 122 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 123 | # IBM's JDK on AIX uses strange locations for the executables 124 | JAVACMD=$JAVA_HOME/jre/sh/java 125 | else 126 | JAVACMD=$JAVA_HOME/bin/java 127 | fi 128 | if [ ! -x "$JAVACMD" ] ; then 129 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 130 | 131 | Please set the JAVA_HOME variable in your environment to match the 132 | location of your Java installation." 133 | fi 134 | else 135 | JAVACMD=java 136 | if ! command -v java >/dev/null 2>&1 137 | then 138 | die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 139 | 140 | Please set the JAVA_HOME variable in your environment to match the 141 | location of your Java installation." 142 | fi 143 | fi 144 | 145 | # Increase the maximum file descriptors if we can. 146 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 147 | case $MAX_FD in #( 148 | max*) 149 | # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. 150 | # shellcheck disable=SC2039,SC3045 151 | MAX_FD=$( ulimit -H -n ) || 152 | warn "Could not query maximum file descriptor limit" 153 | esac 154 | case $MAX_FD in #( 155 | '' | soft) :;; #( 156 | *) 157 | # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. 158 | # shellcheck disable=SC2039,SC3045 159 | ulimit -n "$MAX_FD" || 160 | warn "Could not set maximum file descriptor limit to $MAX_FD" 161 | esac 162 | fi 163 | 164 | # Collect all arguments for the java command, stacking in reverse order: 165 | # * args from the command line 166 | # * the main class name 167 | # * -classpath 168 | # * -D...appname settings 169 | # * --module-path (only if needed) 170 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 171 | 172 | # For Cygwin or MSYS, switch paths to Windows format before running java 173 | if "$cygwin" || "$msys" ; then 174 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 175 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 176 | 177 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 178 | 179 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 180 | for arg do 181 | if 182 | case $arg in #( 183 | -*) false ;; # don't mess with options #( 184 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 185 | [ -e "$t" ] ;; #( 186 | *) false ;; 187 | esac 188 | then 189 | arg=$( cygpath --path --ignore --mixed "$arg" ) 190 | fi 191 | # Roll the args list around exactly as many times as the number of 192 | # args, so each arg winds up back in the position where it started, but 193 | # possibly modified. 194 | # 195 | # NB: a `for` loop captures its iteration list before it begins, so 196 | # changing the positional parameters here affects neither the number of 197 | # iterations, nor the values presented in `arg`. 198 | shift # remove old arg 199 | set -- "$@" "$arg" # push replacement arg 200 | done 201 | fi 202 | 203 | 204 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 205 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 206 | 207 | # Collect all arguments for the java command: 208 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, 209 | # and any embedded shellness will be escaped. 210 | # * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be 211 | # treated as '${Hostname}' itself on the command line. 212 | 213 | set -- \ 214 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 215 | -classpath "$CLASSPATH" \ 216 | -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ 217 | "$@" 218 | 219 | # Stop when "xargs" is not available. 220 | if ! command -v xargs >/dev/null 2>&1 221 | then 222 | die "xargs is not available" 223 | fi 224 | 225 | # Use "xargs" to parse quoted args. 226 | # 227 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 228 | # 229 | # In Bash we could simply go: 230 | # 231 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 232 | # set -- "${ARGS[@]}" "$@" 233 | # 234 | # but POSIX shell has neither arrays nor command substitution, so instead we 235 | # post-process each arg (as a line of input to sed) to backslash-escape any 236 | # character that might be a shell metacharacter, then use eval to reverse 237 | # that process (while maintaining the separation between arguments), and wrap 238 | # the whole thing up as a single "set" statement. 239 | # 240 | # This will of course break if any of these variables contains a newline or 241 | # an unmatched quote. 242 | # 243 | 244 | eval "set -- $( 245 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 246 | xargs -n1 | 247 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 248 | tr '\n' ' ' 249 | )" '"$@"' 250 | 251 | exec "$JAVACMD" "$@" 252 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | @rem SPDX-License-Identifier: Apache-2.0 17 | @rem 18 | 19 | @if "%DEBUG%"=="" @echo off 20 | @rem ########################################################################## 21 | @rem 22 | @rem Gradle startup script for Windows 23 | @rem 24 | @rem ########################################################################## 25 | 26 | @rem Set local scope for the variables with windows NT shell 27 | if "%OS%"=="Windows_NT" setlocal 28 | 29 | set DIRNAME=%~dp0 30 | if "%DIRNAME%"=="" set DIRNAME=. 31 | @rem This is normally unused 32 | set APP_BASE_NAME=%~n0 33 | set APP_HOME=%DIRNAME% 34 | 35 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 36 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 37 | 38 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 39 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 40 | 41 | @rem Find java.exe 42 | if defined JAVA_HOME goto findJavaFromJavaHome 43 | 44 | set JAVA_EXE=java.exe 45 | %JAVA_EXE% -version >NUL 2>&1 46 | if %ERRORLEVEL% equ 0 goto execute 47 | 48 | echo. 1>&2 49 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 50 | echo. 1>&2 51 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 52 | echo location of your Java installation. 1>&2 53 | 54 | goto fail 55 | 56 | :findJavaFromJavaHome 57 | set JAVA_HOME=%JAVA_HOME:"=% 58 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 59 | 60 | if exist "%JAVA_EXE%" goto execute 61 | 62 | echo. 1>&2 63 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 64 | echo. 1>&2 65 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 66 | echo location of your Java installation. 1>&2 67 | 68 | goto fail 69 | 70 | :execute 71 | @rem Setup the command line 72 | 73 | set CLASSPATH= 74 | 75 | 76 | @rem Execute Gradle 77 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* 78 | 79 | :end 80 | @rem End local scope for the variables with windows NT shell 81 | if %ERRORLEVEL% equ 0 goto mainEnd 82 | 83 | :fail 84 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 85 | rem the _cmd.exe /c_ return code! 86 | set EXIT_CODE=%ERRORLEVEL% 87 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 88 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 89 | exit /b %EXIT_CODE% 90 | 91 | :mainEnd 92 | if "%OS%"=="Windows_NT" endlocal 93 | 94 | :omega 95 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:base" 5 | ], 6 | "packageRules": [ 7 | { 8 | "matchUpdateTypes": ["minor", "patch", "pin", "digest"], 9 | "automerge": true 10 | } 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /settings.gradle.kts: -------------------------------------------------------------------------------- 1 | rootProject.name = "schema-registry-gitops" 2 | -------------------------------------------------------------------------------- /src/main/kotlin/dev/domnikl/schemaregistrygitops/CLI.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops 2 | 3 | import org.slf4j.Logger 4 | import org.slf4j.LoggerFactory 5 | import picocli.CommandLine 6 | import picocli.CommandLine.IVersionProvider 7 | import java.io.InputStreamReader 8 | import java.util.concurrent.Callable 9 | import ch.qos.logback.classic.Logger as LogbackClassicLogger 10 | 11 | @CommandLine.Command( 12 | name = "schema-registry-gitops", 13 | mixinStandardHelpOptions = true, 14 | versionProvider = CLI::class, 15 | description = ["Manages schema registries through Infrastructure as Code"] 16 | ) 17 | class CLI : Callable, IVersionProvider { 18 | @CommandLine.Spec 19 | lateinit var spec: CommandLine.Model.CommandSpec 20 | 21 | @CommandLine.Option( 22 | names = ["--properties"], 23 | description = ["a Java Properties file for client configuration (optional)"], 24 | scope = CommandLine.ScopeType.INHERIT 25 | ) 26 | var propertiesFilePath: String? = null 27 | 28 | @CommandLine.Option( 29 | names = ["-r", "--registry"], 30 | description = [ 31 | "schema registry endpoint, overwrites 'schema.registry.url' from properties, can also be a list of urls separated by comma" 32 | ], 33 | scope = CommandLine.ScopeType.INHERIT 34 | ) 35 | var baseUrl: String? = null 36 | 37 | @CommandLine.Option( 38 | names = ["-v", "--verbose"], 39 | description = ["enable verbose logging"], 40 | scope = CommandLine.ScopeType.INHERIT 41 | ) 42 | fun setVerbose(verbose: Boolean) { 43 | if (verbose) { 44 | val rootLogger = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME) as LogbackClassicLogger 45 | rootLogger.level = ch.qos.logback.classic.Level.DEBUG 46 | } 47 | } 48 | 49 | override fun call(): Int { 50 | spec.commandLine().usage(System.out) 51 | 52 | return 0 53 | } 54 | 55 | override fun getVersion(): Array { 56 | val version = InputStreamReader(object {}::class.java.classLoader.getResourceAsStream("version.txt")!!).readText() 57 | 58 | return arrayOf("schema-registry-gitops $version") 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/kotlin/dev/domnikl/schemaregistrygitops/Compatibility.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops 2 | 3 | enum class Compatibility { 4 | NONE, 5 | BACKWARD, 6 | FORWARD, 7 | FULL, 8 | BACKWARD_TRANSITIVE, 9 | FORWARD_TRANSITIVE, 10 | FULL_TRANSITIVE 11 | } 12 | -------------------------------------------------------------------------------- /src/main/kotlin/dev/domnikl/schemaregistrygitops/Configuration.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops 2 | 3 | import io.confluent.kafka.schemaregistry.avro.AvroSchemaProvider 4 | import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient 5 | import io.confluent.kafka.schemaregistry.client.rest.RestService 6 | import io.confluent.kafka.schemaregistry.json.JsonSchemaProvider 7 | import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchemaProvider 8 | import org.slf4j.LoggerFactory 9 | import java.io.File 10 | import java.util.Properties 11 | 12 | class Configuration(private val config: Map) { 13 | val baseUrl by lazy { config[SCHEMA_REGISTRY_URL]!! } 14 | 15 | init { 16 | require(config.containsKey(SCHEMA_REGISTRY_URL)) { "Either property $SCHEMA_REGISTRY_URL or --registry needs to be provided" } 17 | } 18 | 19 | fun toMap() = config 20 | 21 | fun client() = CachedSchemaRegistryClient( 22 | RestService(baseUrl), 23 | 100, 24 | listOf(AvroSchemaProvider(), ProtobufSchemaProvider(), JsonSchemaProvider()), 25 | config, 26 | null 27 | ) 28 | 29 | fun schemaRegistryClient() = SchemaRegistryClient(client()) 30 | 31 | companion object { 32 | private const val ENV_PREFIX = "SCHEMA_REGISTRY_GITOPS_" 33 | private const val SCHEMA_REGISTRY_URL = "schema.registry.url" 34 | 35 | fun from(properties: Properties) = Configuration( 36 | properties.map { it.key.toString() to it.value.toString() }.toMap() 37 | ) 38 | 39 | fun from(cli: CLI, env: Map? = null): Configuration { 40 | val properties = cli.propertiesFilePath?.let { load(it) } ?: Properties() 41 | 42 | properties.putAll(fromEnv(env ?: System.getenv())) 43 | 44 | // CLI-provided baseUrl overwrites properties file and env var 45 | cli.baseUrl?.let { properties.put(SCHEMA_REGISTRY_URL, it) } 46 | 47 | return from(properties) 48 | } 49 | 50 | private fun fromEnv(env: Map): Properties { 51 | val withNormalizedKeys = env.mapKeys { (key, _) -> 52 | if (key.startsWith(ENV_PREFIX)) { 53 | key.removePrefix(ENV_PREFIX).lowercase().replace('_', '.') 54 | } else { 55 | null 56 | } 57 | }.filterNot { it.key == null } 58 | 59 | return Properties().also { it.putAll(withNormalizedKeys) } 60 | } 61 | 62 | private fun load(path: String): Properties { 63 | LoggerFactory.getLogger(this::class.java).debug("Loading properties from '$path'") 64 | 65 | return Properties().also { it.load(File(path).reader()) } 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/kotlin/dev/domnikl/schemaregistrygitops/Main.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops 2 | 3 | import dev.domnikl.schemaregistrygitops.cli.Apply 4 | import dev.domnikl.schemaregistrygitops.cli.Dump 5 | import dev.domnikl.schemaregistrygitops.cli.Plan 6 | import picocli.CommandLine 7 | import kotlin.system.exitProcess 8 | 9 | fun main(args: Array) { 10 | val exitCode = CommandLine(CLI()) 11 | .addSubcommand("apply", Apply::class.java) 12 | .addSubcommand("dump", Dump::class.java) 13 | .addSubcommand("plan", Plan::class.java) 14 | .execute(*args) 15 | 16 | exitProcess(exitCode) 17 | } 18 | -------------------------------------------------------------------------------- /src/main/kotlin/dev/domnikl/schemaregistrygitops/ParsedSchema.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops 2 | 3 | import com.github.difflib.text.DiffRow 4 | import com.github.difflib.text.DiffRowGenerator 5 | import com.squareup.wire.schema.internal.parser.ProtoFileElement 6 | import io.confluent.kafka.schemaregistry.ParsedSchema 7 | import io.confluent.kafka.schemaregistry.avro.AvroSchema 8 | import io.confluent.kafka.schemaregistry.json.JsonSchema 9 | import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema 10 | import org.apache.avro.Schema 11 | 12 | private fun ParsedSchema.lines() = when (this) { 13 | is AvroSchema -> (this.rawSchema() as Schema).toString(true) 14 | is ProtobufSchema -> (this.rawSchema() as ProtoFileElement).toSchema() 15 | is JsonSchema -> this.toJsonNode().toPrettyString() 16 | else -> canonicalString() 17 | }.lines() 18 | 19 | fun ParsedSchema.diff(other: ParsedSchema): String { 20 | val generator = DiffRowGenerator.create() 21 | .showInlineDiffs(false) 22 | .inlineDiffByWord(false) 23 | .build() 24 | 25 | return generator.generateDiffRows(lines(), other.lines()).mapNotNull { 26 | when (it.tag) { 27 | DiffRow.Tag.INSERT -> "+ ${it.newLine}" 28 | DiffRow.Tag.DELETE -> "- ${it.oldLine}" 29 | DiffRow.Tag.CHANGE -> "- ${it.oldLine}\n+ ${it.newLine}" 30 | else -> " ${it.oldLine}" 31 | } 32 | }.joinToString("\n") 33 | } 34 | -------------------------------------------------------------------------------- /src/main/kotlin/dev/domnikl/schemaregistrygitops/SchemaParseException.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops 2 | 3 | class SchemaParseException(message: String) : RuntimeException(message) 4 | -------------------------------------------------------------------------------- /src/main/kotlin/dev/domnikl/schemaregistrygitops/SchemaRegistryClient.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops 2 | 3 | import io.confluent.kafka.schemaregistry.ParsedSchema 4 | import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient 5 | import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException 6 | 7 | class SchemaRegistryClient(private val client: CachedSchemaRegistryClient) { 8 | fun subjects(): List { 9 | return client.allSubjects.toList() 10 | } 11 | 12 | fun globalCompatibility(): Compatibility { 13 | return Compatibility.valueOf(client.getCompatibility("")) 14 | } 15 | 16 | fun normalize(): Boolean { 17 | return client.getConfig("").isNormalize ?: false 18 | } 19 | 20 | fun updateGlobalCompatibility(compatibility: Compatibility): Compatibility { 21 | return Compatibility.valueOf(client.updateCompatibility("", compatibility.toString())) 22 | } 23 | 24 | fun updateNormalize(normalize: Boolean) { 25 | val config = client.getConfig("") 26 | config.isNormalize = normalize 27 | client.updateConfig("", config) 28 | } 29 | 30 | fun compatibility(subject: String): Compatibility { 31 | return handleNotExisting { 32 | Compatibility.valueOf(client.getCompatibility(subject)) 33 | } ?: Compatibility.NONE 34 | } 35 | 36 | fun updateCompatibility(subject: Subject): Compatibility { 37 | return Compatibility.valueOf(client.updateCompatibility(subject.name, subject.compatibility.toString())) 38 | } 39 | 40 | fun testCompatibility(subject: Subject): List { 41 | return handleNotExisting { 42 | handleUnsupportedSchemaType { 43 | client.testCompatibilityVerbose(subject.name, subject.schema) 44 | } 45 | }?.toList() ?: emptyList() 46 | } 47 | 48 | fun getLatestSchema(subject: String): ParsedSchema { 49 | val metadata = client.getLatestSchemaMetadata(subject) 50 | 51 | return client.parseSchema(metadata.schemaType, metadata.schema, metadata.references).get() 52 | } 53 | 54 | fun create(subject: Subject): Int { 55 | return register(subject) 56 | } 57 | 58 | fun evolve(subject: Subject): Int { 59 | return register(subject) 60 | } 61 | 62 | private fun register(subject: Subject): Int { 63 | return handleUnsupportedSchemaType { 64 | client.register(subject.name, subject.schema) 65 | }!! 66 | } 67 | 68 | fun delete(subject: String) { 69 | handleNotExisting { 70 | client.deleteSubject(subject) 71 | } 72 | } 73 | 74 | fun version(subject: Subject): Int? { 75 | return handleNotExisting { 76 | client.getVersion(subject.name, subject.schema) 77 | } 78 | } 79 | 80 | private fun handleUnsupportedSchemaType(f: () -> V?): V? { 81 | return try { 82 | f() 83 | } catch (e: RestClientException) { 84 | throw when (e.errorCode) { 85 | ERROR_CODE_UNPROCESSABLE_ENTITY -> ServerVersionMismatchException( 86 | "Possible server version mismatch. " + 87 | "Note that types other than 'AVRO' are not supported for server versions prior to 5.5", 88 | e 89 | ) 90 | else -> e 91 | } 92 | } 93 | } 94 | 95 | private fun handleNotExisting(f: () -> V?): V? { 96 | return try { 97 | f() 98 | } catch (e: RestClientException) { 99 | when (e.errorCode) { 100 | ERROR_CODE_SUBJECT_NOT_FOUND -> null 101 | ERROR_CODE_VERSION_NOT_FOUND -> null 102 | ERROR_CODE_SCHEMA_NOT_FOUND -> null 103 | ERROR_CODE_SUBJECT_LEVEL_COMPATIBILITY_NOT_CONFIGURED -> null 104 | else -> throw e 105 | } 106 | } 107 | } 108 | 109 | companion object { 110 | private const val ERROR_CODE_SUBJECT_NOT_FOUND = 40401 111 | private const val ERROR_CODE_VERSION_NOT_FOUND = 40402 112 | private const val ERROR_CODE_SCHEMA_NOT_FOUND = 40403 113 | private const val ERROR_CODE_SUBJECT_LEVEL_COMPATIBILITY_NOT_CONFIGURED = 40408 114 | private const val ERROR_CODE_UNPROCESSABLE_ENTITY = 422 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/main/kotlin/dev/domnikl/schemaregistrygitops/ServerVersionMismatchException.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops 2 | 3 | class ServerVersionMismatchException(message: String, cause: Throwable) : RuntimeException(message, cause) 4 | -------------------------------------------------------------------------------- /src/main/kotlin/dev/domnikl/schemaregistrygitops/State.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops 2 | 3 | data class State(val compatibility: Compatibility?, val normalize: Boolean? = false, val subjects: List) { 4 | init { 5 | val duplicates = subjects.duplicatesBy { it.name } 6 | 7 | require(duplicates.isEmpty()) { 8 | "State in YAML configuration is invalid: duplicated subject(s) '${duplicates.joinToString("', '")}' found" 9 | } 10 | } 11 | 12 | fun merge(other: State): State { 13 | val a = subjects.associateBy { it.name } 14 | val b = other.subjects.associateBy { it.name } 15 | 16 | return State( 17 | other.compatibility ?: compatibility, 18 | other.normalize ?: normalize, 19 | (a + b).map { it.value } 20 | ) 21 | } 22 | } 23 | 24 | private fun List.duplicatesBy(f: (T) -> K): List { 25 | return groupBy(f) 26 | .filter { it.value.size > 1 } 27 | .map { it.key } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/kotlin/dev/domnikl/schemaregistrygitops/Subject.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops 2 | 3 | import io.confluent.kafka.schemaregistry.ParsedSchema 4 | import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference 5 | 6 | data class Subject( 7 | val name: String, 8 | val compatibility: Compatibility?, 9 | val schema: ParsedSchema, 10 | val references: List = emptyList() 11 | ) 12 | -------------------------------------------------------------------------------- /src/main/kotlin/dev/domnikl/schemaregistrygitops/cli/Apply.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops.cli 2 | 3 | import dev.domnikl.schemaregistrygitops.CLI 4 | import dev.domnikl.schemaregistrygitops.Configuration 5 | import dev.domnikl.schemaregistrygitops.state.Applier 6 | import dev.domnikl.schemaregistrygitops.state.Diffing 7 | import dev.domnikl.schemaregistrygitops.state.Persistence 8 | import dev.domnikl.schemaregistrygitops.state.Result 9 | import org.slf4j.Logger 10 | import org.slf4j.LoggerFactory 11 | import picocli.CommandLine 12 | import java.io.File 13 | import java.util.concurrent.Callable 14 | 15 | @CommandLine.Command( 16 | name = "apply", 17 | description = ["applies the state to the given schema registry"] 18 | ) 19 | class Apply( 20 | private val configuration: Configuration? = null, 21 | private val persistence: Persistence? = null, 22 | private val diffing: Diffing? = null, 23 | private val applier: Applier? = null, 24 | logger: Logger? = null 25 | ) : Callable { 26 | private val logger = logger ?: LoggerFactory.getLogger(Apply::class.java) 27 | 28 | @CommandLine.ParentCommand 29 | private lateinit var cli: CLI 30 | 31 | @CommandLine.Parameters(description = ["path to input YAML files"]) 32 | private lateinit var inputFiles: List 33 | 34 | @CommandLine.Option( 35 | names = ["-d", "--enable-deletes"], 36 | description = ["allow deleting subjects not listed in input YAML"] 37 | ) 38 | private var enableDeletes: Boolean = false 39 | 40 | override fun call(): Int { 41 | return try { 42 | val configuration = configuration ?: Configuration.from(cli) 43 | val persistence = persistence ?: Persistence(configuration.client(), logger) 44 | val diffing = diffing ?: Diffing(configuration.schemaRegistryClient()) 45 | val applier = applier ?: Applier(configuration.schemaRegistryClient(), logger) 46 | 47 | val state = persistence.load(inputFiles.first().absoluteFile.parentFile, inputFiles.map { it.absoluteFile }) 48 | val diff = diffing.diff(state, enableDeletes) 49 | 50 | if (applier.apply(diff) == Result.SUCCESS) { 51 | inputFiles.forEach { 52 | logger.info("[SUCCESS] Applied state from $it to ${configuration.baseUrl}") 53 | } 54 | 55 | 0 56 | } else { 57 | 1 58 | } 59 | } catch (e: Exception) { 60 | logger.error(e.toString()) 61 | 62 | 1 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/kotlin/dev/domnikl/schemaregistrygitops/cli/Dump.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops.cli 2 | 3 | import dev.domnikl.schemaregistrygitops.CLI 4 | import dev.domnikl.schemaregistrygitops.Configuration 5 | import dev.domnikl.schemaregistrygitops.state.Dumper 6 | import dev.domnikl.schemaregistrygitops.state.Persistence 7 | import org.slf4j.LoggerFactory 8 | import picocli.CommandLine 9 | import java.io.BufferedOutputStream 10 | import java.io.File 11 | import java.io.FileOutputStream 12 | import java.util.concurrent.Callable 13 | 14 | @CommandLine.Command( 15 | name = "dump", 16 | description = ["prints the current state"] 17 | ) 18 | class Dump( 19 | private val configuration: Configuration? = null, 20 | private val persistence: Persistence? = null, 21 | private val dumper: Dumper? = null 22 | ) : Callable { 23 | private val logger = LoggerFactory.getLogger(Dump::class.java) 24 | 25 | @CommandLine.ParentCommand 26 | private lateinit var cli: CLI 27 | 28 | @CommandLine.Parameters( 29 | description = ["optional path to output YAML file, default is \"-\", which prints to STDOUT"], 30 | defaultValue = STDOUT_FILE 31 | ) 32 | private lateinit var outputFile: String 33 | 34 | private val outputStream by lazy { 35 | when (outputFile) { 36 | STDOUT_FILE -> System.out 37 | else -> BufferedOutputStream(FileOutputStream(File(outputFile))) 38 | } 39 | } 40 | 41 | override fun call(): Int { 42 | val configuration = configuration ?: Configuration.from(cli) 43 | val persistence = persistence ?: Persistence(configuration.client(), logger) 44 | val dumper = dumper ?: Dumper(configuration.schemaRegistryClient()) 45 | val state = dumper.dump() 46 | 47 | persistence.save(state, outputStream) 48 | 49 | return 0 50 | } 51 | 52 | companion object { 53 | private const val STDOUT_FILE = "-" 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/kotlin/dev/domnikl/schemaregistrygitops/cli/Plan.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops.cli 2 | 3 | import dev.domnikl.schemaregistrygitops.CLI 4 | import dev.domnikl.schemaregistrygitops.Configuration 5 | import dev.domnikl.schemaregistrygitops.diff 6 | import dev.domnikl.schemaregistrygitops.state.Diffing 7 | import dev.domnikl.schemaregistrygitops.state.Persistence 8 | import org.slf4j.Logger 9 | import org.slf4j.LoggerFactory 10 | import picocli.CommandLine.Command 11 | import picocli.CommandLine.Option 12 | import picocli.CommandLine.Parameters 13 | import picocli.CommandLine.ParentCommand 14 | import java.io.File 15 | import java.util.concurrent.Callable 16 | 17 | @Command( 18 | name = "plan", 19 | description = ["validate and plan schema changes, can be used to see all pending changes"], 20 | mixinStandardHelpOptions = true 21 | ) 22 | class Plan( 23 | private val configuration: Configuration? = null, 24 | private val persistence: Persistence? = null, 25 | private val diffing: Diffing? = null, 26 | logger: Logger? = null 27 | ) : Callable { 28 | private val logger = logger ?: LoggerFactory.getLogger(Plan::class.java) 29 | 30 | @ParentCommand 31 | private lateinit var cli: CLI 32 | 33 | @Parameters(description = ["path to input YAML files"]) 34 | private lateinit var inputFiles: List 35 | 36 | @Option( 37 | names = ["-d", "--enable-deletes"], 38 | description = ["allow deleting subjects not listed in input YAML"] 39 | ) 40 | private var enableDeletes: Boolean = false 41 | 42 | override fun call(): Int { 43 | try { 44 | val configuration = configuration ?: Configuration.from(cli) 45 | val persistence = persistence ?: Persistence(configuration.client(), logger) 46 | val diffing = diffing ?: Diffing(configuration.schemaRegistryClient()) 47 | 48 | val state = persistence.load(inputFiles.first().absoluteFile.parentFile, inputFiles.map { it.absoluteFile }) 49 | val result = diffing.diff(state, enableDeletes) 50 | 51 | if (!result.isEmpty()) { 52 | logger.info("The following changes would be applied:") 53 | logger.info("") 54 | } 55 | 56 | result.compatibility?.let { 57 | logger.info("[GLOBAL]") 58 | logger.info(" ~ compatibility ${it.before} -> ${it.after}") 59 | logger.info("") 60 | } 61 | 62 | if (enableDeletes) { 63 | result.deleted.forEach { 64 | logger.info("[SUBJECT] $it") 65 | logger.info(" - delete") 66 | logger.info("") 67 | } 68 | } 69 | 70 | result.added.forEach { 71 | logger.info("[SUBJECT] ${it.name}") 72 | logger.info(" + register") 73 | 74 | it.compatibility?.let { c -> 75 | logger.info(" + compatibility $c") 76 | } 77 | 78 | logger.info(" + schema ${it.schema}") 79 | logger.info("") 80 | } 81 | 82 | result.modified.forEach { 83 | logger.info("[SUBJECT] ${it.subject.name}") 84 | 85 | it.remoteCompatibility?.let { c -> 86 | logger.info(" ~ compatibility ${c.before} -> ${c.after}") 87 | } 88 | 89 | it.remoteSchema?.let { s -> 90 | logger.info(" ~ schema ${s.before.diff(s.after)}") 91 | } 92 | 93 | logger.info("") 94 | } 95 | 96 | if (result.incompatible.isNotEmpty()) { 97 | result.incompatible.forEach { 98 | logger.error( 99 | "[ERROR] The following schema is incompatible with an earlier version: '${ it.subject.name }': '" + 100 | it.messages.joinToString(",") + 101 | "'" 102 | ) 103 | } 104 | 105 | return 1 106 | } 107 | 108 | if (result.isEmpty()) { 109 | logger.info("[SUCCESS] There are no necessary changes; the actual state matches the desired state.") 110 | } else { 111 | logger.info("[SUCCESS] All changes are compatible and can be applied.") 112 | } 113 | 114 | return 0 115 | } catch (e: Exception) { 116 | logger.error(e.toString()) 117 | 118 | return 2 119 | } 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/main/kotlin/dev/domnikl/schemaregistrygitops/state/Applier.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops.state 2 | 3 | import dev.domnikl.schemaregistrygitops.Compatibility 4 | import dev.domnikl.schemaregistrygitops.SchemaRegistryClient 5 | import dev.domnikl.schemaregistrygitops.Subject 6 | import dev.domnikl.schemaregistrygitops.diff 7 | import io.confluent.kafka.schemaregistry.ParsedSchema 8 | import org.slf4j.Logger 9 | 10 | enum class Result { 11 | ERROR, 12 | SUCCESS 13 | } 14 | 15 | class Applier( 16 | private val client: SchemaRegistryClient, 17 | private val logger: Logger 18 | ) { 19 | fun apply(diff: Diffing.Result): Result { 20 | if (diff.incompatible.isNotEmpty()) { 21 | diff.incompatible.forEach { 22 | logger.error( 23 | "[ERROR] The following schema is incompatible with an earlier version: '${ it.subject.name }': '" + 24 | it.messages.joinToString(",") + 25 | "'" 26 | ) 27 | } 28 | 29 | return Result.ERROR 30 | } 31 | 32 | diff.compatibility?.let { 33 | updateGlobalCompatibility(diff.compatibility) 34 | 35 | logger.info("[GLOBAL]") 36 | logger.info(" ~ compatibility ${diff.compatibility.before} -> ${diff.compatibility.after}") 37 | logger.info("") 38 | } 39 | 40 | diff.normalize?.let { 41 | updateNormalize(diff.normalize) 42 | logger.info("[GLOBAL]") 43 | logger.info(" ~ normalize ${diff.normalize.before} -> ${diff.normalize.after}") 44 | logger.info("") 45 | } 46 | 47 | diff.deleted.forEach { delete(it) } 48 | diff.added.forEach { register(it) } 49 | 50 | diff.modified.forEach { change -> 51 | logger.info("[SUBJECT] ${change.subject.name}") 52 | 53 | change.remoteCompatibility?.let { 54 | updateCompatibility(change.subject) 55 | logger.info(" ~ compatibility ${change.remoteCompatibility.before} -> ${change.remoteCompatibility.after}") 56 | } 57 | 58 | change.remoteSchema?.let { 59 | evolve(change.subject, it) 60 | } 61 | } 62 | 63 | return Result.SUCCESS 64 | } 65 | 66 | private fun delete(subject: String) { 67 | client.delete(subject) 68 | 69 | logger.info("[SUBJECT] $subject") 70 | logger.info(" - deleted") 71 | logger.info("") 72 | } 73 | 74 | private fun register(subject: Subject) { 75 | val versionId = client.create(subject) 76 | 77 | logger.info("[SUBJECT] ${subject.name}") 78 | logger.info(" + registered (version $versionId)") 79 | 80 | subject.compatibility?.let { 81 | updateCompatibility(subject) 82 | logger.info(" + compatibility ${subject.compatibility}") 83 | } 84 | 85 | logger.info("") 86 | } 87 | 88 | private fun evolve(subject: Subject, change: Diffing.Change) { 89 | val versionId = client.evolve(subject) 90 | 91 | logger.info(" ~ evolved (version $versionId)") 92 | logger.info(" ~ schema ${change.before.diff(change.after)}") 93 | logger.info("") 94 | } 95 | 96 | private fun updateCompatibility(subject: Subject) { 97 | client.updateCompatibility(subject) 98 | } 99 | 100 | private fun updateGlobalCompatibility(change: Diffing.Change) { 101 | client.updateGlobalCompatibility(change.after) 102 | } 103 | 104 | private fun updateNormalize(change: Diffing.Change) { 105 | client.updateNormalize(change.after) 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/main/kotlin/dev/domnikl/schemaregistrygitops/state/Diffing.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops.state 2 | 3 | import dev.domnikl.schemaregistrygitops.Compatibility 4 | import dev.domnikl.schemaregistrygitops.SchemaRegistryClient 5 | import dev.domnikl.schemaregistrygitops.State 6 | import dev.domnikl.schemaregistrygitops.Subject 7 | import io.confluent.kafka.schemaregistry.ParsedSchema 8 | 9 | class Diffing(private val client: SchemaRegistryClient) { 10 | fun diff(state: State, enableDeletes: Boolean = false): Result { 11 | val remoteSubjects = client.subjects() 12 | 13 | val checkResults = state.subjects.map { 14 | CompatibilityTestResult(it, client.testCompatibility(it)) 15 | } 16 | 17 | val (compatible, incompatible) = checkResults.partition { it.messages.isEmpty() } 18 | 19 | val deleted = gatherDeletes(enableDeletes, remoteSubjects, state) 20 | val compatibleSubjects = compatible.map { it.subject } 21 | val added = compatibleSubjects.filterNot { remoteSubjects.contains(it.name) } 22 | val modified = compatibleSubjects.filter { !deleted.contains(it.name) && !added.contains(it) } 23 | 24 | return Result( 25 | gatherCompatibilityChange(client.globalCompatibility(), state), 26 | gatherNormalizeChange(client.normalize(), state), 27 | incompatible, 28 | added, 29 | gatherChanges(modified), 30 | deleted 31 | ) 32 | } 33 | 34 | private fun gatherCompatibilityChange(globalCompatibility: Compatibility, state: State): Change? { 35 | if (globalCompatibility == state.compatibility || state.compatibility == null) { 36 | return null 37 | } 38 | 39 | return Change(globalCompatibility, state.compatibility) 40 | } 41 | 42 | private fun gatherNormalizeChange(normalize: Boolean, state: State): Change? { 43 | if (normalize == state.normalize || state.normalize == null) { 44 | return null 45 | } 46 | return Change(normalize, state.normalize) 47 | } 48 | 49 | private fun gatherDeletes(enableDeletes: Boolean, remoteSubjects: List, state: State): List { 50 | if (!enableDeletes) return emptyList() 51 | 52 | val localSubjects = state.subjects.map { it.name } 53 | 54 | return remoteSubjects.filterNot { localSubjects.contains(it) } 55 | } 56 | 57 | private fun gatherChanges(modified: List) = modified.mapNotNull { 58 | val changedCompatibility = gatherCompatibilityChange(it) 59 | val changedSchema = gatherSchemaChange(it) 60 | 61 | if (changedSchema == null && changedCompatibility == null) { 62 | return@mapNotNull null 63 | } 64 | 65 | Changes(it, changedCompatibility, changedSchema) 66 | } 67 | 68 | private fun gatherCompatibilityChange(subject: Subject): Change? { 69 | val remoteCompatibility = client.compatibility(subject.name) 70 | 71 | if (remoteCompatibility == subject.compatibility || subject.compatibility == null) { 72 | return null 73 | } 74 | 75 | return Change(remoteCompatibility, subject.compatibility) 76 | } 77 | 78 | private fun gatherSchemaChange(subject: Subject): Change? { 79 | val remoteSchema = client.getLatestSchema(subject.name) 80 | 81 | if (remoteSchema.canonicalString() != subject.schema.canonicalString() || client.version(subject) == null) { 82 | return Change(remoteSchema, subject.schema) 83 | } 84 | 85 | return null 86 | } 87 | 88 | data class Result( 89 | val compatibility: Change? = null, 90 | val normalize: Change? = null, 91 | val incompatible: List = emptyList(), 92 | val added: List = emptyList(), 93 | val modified: List = emptyList(), 94 | val deleted: List = emptyList() 95 | ) { 96 | fun isEmpty() = compatibility == null && 97 | incompatible.isEmpty() && 98 | added.isEmpty() && 99 | modified.isEmpty() && 100 | deleted.isEmpty() 101 | } 102 | 103 | data class CompatibilityTestResult( 104 | val subject: Subject, 105 | val messages: List 106 | ) 107 | 108 | data class Changes( 109 | val subject: Subject, 110 | val remoteCompatibility: Change?, 111 | val remoteSchema: Change? 112 | ) 113 | 114 | data class Change(val before: T, val after: T) 115 | } 116 | -------------------------------------------------------------------------------- /src/main/kotlin/dev/domnikl/schemaregistrygitops/state/Dumper.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops.state 2 | 3 | import dev.domnikl.schemaregistrygitops.SchemaRegistryClient 4 | import dev.domnikl.schemaregistrygitops.State 5 | import dev.domnikl.schemaregistrygitops.Subject 6 | 7 | class Dumper(private val client: SchemaRegistryClient) { 8 | fun dump() = State( 9 | client.globalCompatibility(), 10 | client.normalize(), 11 | client.subjects().map { subject -> 12 | val schema = client.getLatestSchema(subject) 13 | Subject( 14 | subject, 15 | client.compatibility(subject), 16 | schema, 17 | schema.references() 18 | ) 19 | } 20 | ) 21 | } 22 | -------------------------------------------------------------------------------- /src/main/kotlin/dev/domnikl/schemaregistrygitops/state/Persistence.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops.state 2 | 3 | import com.fasterxml.jackson.databind.DeserializationFeature 4 | import com.fasterxml.jackson.databind.ObjectMapper 5 | import com.fasterxml.jackson.dataformat.yaml.YAMLFactory 6 | import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator 7 | import com.fasterxml.jackson.module.kotlin.registerKotlinModule 8 | import dev.domnikl.schemaregistrygitops.Compatibility 9 | import dev.domnikl.schemaregistrygitops.SchemaParseException 10 | import dev.domnikl.schemaregistrygitops.State 11 | import dev.domnikl.schemaregistrygitops.Subject 12 | import io.confluent.kafka.schemaregistry.ParsedSchema 13 | import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient 14 | import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference 15 | import org.slf4j.Logger 16 | import java.io.File 17 | import java.io.FileNotFoundException 18 | import java.io.OutputStream 19 | import java.nio.file.Files 20 | import java.util.Optional 21 | 22 | class Persistence( 23 | private val schemaRegistryClient: CachedSchemaRegistryClient, 24 | private val logger: Logger 25 | ) { 26 | private val yamlFactory = YAMLFactory() 27 | .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) 28 | .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES) 29 | .enable(YAMLGenerator.Feature.LITERAL_BLOCK_STYLE) 30 | 31 | private val mapper = ObjectMapper(yamlFactory) 32 | .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) 33 | .registerKotlinModule() 34 | 35 | fun load(basePath: File, files: List): State { 36 | require(files.isNotEmpty()) 37 | 38 | val states = files.map { file -> 39 | logger.debug("Loading state file ${file.absolutePath}, referenced schemas from ${basePath.absolutePath}") 40 | 41 | if (!Files.exists(file.toPath())) { 42 | throw FileNotFoundException("Could not find ${file.toPath()}") 43 | } 44 | 45 | require(file.length() > 0) 46 | 47 | val yaml = mapper.readValue(file, Yaml::class.java) 48 | 49 | State( 50 | yaml.compatibility?.let { Compatibility.valueOf(it) }, 51 | yaml.normalize, 52 | yaml.subjects?.map { 53 | Subject( 54 | it.name, 55 | it.compatibility?.let { c -> Compatibility.valueOf(c) }, 56 | it.parseSchema(basePath, schemaRegistryClient), 57 | it.references.map { yamlSubjectReference: YamlSubjectReference -> 58 | SchemaReference(yamlSubjectReference.name, yamlSubjectReference.subject, yamlSubjectReference.version) 59 | } 60 | ) 61 | } ?: emptyList() 62 | ) 63 | } 64 | 65 | return states.fold(states.first()) { a: State, b: State -> a.merge(b) } 66 | } 67 | 68 | fun save(state: State, outputStream: OutputStream) { 69 | val yaml = Yaml( 70 | (state.compatibility ?: Compatibility.NONE).toString(), 71 | // default by confluent schema registry 72 | state.normalize ?: false, 73 | state.subjects.map { 74 | YamlSubject( 75 | it.name, 76 | null, 77 | it.schema.schemaType(), 78 | it.schema.canonicalString(), 79 | it.compatibility?.toString(), 80 | it.references.map { subjectReference: SchemaReference -> 81 | YamlSubjectReference(subjectReference.name, subjectReference.subject, subjectReference.version) 82 | } 83 | ) 84 | } 85 | ) 86 | 87 | mapper.writeValue(outputStream, yaml) 88 | } 89 | 90 | data class Yaml(val compatibility: String?, val normalize: Boolean?, val subjects: List?) 91 | data class YamlSubjectReference(val name: String, val subject: String, val version: Int) 92 | data class YamlSubject( 93 | val name: String, 94 | val file: String?, 95 | val type: String?, 96 | val schema: String?, 97 | val compatibility: String?, 98 | val references: List = emptyList() 99 | ) { 100 | fun parseSchema(basePath: File, schemaRegistryClient: CachedSchemaRegistryClient): ParsedSchema { 101 | val t = type ?: "AVRO" 102 | 103 | val schemaReferences = references.map { 104 | SchemaReference(it.name, it.subject, it.version) 105 | } 106 | 107 | val optional = when { 108 | file != null -> doParseSchema(schemaRegistryClient, t, File("$basePath/$file").readText(), schemaReferences) 109 | schema != null -> doParseSchema(schemaRegistryClient, t, schema, schemaReferences) 110 | else -> throw IllegalArgumentException("Either schema or file must be set") 111 | } 112 | 113 | if (optional == null || optional.isEmpty) { 114 | throw SchemaParseException("Could not parse $t schema for subject '$name'") 115 | } 116 | 117 | return optional.get() 118 | } 119 | 120 | private fun doParseSchema( 121 | client: CachedSchemaRegistryClient, 122 | t: String, 123 | schemaString: String, 124 | schemaReferences: List? 125 | ): Optional? { 126 | return client.parseSchema(t, schemaString, schemaReferences ?: emptyList()) 127 | } 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{YYYY-MM-dd HH:mm:ss.SSS} %level %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /src/main/resources/version.txt: -------------------------------------------------------------------------------- 1 | # this file is being replaced by Gradle build 2 | 0.0.0 3 | -------------------------------------------------------------------------------- /src/test/kotlin/dev/domnikl/schemaregistrygitops/CLITest.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops 2 | 3 | import ch.qos.logback.classic.Level 4 | import org.junit.Assert.assertEquals 5 | import org.junit.Test 6 | import org.junit.jupiter.api.BeforeEach 7 | import org.slf4j.Logger 8 | import org.slf4j.LoggerFactory 9 | import picocli.CommandLine 10 | import java.io.ByteArrayOutputStream 11 | import java.io.PrintWriter 12 | import ch.qos.logback.classic.Logger as LogbackClassicLogger 13 | 14 | class CLITest { 15 | private val out = ByteArrayOutputStream() 16 | private val cli = CLI() 17 | private val commandLine = CommandLine(cli).also { 18 | it.out = PrintWriter(out) 19 | } 20 | 21 | @BeforeEach 22 | fun setupStreams() { 23 | out.reset() 24 | } 25 | 26 | @Test 27 | fun `can print version information`() { 28 | val exitCode = commandLine.execute("--version") 29 | 30 | assertEquals("schema-registry-gitops test", out.toString().trim()) 31 | assertEquals(0, exitCode) 32 | } 33 | 34 | @Test 35 | fun `can enable verbose logging`() { 36 | val logger = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME) as LogbackClassicLogger 37 | assertEquals(Level.INFO, logger.level) 38 | 39 | val exitCode = commandLine.execute("--verbose") 40 | 41 | assertEquals(Level.DEBUG, logger.level) 42 | assertEquals(0, exitCode) 43 | 44 | logger.level = Level.INFO 45 | } 46 | 47 | @Test 48 | fun `can get version`() { 49 | assertEquals("schema-registry-gitops test", cli.version.joinToString("")) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/test/kotlin/dev/domnikl/schemaregistrygitops/ConfigurationTest.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops 2 | 3 | import io.mockk.every 4 | import io.mockk.mockk 5 | import org.junit.Assert.assertEquals 6 | import org.junit.Test 7 | import org.junit.jupiter.api.assertThrows 8 | import java.util.Properties 9 | 10 | class ConfigurationTest { 11 | @Test 12 | fun `can load properties`() { 13 | val properties = Properties().also { it["schema.registry.url"] = "foo" } 14 | val config = Configuration.from(properties) 15 | 16 | assertEquals(mapOf("schema.registry.url" to "foo"), config.toMap()) 17 | } 18 | 19 | @Test 20 | fun `can load properties with integer values`() { 21 | val properties = Properties().also { it["schema.registry.url"] = 1 } 22 | val config = Configuration.from(properties) 23 | 24 | assertEquals(mapOf("schema.registry.url" to "1"), config.toMap()) 25 | } 26 | 27 | @Test 28 | fun `throws exception when baseUrl has not been set`() { 29 | assertThrows { 30 | Configuration.from(Properties()) 31 | } 32 | } 33 | 34 | @Test 35 | fun `can load from cli`() { 36 | val cli = mockk() 37 | 38 | every { cli.propertiesFilePath } returns fromResources("client.properties").path 39 | every { cli.baseUrl } returns null 40 | 41 | val configuration = Configuration.from(cli) 42 | 43 | assertEquals(mapOf("schema.registry.url" to "foo"), configuration.toMap()) 44 | } 45 | 46 | @Test 47 | fun `can load from cli and overwrite baseUrl from CLI`() { 48 | val cli = mockk() 49 | 50 | every { cli.propertiesFilePath } returns fromResources("client.properties").path 51 | every { cli.baseUrl } returns "bar" 52 | 53 | val configuration = Configuration.from(cli) 54 | 55 | assertEquals(mapOf("schema.registry.url" to "bar"), configuration.toMap()) 56 | } 57 | 58 | @Test 59 | fun `allow not using properties at all`() { 60 | val cli = mockk() 61 | 62 | every { cli.propertiesFilePath } returns null 63 | every { cli.baseUrl } returns "foo" 64 | 65 | val configuration = Configuration.from(cli) 66 | 67 | assertEquals(mapOf("schema.registry.url" to "foo"), configuration.toMap()) 68 | } 69 | 70 | @Test 71 | fun `throws exception if neither properties nor baseUrl are being set`() { 72 | val cli = mockk() 73 | 74 | every { cli.propertiesFilePath } returns null 75 | every { cli.baseUrl } returns null 76 | 77 | assertThrows { 78 | Configuration.from(cli) 79 | } 80 | } 81 | 82 | @Test 83 | fun `configuration from env will overwrite properties file and CLI arguments`() { 84 | val cli = mockk() 85 | 86 | every { cli.propertiesFilePath } returns fromResources("client.properties").path 87 | every { cli.baseUrl } returns null 88 | 89 | val configuration = Configuration.from( 90 | cli, 91 | mapOf( 92 | "SCHEMA_REGISTRY_GITOPS_SCHEMA_REGISTRY_URL" to "foobar", 93 | "SCHEMA_FOO" to "should-be-ignored" 94 | ) 95 | ) 96 | 97 | assertEquals(mapOf("schema.registry.url" to "foobar"), configuration.toMap()) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/test/kotlin/dev/domnikl/schemaregistrygitops/ParsedSchemaTest.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops 2 | 3 | import org.junit.Assert.assertEquals 4 | import org.junit.Test 5 | 6 | class ParsedSchemaTest { 7 | @Test 8 | fun `can get deltas for AVRO`() { 9 | val a = avroFromResources("schemas/deltaA.avsc") 10 | val b = avroFromResources("schemas/deltaB.avsc") 11 | 12 | val diff = a.diff(b) 13 | 14 | assertEquals(stringFromResources("schemas/avsc.diff"), diff) 15 | } 16 | 17 | @Test 18 | fun `can get deltas for Protobuf`() { 19 | val a = protoFromResources("schemas/deltaA.proto") 20 | val b = protoFromResources("schemas/deltaB.proto") 21 | 22 | val diff = a.diff(b) 23 | 24 | assertEquals(stringFromResources("schemas/proto.diff"), diff) 25 | } 26 | 27 | @Test 28 | fun `can get deltas for JSON Schema`() { 29 | val a = jsonFromResources("schemas/deltaA.json") 30 | val b = jsonFromResources("schemas/deltaB.json") 31 | 32 | val diff = a.diff(b) 33 | 34 | assertEquals(stringFromResources("schemas/json.diff"), diff) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/test/kotlin/dev/domnikl/schemaregistrygitops/SchemaRegistryClientTest.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops 2 | 3 | import io.confluent.kafka.schemaregistry.ParsedSchema 4 | import io.confluent.kafka.schemaregistry.client.SchemaMetadata 5 | import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException 6 | import io.mockk.every 7 | import io.mockk.mockk 8 | import io.mockk.verify 9 | import org.junit.Assert.assertEquals 10 | import org.junit.Assert.assertNull 11 | import org.junit.Test 12 | import org.junit.jupiter.api.assertThrows 13 | import java.util.Optional 14 | import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient as WrappedSchemaRegistryClient 15 | 16 | class SchemaRegistryClientTest { 17 | private val client = mockk() 18 | private val wrapper = SchemaRegistryClient(client) 19 | 20 | @Test 21 | fun `can get subjects`() { 22 | every { client.allSubjects } returns mutableListOf("foo", "bar") 23 | 24 | assertEquals(listOf("foo", "bar"), wrapper.subjects()) 25 | } 26 | 27 | @Test 28 | fun `can get global compatibility`() { 29 | every { client.getCompatibility("") } returns "BACKWARD" 30 | 31 | assertEquals(Compatibility.BACKWARD, wrapper.globalCompatibility()) 32 | } 33 | 34 | @Test 35 | fun `can update global compatibility`() { 36 | every { client.updateCompatibility("", "FULL") } returns "FULL" 37 | 38 | assertEquals( 39 | Compatibility.FULL, 40 | wrapper.updateGlobalCompatibility(Compatibility.FULL) 41 | ) 42 | } 43 | 44 | @Test 45 | fun `can get compatibility`() { 46 | every { client.getCompatibility("foo") } returns "FORWARD" 47 | 48 | assertEquals( 49 | Compatibility.FORWARD, 50 | wrapper.compatibility("foo") 51 | ) 52 | } 53 | 54 | @Test 55 | fun `will return Compatibility None if not set explicitly`() { 56 | every { client.getCompatibility("foo") } throws RestClientException("", 404, 40408) 57 | 58 | assertEquals( 59 | Compatibility.NONE, 60 | wrapper.compatibility("foo") 61 | ) 62 | } 63 | 64 | @Test 65 | fun `can get compatibility if none is set`() { 66 | every { client.getCompatibility("foo") } throws RestClientException("", 404, 40403) 67 | 68 | assertEquals( 69 | Compatibility.NONE, 70 | wrapper.compatibility("foo") 71 | ) 72 | } 73 | 74 | @Test 75 | fun `rethrows RestClientExceptionIfAnotherErrorOccurred`() { 76 | every { client.getCompatibility("foo") } throws RestClientException("", 404, 50001) 77 | 78 | assertThrows { 79 | wrapper.compatibility("foo") 80 | } 81 | } 82 | 83 | @Test 84 | fun `can update compatibility`() { 85 | val subject = Subject("foo", Compatibility.FORWARD, mockk()) 86 | 87 | every { client.updateCompatibility("foo", "FORWARD") } returns "NONE" 88 | 89 | assertEquals( 90 | Compatibility.NONE, 91 | wrapper.updateCompatibility(subject) 92 | ) 93 | } 94 | 95 | @Test 96 | fun `can test compatibility`() { 97 | val schema = mockk() 98 | val subject = Subject("foo", Compatibility.FORWARD, schema) 99 | 100 | every { client.testCompatibilityVerbose("foo", schema) } returns listOf("hello", "world") 101 | 102 | assertEquals(listOf("hello", "world"), wrapper.testCompatibility(subject)) 103 | } 104 | 105 | @Test 106 | fun `test compatibility returns empty list if subject is new`() { 107 | val schema = mockk() 108 | val subject = Subject("foo", null, schema) 109 | 110 | every { client.testCompatibilityVerbose("foo", schema) } throws RestClientException("", 404, 40401) 111 | 112 | assertEquals(emptyList(), wrapper.testCompatibility(subject)) 113 | } 114 | 115 | @Test 116 | fun `test compatibility returns empty list if version is new`() { 117 | val schema = mockk() 118 | val subject = Subject("foo", null, schema) 119 | 120 | every { client.testCompatibilityVerbose("foo", schema) } throws RestClientException("", 404, 40402) 121 | 122 | assertEquals(emptyList(), wrapper.testCompatibility(subject)) 123 | } 124 | 125 | @Test 126 | fun `test compatibility returns empty list if schema is new`() { 127 | val schema = mockk() 128 | val subject = Subject("foo", null, schema) 129 | 130 | every { client.testCompatibilityVerbose("foo", schema) } throws RestClientException("", 404, 40403) 131 | 132 | assertEquals(emptyList(), wrapper.testCompatibility(subject)) 133 | } 134 | 135 | @Test 136 | fun `test compatibility returns messages if client returns them`() { 137 | val schema = mockk() 138 | val subject = Subject("foo", null, schema) 139 | 140 | every { client.testCompatibilityVerbose("foo", schema) } returns emptyList() 141 | 142 | assertEquals(emptyList(), wrapper.testCompatibility(subject)) 143 | } 144 | 145 | @Test 146 | fun `test compatibility verbose throws exception if schema type is not supported`() { 147 | val schema = mockk() 148 | val subject = Subject("foo", null, schema) 149 | 150 | every { client.testCompatibilityVerbose("foo", schema) } throws RestClientException("", 422, 422) 151 | 152 | assertThrows { 153 | wrapper.testCompatibility(subject) 154 | } 155 | } 156 | 157 | @Test 158 | fun `can get latest schema`() { 159 | val schema = avroFromResources("schemas/key.avsc") 160 | val schemaMetadata = SchemaMetadata(42, 1, "AVRO", emptyList(), schema.toString()) 161 | val optionalSchema: Optional = Optional.of(schema) 162 | 163 | every { client.parseSchema("AVRO", schema.toString(), emptyList()) } returns optionalSchema 164 | every { client.getLatestSchemaMetadata("foo") } returns schemaMetadata 165 | 166 | assertEquals(schema, wrapper.getLatestSchema("foo")) 167 | } 168 | 169 | @Test 170 | fun `can create subject`() { 171 | val schema = mockk() 172 | val subject = Subject("foo", Compatibility.FORWARD, schema) 173 | 174 | every { client.register("foo", schema) } returns 42 175 | 176 | assertEquals(42, wrapper.create(subject)) 177 | } 178 | 179 | @Test 180 | fun `can delete subject`() { 181 | every { client.deleteSubject("foo") } returns listOf(1, 2) 182 | 183 | wrapper.delete("foo") 184 | 185 | verify { 186 | client.deleteSubject("foo") 187 | } 188 | } 189 | 190 | @Test 191 | fun `create will throw exception when schemaType is used prior to server version 55`() { 192 | val schema = mockk() 193 | val subject = Subject("foo", Compatibility.FORWARD, schema) 194 | 195 | every { client.register("foo", schema) } throws RestClientException("", 422, 422) 196 | 197 | assertThrows { 198 | wrapper.create(subject) 199 | } 200 | } 201 | 202 | @Test 203 | fun `can evolve schema for subject`() { 204 | val schema = mockk() 205 | val subject = Subject("foo", Compatibility.FORWARD, schema) 206 | 207 | every { client.register("foo", schema) } returns 21 208 | 209 | assertEquals(21, wrapper.evolve(subject)) 210 | } 211 | 212 | @Test 213 | fun `can get version for schema`() { 214 | val subject = Subject("foo", Compatibility.FORWARD, mockk()) 215 | 216 | every { client.getVersion("foo", subject.schema) } returns 21 217 | 218 | assertEquals(21, wrapper.version(subject)) 219 | } 220 | 221 | @Test 222 | fun `version returns null if it is not registered yet`() { 223 | val subject = Subject("foo", Compatibility.FORWARD, mockk()) 224 | 225 | every { client.getVersion("foo", subject.schema) } throws RestClientException("", 404, 40403) 226 | 227 | assertNull(wrapper.version(subject)) 228 | } 229 | } 230 | -------------------------------------------------------------------------------- /src/test/kotlin/dev/domnikl/schemaregistrygitops/StateTest.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops 2 | 3 | import io.confluent.kafka.schemaregistry.ParsedSchema 4 | import io.mockk.mockk 5 | import org.junit.Test 6 | import org.junit.jupiter.api.Assertions.assertEquals 7 | import org.junit.jupiter.api.assertThrows 8 | 9 | class StateTest { 10 | @Test 11 | fun `validates uniqueness of subjects`() { 12 | assertThrows { 13 | State( 14 | null, 15 | null, 16 | listOf( 17 | Subject("foo", null, mockk()), 18 | Subject("foo", null, mockk()) 19 | ) 20 | ) 21 | } 22 | } 23 | 24 | @Test 25 | fun `can merge two states`() { 26 | val schema1 = mockk() 27 | val schema2 = mockk() 28 | val schema3 = mockk() 29 | val schema4 = mockk() 30 | 31 | val a = State( 32 | null, 33 | null, 34 | listOf( 35 | Subject("abc", null, schema1), 36 | Subject("foo", Compatibility.BACKWARD, schema2) 37 | ) 38 | ) 39 | 40 | val b = State( 41 | Compatibility.BACKWARD, 42 | true, 43 | listOf( 44 | Subject("foo", null, schema3), 45 | Subject("bar", null, schema4) 46 | ) 47 | ) 48 | 49 | val merged = a.merge(b) 50 | 51 | assertEquals(Compatibility.BACKWARD, merged.compatibility) 52 | assertEquals(true, merged.normalize) 53 | assertEquals(listOf("abc", "foo", "bar"), merged.subjects.map { it.name }) 54 | assertEquals(listOf(null, null, null), merged.subjects.map { it.compatibility }) 55 | assertEquals(listOf(schema1, schema3, schema4), merged.subjects.map { it.schema }) 56 | } 57 | 58 | @Test 59 | fun `can merge and keep latest global compatibility`() { 60 | val a = State(Compatibility.FORWARD, null, emptyList()) 61 | val b = State(Compatibility.BACKWARD, null, emptyList()) 62 | 63 | assertEquals(Compatibility.BACKWARD, a.merge(b).compatibility) 64 | assertEquals(Compatibility.FORWARD, b.merge(a).compatibility) 65 | } 66 | 67 | @Test 68 | fun `can merge and keep latest normalize`() { 69 | val a = State(null, true, emptyList()) 70 | val b = State(null, false, emptyList()) 71 | 72 | assertEquals(false, a.merge(b).normalize) 73 | assertEquals(true, b.merge(a).normalize) 74 | } 75 | 76 | @Test 77 | fun `can handle defaulted normalize`() { 78 | val a = State(null, true, emptyList()) 79 | val b = State(null, null, emptyList()) 80 | 81 | assertEquals(true, a.merge(b).normalize) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/test/kotlin/dev/domnikl/schemaregistrygitops/TestUtils.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops 2 | 3 | import io.confluent.kafka.schemaregistry.avro.AvroSchema 4 | import io.confluent.kafka.schemaregistry.json.JsonSchema 5 | import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema 6 | import org.apache.avro.Schema 7 | import java.io.File 8 | 9 | fun fromResources(name: String) = File(object {}.javaClass.classLoader.getResource(name)!!.toURI()) 10 | fun stringFromResources(name: String) = fromResources(name).readText() 11 | fun avroFromResources(name: String) = AvroSchema(Schema.Parser().parse(fromResources(name))) 12 | fun protoFromResources(name: String) = ProtobufSchema(stringFromResources(name)) 13 | fun jsonFromResources(name: String) = JsonSchema(stringFromResources(name)) 14 | -------------------------------------------------------------------------------- /src/test/kotlin/dev/domnikl/schemaregistrygitops/cli/ApplyTest.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops.cli 2 | 3 | import dev.domnikl.schemaregistrygitops.CLI 4 | import dev.domnikl.schemaregistrygitops.Configuration 5 | import dev.domnikl.schemaregistrygitops.State 6 | import dev.domnikl.schemaregistrygitops.fromResources 7 | import dev.domnikl.schemaregistrygitops.state.Applier 8 | import dev.domnikl.schemaregistrygitops.state.Diffing 9 | import dev.domnikl.schemaregistrygitops.state.Persistence 10 | import dev.domnikl.schemaregistrygitops.state.Result 11 | import io.mockk.every 12 | import io.mockk.just 13 | import io.mockk.mockk 14 | import io.mockk.runs 15 | import io.mockk.verify 16 | import org.junit.Assert.assertEquals 17 | import org.junit.Test 18 | import org.slf4j.Logger 19 | import picocli.CommandLine 20 | 21 | class ApplyTest { 22 | private val configuration = mockk() 23 | private val persistence = mockk() 24 | private val diffing = mockk() 25 | private val applier = mockk() 26 | private val logger = mockk(relaxed = true) 27 | private val apply = Apply(configuration, persistence, diffing, applier, logger) 28 | private val commandLine = CommandLine(CLI()).addSubcommand(apply) 29 | 30 | @Test 31 | fun `can apply state to schema registry`() { 32 | val state = mockk() 33 | val diffingResult = Diffing.Result(added = listOf(mockk())) 34 | 35 | every { configuration.baseUrl } returns "https://foo.bar" 36 | every { persistence.load(any(), any()) } returns state 37 | every { applier.apply(any()) } returns Result.SUCCESS 38 | every { logger.info(any()) } just runs 39 | every { diffing.diff(state, false) } returns diffingResult 40 | 41 | val input = fromResources("only_compatibility.yml") 42 | val exitCode = commandLine.execute("apply", "--registry", "https://foo.bar", input.path) 43 | 44 | verify { logger.info("[SUCCESS] Applied state from ${input.path} to https://foo.bar") } 45 | 46 | assertEquals(0, exitCode) 47 | } 48 | 49 | @Test 50 | fun `can handle relative inputFile paths`() { 51 | every { persistence.load(any(), any()) } throws IllegalArgumentException("foobar") 52 | every { applier.apply(any()) } returns Result.ERROR 53 | every { logger.error(any()) } just runs 54 | 55 | val input = "only_compatibility.yml" 56 | val exitCode = commandLine.execute("apply", "--registry", "https://foo.bar", input) 57 | 58 | assertEquals(1, exitCode) 59 | 60 | verify { logger.error("java.lang.IllegalArgumentException: foobar") } 61 | } 62 | 63 | @Test 64 | fun `logs errors it encounters`() { 65 | every { persistence.load(any(), any()) } throws IllegalArgumentException("foobar") 66 | every { applier.apply(any()) } returns Result.ERROR 67 | every { logger.error(any()) } just runs 68 | 69 | val input = fromResources("only_compatibility.yml") 70 | val exitCode = commandLine.execute("apply", "--registry", "https://foo.bar", input.path) 71 | 72 | assertEquals(1, exitCode) 73 | 74 | verify { logger.error("java.lang.IllegalArgumentException: foobar") } 75 | } 76 | 77 | @Test 78 | fun `only normalize`() { 79 | val state = mockk() 80 | val diffingResult = Diffing.Result(added = listOf(mockk())) 81 | 82 | every { configuration.baseUrl } returns "https://foo.bar" 83 | every { persistence.load(any(), any()) } returns state 84 | every { applier.apply(any()) } returns Result.SUCCESS 85 | every { logger.info(any()) } just runs 86 | every { diffing.diff(state, false) } returns diffingResult 87 | 88 | val input = fromResources("only_normalize.yml") 89 | val exitCode = commandLine.execute("apply", "--registry", "https://foo.bar", input.path) 90 | 91 | verify { logger.info("[SUCCESS] Applied state from ${input.path} to https://foo.bar") } 92 | 93 | assertEquals(0, exitCode) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/test/kotlin/dev/domnikl/schemaregistrygitops/cli/DumpTest.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops.cli 2 | 3 | import dev.domnikl.schemaregistrygitops.CLI 4 | import dev.domnikl.schemaregistrygitops.Configuration 5 | import dev.domnikl.schemaregistrygitops.State 6 | import dev.domnikl.schemaregistrygitops.state.Dumper 7 | import dev.domnikl.schemaregistrygitops.state.Persistence 8 | import io.mockk.every 9 | import io.mockk.just 10 | import io.mockk.mockk 11 | import io.mockk.runs 12 | import org.junit.Assert.assertEquals 13 | import org.junit.Test 14 | import picocli.CommandLine 15 | import java.io.File 16 | 17 | class DumpTest { 18 | private val configuration = mockk() 19 | private val persistence = mockk() 20 | private val dumper = mockk() 21 | private val dump = Dump(configuration, persistence, dumper) 22 | private val commandLine = CommandLine(CLI()).addSubcommand(dump) 23 | 24 | @Test 25 | fun `can dump state to file`() { 26 | val tempFile = File.createTempFile(javaClass.simpleName, "can-dump-state-to-file") 27 | tempFile.deleteOnExit() 28 | 29 | val state = mockk() 30 | 31 | every { dumper.dump() } returns state 32 | every { persistence.save(state, any()) } just runs 33 | 34 | val exitCode = commandLine.execute("dump", "--registry", "http://foo.bar", tempFile.path) 35 | 36 | assertEquals(0, exitCode) 37 | } 38 | 39 | @Test 40 | fun `can dump state to stdout`() { 41 | val state = mockk() 42 | 43 | every { dumper.dump() } returns state 44 | every { persistence.save(state, System.out) } just runs 45 | 46 | val exitCode = commandLine.execute("dump", "--registry", "http://foo.bar") 47 | 48 | assertEquals(0, exitCode) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/test/kotlin/dev/domnikl/schemaregistrygitops/cli/PlanTest.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops.cli 2 | 3 | import dev.domnikl.schemaregistrygitops.CLI 4 | import dev.domnikl.schemaregistrygitops.Compatibility 5 | import dev.domnikl.schemaregistrygitops.Configuration 6 | import dev.domnikl.schemaregistrygitops.State 7 | import dev.domnikl.schemaregistrygitops.Subject 8 | import dev.domnikl.schemaregistrygitops.state.Diffing 9 | import dev.domnikl.schemaregistrygitops.state.Persistence 10 | import io.confluent.kafka.schemaregistry.ParsedSchema 11 | import io.mockk.every 12 | import io.mockk.mockk 13 | import io.mockk.verify 14 | import io.mockk.verifyOrder 15 | import org.junit.Assert.assertEquals 16 | import org.junit.Rule 17 | import org.junit.Test 18 | import org.junit.contrib.java.lang.system.ExpectedSystemExit 19 | import org.slf4j.Logger 20 | import picocli.CommandLine 21 | import java.io.File 22 | 23 | class PlanTest { 24 | @Rule 25 | @JvmField 26 | val exit: ExpectedSystemExit = ExpectedSystemExit.none() 27 | 28 | private val configuration = mockk() 29 | private val diffing = mockk() 30 | private val persistence = mockk() 31 | private val logger = mockk(relaxed = true) 32 | private val plan = Plan(configuration, persistence, diffing, logger) 33 | private val commandLine = CommandLine(CLI()).addSubcommand(plan) 34 | 35 | @Test 36 | fun `can validate YAML state file`() { 37 | val state = State( 38 | null, 39 | null, 40 | listOf(Subject("foo", null, mockk())) 41 | ) 42 | 43 | val input = fromResources("with_inline_schema.yml") 44 | 45 | every { persistence.load(any(), input) } returns state 46 | every { diffing.diff(any()) } returns Diffing.Result(compatibility = Diffing.Change(Compatibility.NONE, Compatibility.BACKWARD)) 47 | 48 | val exitCode = commandLine.execute("plan", "--registry", "https://foo.bar", *input.map { it.path }.toTypedArray()) 49 | 50 | assertEquals(0, exitCode) 51 | 52 | verify { 53 | logger.info("[GLOBAL]") 54 | logger.info(" ~ compatibility NONE -> BACKWARD") 55 | logger.info("") 56 | logger.info("[SUCCESS] All changes are compatible and can be applied.") 57 | } 58 | verify(exactly = 0) { logger.error(any()) } 59 | } 60 | 61 | @Test 62 | fun `will log success when no changes were made`() { 63 | val state = State( 64 | null, 65 | null, 66 | listOf(Subject("foo", null, mockk())) 67 | ) 68 | 69 | val input = fromResources("with_inline_schema.yml") 70 | 71 | every { persistence.load(any(), input) } returns state 72 | every { diffing.diff(any()) } returns Diffing.Result() 73 | 74 | val exitCode = commandLine.execute("plan", "--registry", "https://foo.bar", *input.map { it.path }.toTypedArray()) 75 | 76 | assertEquals(0, exitCode) 77 | 78 | verify { 79 | logger.info("[SUCCESS] There are no necessary changes; the actual state matches the desired state.") 80 | } 81 | verify(exactly = 0) { logger.error(any()) } 82 | } 83 | 84 | @Test 85 | fun `will log deletes`() { 86 | val state = State( 87 | null, 88 | null, 89 | listOf(Subject("foo", null, mockk())) 90 | ) 91 | 92 | val input = fromResources("with_inline_schema.yml") 93 | 94 | every { persistence.load(any(), input) } returns state 95 | every { diffing.diff(any(), true) } returns Diffing.Result(deleted = listOf("foobar")) 96 | 97 | val exitCode = 98 | commandLine.execute("plan", "--enable-deletes", "--registry", "https://foo.bar", *input.map { it.path }.toTypedArray()) 99 | 100 | assertEquals(0, exitCode) 101 | 102 | verify { 103 | logger.info("The following changes would be applied:") 104 | logger.info("") 105 | logger.info("[SUBJECT] foobar") 106 | logger.info(" - delete") 107 | logger.info("") 108 | logger.info("[SUCCESS] All changes are compatible and can be applied.") 109 | } 110 | verify(exactly = 0) { logger.error(any()) } 111 | } 112 | 113 | @Test 114 | fun `will log adds`() { 115 | val state = State( 116 | null, 117 | null, 118 | listOf(Subject("foo", null, mockk())) 119 | ) 120 | 121 | val input = fromResources("with_inline_schema.yml") 122 | 123 | every { persistence.load(any(), input) } returns state 124 | every { diffing.diff(any()) } returns Diffing.Result(added = state.subjects) 125 | 126 | val exitCode = commandLine.execute("plan", "--registry", "https://foo.bar", *input.map { it.path }.toTypedArray()) 127 | 128 | assertEquals(0, exitCode) 129 | 130 | verify { 131 | logger.info("The following changes would be applied:") 132 | logger.info("") 133 | logger.info("[SUBJECT] foo") 134 | logger.info(" + register") 135 | logger.info("") 136 | logger.info("[SUCCESS] All changes are compatible and can be applied.") 137 | } 138 | verify(exactly = 0) { logger.error(any()) } 139 | } 140 | 141 | @Test 142 | fun `will log changes`() { 143 | val state = State( 144 | null, 145 | null, 146 | listOf(Subject("foo", null, mockk())) 147 | ) 148 | 149 | val schemaBefore = mockk(relaxed = true) 150 | val schemaAfter = mockk(relaxed = true) 151 | 152 | val input = fromResources("with_inline_schema.yml") 153 | 154 | every { persistence.load(any(), input) } returns state 155 | every { diffing.diff(any()) } returns Diffing.Result( 156 | modified = listOf( 157 | Diffing.Changes( 158 | state.subjects.first(), 159 | Diffing.Change(Compatibility.NONE, Compatibility.BACKWARD), 160 | Diffing.Change(schemaBefore, schemaAfter) 161 | ) 162 | ) 163 | ) 164 | 165 | val exitCode = commandLine.execute("plan", "--registry", "https://foo.bar", *input.map { it.path }.toTypedArray()) 166 | 167 | verify { 168 | logger.info("The following changes would be applied:") 169 | logger.info("") 170 | logger.info("[SUBJECT] foo") 171 | logger.info(" ~ compatibility NONE -> BACKWARD") 172 | logger.info(" ~ schema ") 173 | logger.info("") 174 | logger.info("[SUCCESS] All changes are compatible and can be applied.") 175 | } 176 | 177 | verify(exactly = 0) { logger.error(any()) } 178 | 179 | assertEquals(0, exitCode) 180 | } 181 | 182 | @Test 183 | fun `can handle relative inputFile paths`() { 184 | val state = State( 185 | null, 186 | null, 187 | listOf(Subject("foo", null, mockk())) 188 | ) 189 | 190 | val input = "with_inline_schema.yml" 191 | 192 | every { persistence.load(any(), any()) } returns state 193 | every { diffing.diff(any()) } returns Diffing.Result() 194 | 195 | val exitCode = commandLine.execute("plan", "--registry", "https://foo.bar", input) 196 | 197 | assertEquals(0, exitCode) 198 | } 199 | 200 | @Test 201 | fun `can report validation fails`() { 202 | val state = State( 203 | null, 204 | null, 205 | listOf( 206 | Subject("foo", null, mockk()), 207 | Subject("bar", null, mockk()) 208 | ) 209 | ) 210 | 211 | val input = fromResources("with_inline_schema.yml") 212 | 213 | every { persistence.load(any(), input) } returns state 214 | every { diffing.diff(any()) } returns Diffing.Result( 215 | incompatible = listOf( 216 | Diffing.CompatibilityTestResult( 217 | state.subjects[0], 218 | listOf("my message") 219 | ) 220 | ) 221 | ) 222 | 223 | val exitCode = commandLine.execute("plan", "--registry", "foo", *input.map { it.path }.toTypedArray()) 224 | 225 | assertEquals(1, exitCode) 226 | 227 | verifyOrder { 228 | logger.error("[ERROR] The following schema is incompatible with an earlier version: 'foo': 'my message'") 229 | } 230 | } 231 | 232 | @Test 233 | fun `can report other errors`() { 234 | val state = State( 235 | null, 236 | null, 237 | listOf( 238 | Subject("foo", null, mockk()), 239 | Subject("bar", null, mockk()) 240 | ) 241 | ) 242 | 243 | every { persistence.load(any(), any()) } returns state 244 | every { diffing.diff(any()) } throws IllegalArgumentException("foobar") 245 | 246 | val input = fromResources("with_inline_schema.yml") 247 | val exitCode = commandLine.execute("plan", "--registry", "foo", *input.map { it.path }.toTypedArray()) 248 | 249 | assertEquals(2, exitCode) 250 | 251 | verify { logger.error("java.lang.IllegalArgumentException: foobar") } 252 | } 253 | 254 | private fun fromResources(name: String) = listOf(File(javaClass.classLoader.getResource(name)!!.toURI())) 255 | } 256 | -------------------------------------------------------------------------------- /src/test/kotlin/dev/domnikl/schemaregistrygitops/state/ApplierTest.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops.state 2 | 3 | import dev.domnikl.schemaregistrygitops.Compatibility 4 | import dev.domnikl.schemaregistrygitops.SchemaRegistryClient 5 | import dev.domnikl.schemaregistrygitops.Subject 6 | import dev.domnikl.schemaregistrygitops.avroFromResources 7 | import io.confluent.kafka.schemaregistry.ParsedSchema 8 | import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference 9 | import io.mockk.every 10 | import io.mockk.just 11 | import io.mockk.mockk 12 | import io.mockk.runs 13 | import io.mockk.verify 14 | import io.mockk.verifyOrder 15 | import org.junit.Assert.assertEquals 16 | import org.junit.Test 17 | import org.slf4j.Logger 18 | 19 | class ApplierTest { 20 | private val client = mockk() 21 | private val logger = mockk(relaxed = true) 22 | private val stateApplier = Applier(client, logger) 23 | 24 | @Test 25 | fun `throws exception when trying to apply with incompatibilities`() { 26 | val schema = mockk() 27 | val subject = Subject("foo", null, schema) 28 | 29 | val diff = Diffing.Result(incompatible = listOf(Diffing.CompatibilityTestResult(subject, listOf("error!")))) 30 | assertEquals(Result.ERROR, stateApplier.apply(diff)) 31 | } 32 | 33 | @Test 34 | fun `can apply global compatibility`() { 35 | every { client.updateGlobalCompatibility(Compatibility.FULL_TRANSITIVE) } returns Compatibility.FULL_TRANSITIVE 36 | 37 | val diff = Diffing.Result(compatibility = Diffing.Change(Compatibility.FULL, Compatibility.FULL_TRANSITIVE)) 38 | 39 | stateApplier.apply(diff) 40 | 41 | verifyOrder { 42 | client.updateGlobalCompatibility(Compatibility.FULL_TRANSITIVE) 43 | logger.info("[GLOBAL]") 44 | logger.info(" ~ compatibility FULL -> FULL_TRANSITIVE") 45 | logger.info("") 46 | } 47 | } 48 | 49 | @Test 50 | fun `can apply normalize`() { 51 | every { client.updateNormalize(true) } just runs 52 | 53 | val diff = Diffing.Result(normalize = Diffing.Change(false, true)) 54 | 55 | stateApplier.apply(diff) 56 | 57 | verifyOrder { 58 | client.updateNormalize(true) 59 | logger.info("[GLOBAL]") 60 | logger.info(" ~ normalize false -> true") 61 | logger.info("") 62 | } 63 | } 64 | 65 | @Test 66 | fun `will not change global compatibility if matches state`() { 67 | val diff = Diffing.Result() 68 | 69 | stateApplier.apply(diff) 70 | 71 | verify(exactly = 0) { client.updateGlobalCompatibility(Compatibility.FULL_TRANSITIVE) } 72 | } 73 | 74 | @Test 75 | fun `can create new subject`() { 76 | val schema = avroFromResources("schemas/key.avsc") 77 | val subject = Subject("foo", null, schema) 78 | 79 | every { client.subjects() } returns emptyList() 80 | every { client.create(subject) } returns 1 81 | 82 | val diff = Diffing.Result(added = listOf(subject)) 83 | 84 | stateApplier.apply(diff) 85 | 86 | verifyOrder { 87 | client.create(subject) 88 | logger.info("[SUBJECT] foo") 89 | logger.info(" + registered (version 1)") 90 | logger.info("") 91 | } 92 | 93 | verify(exactly = 0) { client.updateCompatibility(subject) } 94 | } 95 | 96 | @Test 97 | fun `can create new subject with references`() { 98 | val schema = avroFromResources("schemas/key.avsc") 99 | val subject = Subject("foo", null, schema, listOf(SchemaReference("bar", "bar", 1))) 100 | 101 | every { client.subjects() } returns emptyList() 102 | every { client.create(subject) } returns 1 103 | 104 | val diff = Diffing.Result(added = listOf(subject)) 105 | 106 | stateApplier.apply(diff) 107 | 108 | verifyOrder { 109 | client.create(subject) 110 | logger.info("[SUBJECT] foo") 111 | logger.info(" + registered (version 1)") 112 | logger.info("") 113 | } 114 | 115 | verify(exactly = 0) { client.updateCompatibility(subject) } 116 | } 117 | 118 | @Test 119 | fun `can register new subject and set compatibility`() { 120 | val schema = avroFromResources("schemas/key.avsc") 121 | val subject = Subject("foo", Compatibility.BACKWARD, schema) 122 | 123 | every { client.version(subject) } returns null 124 | every { client.create(subject) } returns 1 125 | every { client.updateCompatibility(subject) } returns Compatibility.BACKWARD 126 | 127 | val diff = Diffing.Result(added = listOf(subject)) 128 | 129 | stateApplier.apply(diff) 130 | 131 | verifyOrder { 132 | client.create(subject) 133 | logger.info("[SUBJECT] foo") 134 | logger.info(" + registered (version 1)") 135 | client.updateCompatibility(subject) 136 | logger.info(" + compatibility BACKWARD") 137 | logger.info("") 138 | } 139 | } 140 | 141 | @Test 142 | fun `can evolve schema`() { 143 | val schema = avroFromResources("schemas/key.avsc") 144 | val subject = Subject("foo", null, schema) 145 | 146 | every { client.version(subject) } returns null 147 | every { client.evolve(subject) } returns 5 148 | 149 | val diff = Diffing.Result( 150 | modified = listOf(Diffing.Changes(subject, null, Diffing.Change(schema, schema))) 151 | ) 152 | 153 | stateApplier.apply(diff) 154 | 155 | verifyOrder { 156 | logger.info("[SUBJECT] foo") 157 | client.evolve(subject) 158 | logger.info(" ~ evolved (version 5)") 159 | logger.info(" ~ schema \"string\"") 160 | logger.info("") 161 | } 162 | verify(exactly = 0) { client.updateCompatibility(subject) } 163 | } 164 | 165 | @Test 166 | fun `can delete subject`() { 167 | val diff = Diffing.Result(deleted = listOf("foo")) 168 | 169 | every { client.delete("foo") } just runs 170 | 171 | stateApplier.apply(diff) 172 | 173 | verifyOrder { 174 | client.delete("foo") 175 | 176 | logger.info("[SUBJECT] foo") 177 | logger.info(" - deleted") 178 | logger.info("") 179 | } 180 | } 181 | 182 | @Test 183 | fun `can update compatibility and evolve schema`() { 184 | val schema = avroFromResources("schemas/key.avsc") 185 | val subject = Subject("foo", Compatibility.FORWARD_TRANSITIVE, schema) 186 | 187 | every { client.version(subject) } returns null 188 | every { client.updateCompatibility(subject) } returns Compatibility.FULL 189 | every { client.evolve(subject) } returns 2 190 | 191 | val diff = Diffing.Result( 192 | modified = listOf( 193 | Diffing.Changes( 194 | subject, 195 | Diffing.Change(Compatibility.FORWARD_TRANSITIVE, Compatibility.FULL), 196 | Diffing.Change(schema, schema) 197 | ) 198 | ) 199 | ) 200 | 201 | stateApplier.apply(diff) 202 | 203 | verifyOrder { 204 | logger.info("[SUBJECT] foo") 205 | client.updateCompatibility(subject) 206 | logger.info(" ~ compatibility FORWARD_TRANSITIVE -> FULL") 207 | client.evolve(subject) 208 | logger.info(" ~ evolved (version 2)") 209 | logger.info(" ~ schema \"string\"") 210 | logger.info("") 211 | } 212 | } 213 | 214 | @Test 215 | fun `will not change subject compatibility if matches state`() { 216 | val schema = avroFromResources("schemas/key.avsc") 217 | val subject = Subject("foo", Compatibility.FULL, schema) 218 | 219 | every { client.version(subject) } returns 1 220 | 221 | val diff = Diffing.Result( 222 | modified = listOf( 223 | Diffing.Changes( 224 | subject, 225 | null, 226 | null 227 | ) 228 | ) 229 | ) 230 | 231 | stateApplier.apply(diff) 232 | 233 | verify(exactly = 0) { 234 | client.version(subject) 235 | client.updateCompatibility(subject) 236 | } 237 | } 238 | } 239 | -------------------------------------------------------------------------------- /src/test/kotlin/dev/domnikl/schemaregistrygitops/state/DiffingTest.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops.state 2 | 3 | import dev.domnikl.schemaregistrygitops.Compatibility 4 | import dev.domnikl.schemaregistrygitops.SchemaRegistryClient 5 | import dev.domnikl.schemaregistrygitops.State 6 | import dev.domnikl.schemaregistrygitops.Subject 7 | import io.confluent.kafka.schemaregistry.ParsedSchema 8 | import io.confluent.kafka.schemaregistry.avro.AvroSchema 9 | import io.mockk.every 10 | import io.mockk.mockk 11 | import org.junit.Assert.assertEquals 12 | import org.junit.Assert.assertFalse 13 | import org.junit.Assert.assertNull 14 | import org.junit.Test 15 | 16 | class DiffingTest { 17 | private val client = mockk(relaxed = true) 18 | private val diff = Diffing(client) 19 | private val schema = AvroSchema( 20 | "{\"type\": \"record\",\"name\": \"HelloWorld\"," + 21 | "\"namespace\": \"dev.domnikl.schema_registry_gitops\",\"doc\": \"this is some docs to be replaced ...\"," + 22 | "\"fields\": [{\"name\": \"greeting\",\"type\": \"string\"}]}" 23 | ) 24 | private val subject = Subject("foobar", Compatibility.FORWARD, schema) 25 | private val subject2 = Subject("bar", Compatibility.BACKWARD, schema) 26 | 27 | @Test 28 | fun `can detect global compatibility change`() { 29 | val state = State(Compatibility.BACKWARD, null, emptyList()) 30 | 31 | every { client.globalCompatibility() } returns Compatibility.NONE 32 | 33 | val result = diff.diff(state) 34 | 35 | assertEquals(Diffing.Change(Compatibility.NONE, Compatibility.BACKWARD), result.compatibility) 36 | assertEquals(emptyList(), result.incompatible) 37 | assertEquals(emptyList(), result.added) 38 | assertEquals(emptyList(), result.deleted) 39 | assertEquals(emptyList(), result.modified) 40 | } 41 | 42 | @Test 43 | fun `no diff on null normalize`() { 44 | val state = State(null, null, emptyList()) 45 | 46 | every { client.normalize() } returns true 47 | 48 | val result = diff.diff(state) 49 | 50 | assertNull(result.normalize) 51 | } 52 | 53 | @Test 54 | fun `can detect normalize change`() { 55 | val state = State(null, true, emptyList()) 56 | 57 | every { client.normalize() } returns false 58 | 59 | val result = diff.diff(state) 60 | 61 | assertEquals(Diffing.Change(false, true), result.normalize) 62 | } 63 | 64 | @Test 65 | fun `can detect incompatible changes`() { 66 | val state = State(Compatibility.BACKWARD, null, listOf(subject)) 67 | 68 | every { client.subjects() } returns listOf("foobar") 69 | every { client.testCompatibility(any()) } returns listOf("incompatible!") 70 | 71 | val result = diff.diff(state) 72 | 73 | assertEquals(listOf(Diffing.CompatibilityTestResult(subject, listOf("incompatible!"))), result.incompatible) 74 | assertEquals(emptyList(), result.added) 75 | assertEquals(emptyList(), result.modified) 76 | assertEquals(emptyList(), result.deleted) 77 | } 78 | 79 | @Test 80 | fun `can detect added subjects`() { 81 | val state = State(Compatibility.BACKWARD, null, listOf(subject)) 82 | 83 | every { client.subjects() } returns emptyList() 84 | every { client.testCompatibility(any()) } returns emptyList() 85 | 86 | val result = diff.diff(state) 87 | 88 | assertEquals(emptyList(), result.incompatible) 89 | assertEquals(listOf(subject), result.added) 90 | assertEquals(emptyList(), result.modified) 91 | assertEquals(emptyList(), result.deleted) 92 | } 93 | 94 | @Test 95 | fun `can detect deleted subjects`() { 96 | val state = State(Compatibility.BACKWARD, null, emptyList()) 97 | 98 | every { client.subjects() } returns listOf("foobar") 99 | 100 | val result = diff.diff(state, true) 101 | 102 | assertEquals(emptyList(), result.incompatible) 103 | assertEquals(emptyList(), result.added) 104 | assertEquals(emptyList(), result.modified) 105 | assertEquals(listOf("foobar"), result.deleted) 106 | } 107 | 108 | @Test 109 | fun `will hide deletes if not enabled`() { 110 | val state = State(Compatibility.BACKWARD, null, emptyList()) 111 | 112 | every { client.subjects() } returns listOf("foobar") 113 | 114 | val result = diff.diff(state, false) 115 | 116 | assertEquals(emptyList(), result.incompatible) 117 | assertEquals(emptyList(), result.added) 118 | assertEquals(emptyList(), result.modified) 119 | assertEquals(emptyList(), result.deleted) 120 | } 121 | 122 | @Test 123 | fun `can detect if compatibility has been modified for subjects`() { 124 | val state = State(Compatibility.BACKWARD, null, listOf(subject)) 125 | val remoteSchema = mockk() 126 | 127 | every { client.subjects() } returns listOf("foobar") 128 | every { remoteSchema.canonicalString() } returns subject.schema.canonicalString() 129 | every { client.getLatestSchema("foobar") } returns remoteSchema 130 | every { client.testCompatibility(any()) } returns emptyList() 131 | every { client.compatibility("foobar") } returns Compatibility.BACKWARD_TRANSITIVE 132 | 133 | val result = diff.diff(state) 134 | 135 | assertEquals(emptyList(), result.incompatible) 136 | assertEquals(emptyList(), result.added) 137 | assertEquals(emptyList(), result.deleted) 138 | assertEquals(1, result.modified.size) 139 | assertEquals( 140 | Diffing.Changes( 141 | subject, 142 | Diffing.Change(Compatibility.BACKWARD_TRANSITIVE, Compatibility.FORWARD), 143 | null 144 | ), 145 | result.modified.first() 146 | ) 147 | } 148 | 149 | @Test 150 | fun `can detect that schema has been modified for subject`() { 151 | val state = State(Compatibility.BACKWARD, null, listOf(subject, subject2)) 152 | val remoteSchema = mockk() 153 | 154 | every { client.subjects() } returns listOf("foobar", "bar") 155 | every { remoteSchema.canonicalString() } returns subject.schema.canonicalString() 156 | every { client.getLatestSchema("foobar") } returns remoteSchema 157 | every { client.getLatestSchema("bar") } returns remoteSchema 158 | every { client.version(subject) } returns null 159 | every { client.version(subject2) } returns null 160 | every { client.testCompatibility(any()) } returns emptyList() 161 | every { client.compatibility("foobar") } returns subject.compatibility!! 162 | every { client.compatibility("bar") } returns subject2.compatibility!! 163 | 164 | val result = diff.diff(state) 165 | 166 | assertEquals(emptyList(), result.incompatible) 167 | assertEquals(emptyList(), result.added) 168 | assertEquals(emptyList(), result.deleted) 169 | assertEquals( 170 | listOf( 171 | Diffing.Changes(subject, null, Diffing.Change(remoteSchema, subject.schema)), 172 | Diffing.Changes(subject2, null, Diffing.Change(remoteSchema, subject.schema)) 173 | ), 174 | result.modified 175 | ) 176 | } 177 | 178 | @Test 179 | fun `can detect that schema already exists in an older version`() { 180 | val state = State(Compatibility.BACKWARD, null, listOf(subject)) 181 | val remoteSchema = mockk() 182 | 183 | every { client.subjects() } returns listOf("foobar") 184 | every { remoteSchema.canonicalString() } returns subject.schema.canonicalString() 185 | every { client.getLatestSchema("foobar") } returns remoteSchema 186 | every { client.version(subject) } returns 5 187 | every { client.testCompatibility(any()) } returns emptyList() 188 | every { client.compatibility("foobar") } returns subject.compatibility!! 189 | 190 | val result = diff.diff(state) 191 | 192 | assertEquals(emptyList(), result.incompatible) 193 | assertEquals(emptyList(), result.added) 194 | assertEquals(emptyList(), result.deleted) 195 | assertEquals(emptyList(), result.modified) 196 | } 197 | 198 | @Test 199 | fun `can detect that nothing has changed`() { 200 | val state = State(Compatibility.BACKWARD, null, listOf(subject)) 201 | val remoteSchema = mockk() 202 | 203 | every { client.subjects() } returns listOf("foobar") 204 | every { remoteSchema.canonicalString() } returns subject.schema.canonicalString() 205 | every { client.getLatestSchema("foobar") } returns remoteSchema 206 | every { client.testCompatibility(any()) } returns emptyList() 207 | every { client.compatibility("foobar") } returns subject.compatibility!! 208 | 209 | val result = diff.diff(state) 210 | 211 | assertEquals(emptyList(), result.incompatible) 212 | assertEquals(emptyList(), result.added) 213 | assertEquals(emptyList(), result.deleted) 214 | assertEquals(emptyList(), result.modified) 215 | } 216 | 217 | @Test 218 | fun `can detect doc changes`() { 219 | val changedSchema = 220 | AvroSchema( 221 | "{\"type\": \"record\",\"name\": \"HelloWorld\"," + 222 | "\"namespace\": \"dev.domnikl.schema_registry_gitops\",\"doc\": \"This is the new docs.\"," + 223 | "\"fields\": [{\"name\": \"greeting\",\"type\": \"string\"}]}" 224 | ) 225 | 226 | val subject = Subject("foobar", Compatibility.FORWARD, changedSchema) 227 | val state = State(Compatibility.BACKWARD, null, listOf(subject)) 228 | 229 | every { client.subjects() } returns listOf("foobar") 230 | every { client.getLatestSchema("foobar") } returns schema 231 | every { client.testCompatibility(any()) } returns emptyList() 232 | every { client.compatibility("foobar") } returns subject.compatibility!! 233 | 234 | val result = diff.diff(state) 235 | 236 | assertEquals(emptyList(), result.incompatible) 237 | assertEquals(emptyList(), result.added) 238 | assertEquals(emptyList(), result.deleted) 239 | assertEquals(listOf(Diffing.Changes(subject, null, Diffing.Change(schema, changedSchema))), result.modified) 240 | } 241 | 242 | class ResultTest { 243 | @Test 244 | fun `can be check if empty`() { 245 | assert(Diffing.Result().isEmpty()) 246 | assertFalse(Diffing.Result(incompatible = listOf(mockk())).isEmpty()) 247 | assertFalse(Diffing.Result(added = listOf(mockk())).isEmpty()) 248 | assertFalse(Diffing.Result(modified = listOf(mockk())).isEmpty()) 249 | assertFalse(Diffing.Result(compatibility = Diffing.Change(Compatibility.NONE, Compatibility.FULL)).isEmpty()) 250 | assertFalse(Diffing.Result(deleted = listOf("foo")).isEmpty()) 251 | } 252 | } 253 | } 254 | -------------------------------------------------------------------------------- /src/test/kotlin/dev/domnikl/schemaregistrygitops/state/DumperTest.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops.state 2 | 3 | import dev.domnikl.schemaregistrygitops.Compatibility 4 | import dev.domnikl.schemaregistrygitops.SchemaRegistryClient 5 | import dev.domnikl.schemaregistrygitops.State 6 | import dev.domnikl.schemaregistrygitops.Subject 7 | import io.confluent.kafka.schemaregistry.avro.AvroSchema 8 | import io.mockk.every 9 | import io.mockk.mockk 10 | import org.junit.Assert.assertEquals 11 | import org.junit.Test 12 | 13 | class DumperTest { 14 | private val client = mockk() 15 | private val stateDumper = Dumper(client) 16 | 17 | @Test 18 | fun `can dump current state with subjects`() { 19 | val schema = AvroSchema("{\"name\":\"FooKey\",\"type\":\"string\"}") 20 | 21 | every { client.globalCompatibility() } returns Compatibility.FULL 22 | every { client.subjects() } returns listOf("foo", "bar") 23 | 24 | every { client.normalize() } returns true 25 | 26 | every { client.compatibility("foo") } returns Compatibility.FULL_TRANSITIVE 27 | every { client.getLatestSchema("foo") } returns schema 28 | 29 | every { client.compatibility("bar") } returns Compatibility.BACKWARD 30 | every { client.getLatestSchema("bar") } returns schema 31 | 32 | val expectedState = State( 33 | Compatibility.FULL, 34 | true, 35 | listOf( 36 | Subject( 37 | "foo", 38 | Compatibility.FULL_TRANSITIVE, 39 | schema 40 | ), 41 | Subject( 42 | "bar", 43 | Compatibility.BACKWARD, 44 | schema 45 | ) 46 | ) 47 | ) 48 | 49 | val state = stateDumper.dump() 50 | 51 | assertEquals(expectedState, state) 52 | } 53 | 54 | @Test 55 | fun `can dump current state handling implicit compatibility`() { 56 | val schema = AvroSchema("{\"name\":\"FooKey\",\"type\":\"string\"}") 57 | 58 | every { client.globalCompatibility() } returns Compatibility.FULL 59 | every { client.normalize() } returns false 60 | every { client.subjects() } returns listOf("bar") 61 | 62 | every { client.compatibility("bar") } returns Compatibility.NONE 63 | every { client.getLatestSchema("bar") } returns schema 64 | 65 | val expectedState = State( 66 | Compatibility.FULL, 67 | false, 68 | listOf( 69 | Subject( 70 | "bar", 71 | Compatibility.NONE, 72 | schema 73 | ) 74 | ) 75 | ) 76 | 77 | val state = stateDumper.dump() 78 | 79 | assertEquals(expectedState, state) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/test/kotlin/dev/domnikl/schemaregistrygitops/state/PersistenceTest.kt: -------------------------------------------------------------------------------- 1 | package dev.domnikl.schemaregistrygitops.state 2 | 3 | import com.fasterxml.jackson.databind.exc.MismatchedInputException 4 | import dev.domnikl.schemaregistrygitops.Compatibility 5 | import dev.domnikl.schemaregistrygitops.SchemaParseException 6 | import dev.domnikl.schemaregistrygitops.State 7 | import dev.domnikl.schemaregistrygitops.Subject 8 | import dev.domnikl.schemaregistrygitops.avroFromResources 9 | import dev.domnikl.schemaregistrygitops.fromResources 10 | import dev.domnikl.schemaregistrygitops.stringFromResources 11 | import io.confluent.kafka.schemaregistry.ParsedSchema 12 | import io.confluent.kafka.schemaregistry.avro.AvroSchema 13 | import io.confluent.kafka.schemaregistry.avro.AvroSchemaProvider 14 | import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient 15 | import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference 16 | import io.confluent.kafka.schemaregistry.json.JsonSchema 17 | import io.confluent.kafka.schemaregistry.json.JsonSchemaProvider 18 | import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema 19 | import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchemaProvider 20 | import io.mockk.every 21 | import io.mockk.mockk 22 | import io.mockk.verify 23 | import org.junit.Assert.assertEquals 24 | import org.junit.Assert.assertNull 25 | import org.junit.Test 26 | import org.junit.jupiter.api.assertThrows 27 | import org.slf4j.Logger 28 | import java.io.BufferedOutputStream 29 | import java.io.ByteArrayOutputStream 30 | import java.io.File 31 | import java.io.FileNotFoundException 32 | import java.io.FileOutputStream 33 | import java.util.Optional 34 | 35 | class PersistenceTest { 36 | private val logger = mockk(relaxed = true) 37 | private val schemaRegistryClient = mockk() 38 | private val loader = Persistence(schemaRegistryClient, logger) 39 | 40 | @Test 41 | fun `throws exception when trying to load without a file`() { 42 | assertThrows { 43 | loader.load(fromResources("schemas"), emptyList()) 44 | } 45 | } 46 | 47 | @Test 48 | fun `throws exception when trying to load empty file`() { 49 | assertThrows { 50 | loader.load(fromResources("schemas"), listOf(fromResources("empty.yml"))) 51 | } 52 | } 53 | 54 | @Test 55 | fun `throws exception when trying to load non-existing file`() { 56 | assertThrows { 57 | loader.load(fromResources("schemas"), listOf(File("foo"))) 58 | } 59 | } 60 | 61 | @Test 62 | fun `can load file with only compatibility`() { 63 | val state = loader.load(fromResources("schemas"), listOf(fromResources("only_compatibility.yml"))) 64 | 65 | assertEquals(Compatibility.FORWARD, state.compatibility) 66 | assertEquals(null, state.normalize) 67 | assertEquals(emptyList(), state.subjects) 68 | } 69 | 70 | @Test 71 | fun `can load file with only normalize`() { 72 | val state = loader.load(fromResources("schemas"), listOf(fromResources("only_normalize.yml"))) 73 | 74 | assertEquals(null, state.compatibility) 75 | assertEquals(true, state.normalize) 76 | assertEquals(emptyList(), state.subjects) 77 | } 78 | 79 | @Test 80 | fun `can load multiple files`() { 81 | val schemaString = stringFromResources("schemas/with_subjects.avsc") 82 | val schema = mockk() 83 | 84 | every { schemaRegistryClient.parseSchema("AVRO", schemaString, emptyList()) } returns Optional.of(schema) 85 | 86 | val state = loader.load( 87 | fromResources("schemas"), 88 | listOf( 89 | fromResources("only_compatibility.yml"), 90 | fromResources("with_subjects.yml") 91 | ) 92 | ) 93 | 94 | assertEquals(Compatibility.FORWARD, state.compatibility) 95 | assertEquals(listOf(Subject("foo", null, schema)), state.subjects) 96 | } 97 | 98 | @Test 99 | fun `debug logs which file it loads`() { 100 | val basePath = fromResources("schemas") 101 | val file = fromResources("no_compatibility.yml") 102 | 103 | loader.load(basePath, listOf(file)) 104 | 105 | verify { logger.debug("Loading state file ${file.absolutePath}, referenced schemas from ${basePath.absolutePath}") } 106 | } 107 | 108 | @Test 109 | fun `can load file without subjects`() { 110 | val state = loader.load(fromResources("schemas"), listOf(fromResources("no_compatibility.yml"))) 111 | 112 | assertNull(state.compatibility) 113 | assertEquals(emptyList(), state.subjects) 114 | } 115 | 116 | @Test 117 | fun `can load file with subjects`() { 118 | val schemaString = stringFromResources("schemas/with_subjects.avsc") 119 | val schema = mockk() 120 | 121 | every { schemaRegistryClient.parseSchema("AVRO", schemaString, emptyList()) } returns Optional.of(schema) 122 | 123 | val state = loader.load(fromResources("schemas"), listOf(fromResources("with_subjects.yml"))) 124 | 125 | assertNull(state.compatibility) 126 | assertEquals(listOf(Subject("foo", null, schema)), state.subjects) 127 | } 128 | 129 | @Test 130 | fun `can load file with subjects and references`() { 131 | val schemaString = stringFromResources("schemas/with_subjects_and_references.avsc") 132 | val schema = mockk() 133 | 134 | every { 135 | schemaRegistryClient.parseSchema( 136 | "AVRO", 137 | schemaString, 138 | listOf(SchemaReference("dev.domnikl.schema_registry_gitops.foo", "foo", 1)) 139 | ) 140 | } returns Optional.of(schema) 141 | 142 | val state = loader.load(fromResources("schemas"), listOf(fromResources("with_subjects_and_references.yml"))) 143 | 144 | assertNull(state.compatibility) 145 | 146 | assertEquals( 147 | listOf( 148 | Subject( 149 | "bar", 150 | null, 151 | schema, 152 | listOf(SchemaReference("dev.domnikl.schema_registry_gitops.foo", "foo", 1)) 153 | ) 154 | ), 155 | state.subjects 156 | ) 157 | } 158 | 159 | @Test 160 | fun `can load file with inline schema`() { 161 | val schemaString = 162 | """ 163 | { 164 | "type": "record", 165 | "name": "HelloWorld", 166 | "namespace": "dev.domnikl.schema_registry_gitops", 167 | "fields": [ 168 | { 169 | "name": "greeting", 170 | "type": "string" 171 | } 172 | ] 173 | } 174 | """ 175 | val schema = mockk() 176 | 177 | every { 178 | schemaRegistryClient.parseSchema( 179 | "AVRO", 180 | match { it.replace("\\s".toRegex(), "") == schemaString.replace("\\s".toRegex(), "") }, 181 | emptyList() 182 | ) 183 | } returns Optional.of(schema) 184 | 185 | val state = loader.load(fromResources("schemas"), listOf(fromResources("with_inline_schema.yml"))) 186 | 187 | assertEquals(listOf(Subject("foo", Compatibility.BACKWARD, schema)), state.subjects) 188 | } 189 | 190 | @Test 191 | fun `can load file with subjects and compatibility`() { 192 | val schemaString = stringFromResources("schemas/with_subjects.avsc") 193 | val schema = mockk() 194 | 195 | every { schemaRegistryClient.parseSchema("AVRO", schemaString, emptyList()) } returns Optional.of(schema) 196 | 197 | val state = loader.load(fromResources("schemas"), listOf(fromResources("with_subjects_and_compatibility.yml"))) 198 | 199 | assertEquals(Compatibility.FULL, state.compatibility) 200 | assertEquals(listOf(Subject("foo", Compatibility.FORWARD, schema)), state.subjects) 201 | } 202 | 203 | @Test 204 | fun `throws exception when neither schema nor file was given`() { 205 | assertThrows { 206 | loader.load(fromResources("schemas"), listOf(fromResources("neither_schema_nor_file.yml"))) 207 | } 208 | } 209 | 210 | @Test 211 | fun `throws exception when referenced file does not exist`() { 212 | assertThrows { 213 | loader.load(fromResources("schemas"), listOf(fromResources("referenced_file_does_not_exist.yml"))) 214 | } 215 | } 216 | 217 | @Test 218 | fun `throws exception when subject name is missing`() { 219 | assertThrows { 220 | loader.load(fromResources("schemas"), listOf(fromResources("subject_name_is_missing.yml"))) 221 | } 222 | } 223 | 224 | @Test 225 | fun `can save state to a file`() { 226 | val schema1 = avroFromResources("schemas/with_subjects.avsc") 227 | val schema2 = avroFromResources("schemas/key.avsc") 228 | 229 | every { schemaRegistryClient.parseSchema("AVRO", schema1.toString(), emptyList()) } returns Optional.of(schema1) 230 | every { schemaRegistryClient.parseSchema("AVRO", schema2.toString(), emptyList()) } returns Optional.of(schema2) 231 | 232 | val tempFile = File.createTempFile(javaClass.simpleName, "can-save-state-to-a-file") 233 | tempFile.deleteOnExit() 234 | 235 | val outputStream = BufferedOutputStream(FileOutputStream(tempFile)) 236 | 237 | val currentState = State( 238 | Compatibility.BACKWARD_TRANSITIVE, 239 | false, 240 | listOf( 241 | Subject("foobar-value", null, schema1), 242 | Subject("foobar-key", Compatibility.FULL, schema2) 243 | ) 244 | ) 245 | 246 | loader.save(currentState, outputStream) 247 | 248 | assertEquals(currentState, loader.load(fromResources("schemas"), listOf(tempFile))) 249 | } 250 | 251 | @Test 252 | fun `can save state to a file without global compatibility`() { 253 | val outputStream = ByteArrayOutputStream() 254 | 255 | val currentState = State( 256 | null, 257 | null, 258 | emptyList() 259 | ) 260 | 261 | loader.save(currentState, outputStream) 262 | 263 | val expectedOutput = 264 | """ 265 | compatibility: NONE 266 | normalize: false 267 | subjects: [] 268 | 269 | """.trimIndent() 270 | 271 | assertEquals(expectedOutput, outputStream.toString()) 272 | } 273 | 274 | @Test 275 | fun `can save state to a file with normalize set`() { 276 | val outputStream = ByteArrayOutputStream() 277 | 278 | val currentState = State( 279 | null, 280 | true, 281 | emptyList() 282 | ) 283 | 284 | loader.save(currentState, outputStream) 285 | 286 | val expectedOutput = 287 | """ 288 | compatibility: NONE 289 | normalize: true 290 | subjects: [] 291 | 292 | """.trimIndent() 293 | 294 | assertEquals(expectedOutput, outputStream.toString()) 295 | } 296 | 297 | class YamlSubjectTest { 298 | private val schemaRegistryClient = CachedSchemaRegistryClient( 299 | listOf("http://foo.bar"), 300 | 100, 301 | listOf(AvroSchemaProvider(), ProtobufSchemaProvider(), JsonSchemaProvider()), 302 | null, 303 | null 304 | ) 305 | 306 | @Test 307 | fun `file takes precedence over inline schema`() { 308 | val subject = Persistence.YamlSubject( 309 | "foo", 310 | "key.avsc", 311 | "AVRO", 312 | """ 313 | syntax = "proto3"; 314 | package com.acme; 315 | 316 | message OtherRecord { 317 | int32 an_id = 1; 318 | } 319 | """.trimIndent(), 320 | null 321 | ) 322 | 323 | val schema = subject.parseSchema(fromResources("schemas"), schemaRegistryClient) 324 | 325 | assert(schema is AvroSchema) 326 | } 327 | 328 | @Test 329 | fun `can parse Protobuf schema`() { 330 | val subject = Persistence.YamlSubject( 331 | "foo", 332 | null, 333 | "PROTOBUF", 334 | """ 335 | syntax = "proto3"; 336 | package com.acme; 337 | 338 | message OtherRecord { 339 | int32 an_id = 1; 340 | } 341 | 342 | """.trimIndent(), 343 | null 344 | ) 345 | 346 | val schema = subject.parseSchema(File("."), schemaRegistryClient) 347 | 348 | assert(schema is ProtobufSchema) 349 | } 350 | 351 | @Test 352 | fun `can parse Avro schema`() { 353 | val subject = Persistence.YamlSubject( 354 | "foo", 355 | fromResources("schemas/key.avsc").name, 356 | "AVRO", 357 | null, 358 | null 359 | ) 360 | 361 | val schema = subject.parseSchema(fromResources("schemas"), schemaRegistryClient) 362 | 363 | assert(schema is AvroSchema) 364 | } 365 | 366 | @Test 367 | fun `can parse JSON schema`() { 368 | val subject = Persistence.YamlSubject( 369 | "foo", 370 | null, 371 | "JSON", 372 | """ 373 | { 374 | "type": "object", 375 | "properties": { 376 | "f1": { 377 | "type": "string" 378 | } 379 | } 380 | } 381 | 382 | """.trimIndent(), 383 | null 384 | ) 385 | 386 | val schema = subject.parseSchema(File("."), schemaRegistryClient) 387 | 388 | assert(schema is JsonSchema) 389 | } 390 | 391 | @Test 392 | fun `throws exception when neither file nor schema is set`() { 393 | val subject = Persistence.YamlSubject( 394 | "foo", 395 | null, 396 | "JSON", 397 | null, 398 | null 399 | ) 400 | 401 | assertThrows { 402 | subject.parseSchema(File("."), schemaRegistryClient) 403 | } 404 | } 405 | 406 | @Test 407 | fun `throws exception when schema could not be parsed`() { 408 | val subject = Persistence.YamlSubject( 409 | "foo", 410 | null, 411 | "JSON", 412 | "foobar", 413 | null 414 | ) 415 | 416 | assertThrows { 417 | subject.parseSchema(File("."), schemaRegistryClient) 418 | } 419 | } 420 | } 421 | } 422 | -------------------------------------------------------------------------------- /src/test/resources/client.properties: -------------------------------------------------------------------------------- 1 | schema.registry.url=foo 2 | -------------------------------------------------------------------------------- /src/test/resources/empty.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/domnikl/schema-registry-gitops/0667175bcc1bf3156cedeab85bbc87fa87fd4656/src/test/resources/empty.yml -------------------------------------------------------------------------------- /src/test/resources/neither_schema_nor_file.yml: -------------------------------------------------------------------------------- 1 | subjects: 2 | - name: foo 3 | -------------------------------------------------------------------------------- /src/test/resources/no_compatibility.yml: -------------------------------------------------------------------------------- 1 | subjects: 2 | -------------------------------------------------------------------------------- /src/test/resources/only_compatibility.yml: -------------------------------------------------------------------------------- 1 | compatibility: FORWARD 2 | -------------------------------------------------------------------------------- /src/test/resources/only_normalize.yml: -------------------------------------------------------------------------------- 1 | normalize: true 2 | -------------------------------------------------------------------------------- /src/test/resources/referenced_file_does_not_exist.yml: -------------------------------------------------------------------------------- 1 | subjects: 2 | - name: foo 3 | file: does_not_exist.avsc 4 | -------------------------------------------------------------------------------- /src/test/resources/schemas/avsc.diff: -------------------------------------------------------------------------------- 1 | { 2 | "type" : "record", 3 | - "name" : "HelloWorld", 4 | + "name" : "ByeWorld", 5 | "namespace" : "dev.domnikl.schema_registry_gitops", 6 | - "doc" : "this is some docs to be replaced ...", 7 | "fields" : [ { 8 | - "name" : "greeting", 9 | + "name" : "words", 10 | "type" : "string" 11 | + }, { 12 | + "name" : "foobar", 13 | + "type" : "int" 14 | } ] 15 | } -------------------------------------------------------------------------------- /src/test/resources/schemas/deltaA.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "HelloWorld", 4 | "namespace": "dev.domnikl.schema_registry_gitops", 5 | "doc": "this is some docs to be replaced ...", 6 | "fields": [ 7 | { 8 | "name": "greeting", 9 | "type": "string" 10 | } 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /src/test/resources/schemas/deltaA.json: -------------------------------------------------------------------------------- 1 | { 2 | "$id": "https://example.com/address.schema.json", 3 | "$schema": "https://json-schema.org/draft/2020-12/schema", 4 | "description": "An address similar to http://microformats.org/wiki/h-card", 5 | "type": "object", 6 | "properties": { 7 | "post-office-box": { 8 | "type": "string" 9 | }, 10 | "extended-address": { 11 | "type": "string" 12 | }, 13 | "street-address": { 14 | "type": "string" 15 | }, 16 | "locality": { 17 | "type": "string" 18 | }, 19 | "region": { 20 | "type": "string" 21 | }, 22 | "postal-code": { 23 | "type": "string" 24 | }, 25 | "country-name": { 26 | "type": "string" 27 | } 28 | }, 29 | "required": [ "locality", "region", "country-name" ], 30 | "dependentRequired": { 31 | "post-office-box": [ "street-address" ], 32 | "extended-address": [ "street-address" ] 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/test/resources/schemas/deltaA.proto: -------------------------------------------------------------------------------- 1 | message Person { 2 | required string name = 1; 3 | required int32 id = 2; 4 | optional string email = 3; 5 | } 6 | -------------------------------------------------------------------------------- /src/test/resources/schemas/deltaB.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "ByeWorld", 4 | "namespace": "dev.domnikl.schema_registry_gitops", 5 | "fields": [ 6 | { 7 | "name": "words", 8 | "type": "string" 9 | }, 10 | { 11 | "name": "foobar", 12 | "type": "int" 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /src/test/resources/schemas/deltaB.json: -------------------------------------------------------------------------------- 1 | { 2 | "$id": "https://example.com/address.schema.json", 3 | "$schema": "https://json-schema.org/draft/2020-12/schema", 4 | "description": "An address similar to http://microformats.org/wiki/h-card", 5 | "type": "object", 6 | "properties": { 7 | "post-office-box": { 8 | "type": "string" 9 | }, 10 | "extended-address": { 11 | "type": "string" 12 | }, 13 | "street-address": { 14 | "type": "string" 15 | }, 16 | "locality": { 17 | "type": "string" 18 | }, 19 | "region": { 20 | "type": "string" 21 | }, 22 | "postal-code": { 23 | "type": "string" 24 | }, 25 | "country-name": { 26 | "type": "string" 27 | } 28 | }, 29 | "required": [ "region", "country-name" ], 30 | "dependentRequired": { 31 | "post-office-box": [ "street-address" ] 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/resources/schemas/deltaB.proto: -------------------------------------------------------------------------------- 1 | message Person { 2 | required string name = 1; 3 | required int32 id = 2; 4 | optional string photo = 3; 5 | } 6 | -------------------------------------------------------------------------------- /src/test/resources/schemas/json.diff: -------------------------------------------------------------------------------- 1 | { 2 | "$id" : "https://example.com/address.schema.json", 3 | "$schema" : "https://json-schema.org/draft/2020-12/schema", 4 | "description" : "An address similar to http://microformats.org/wiki/h-card", 5 | "type" : "object", 6 | "properties" : { 7 | "post-office-box" : { 8 | "type" : "string" 9 | }, 10 | "extended-address" : { 11 | "type" : "string" 12 | }, 13 | "street-address" : { 14 | "type" : "string" 15 | }, 16 | "locality" : { 17 | "type" : "string" 18 | }, 19 | "region" : { 20 | "type" : "string" 21 | }, 22 | "postal-code" : { 23 | "type" : "string" 24 | }, 25 | "country-name" : { 26 | "type" : "string" 27 | } 28 | }, 29 | - "required" : [ "locality", "region", "country-name" ], 30 | + "required" : [ "region", "country-name" ], 31 | "dependentRequired" : { 32 | - "post-office-box" : [ "street-address" ], 33 | + "post-office-box" : [ "street-address" ] 34 | - "extended-address" : [ "street-address" ] 35 | } 36 | } -------------------------------------------------------------------------------- /src/test/resources/schemas/key.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "name": "FooKey", 3 | "type": "string" 4 | } 5 | -------------------------------------------------------------------------------- /src/test/resources/schemas/proto.diff: -------------------------------------------------------------------------------- 1 | // Proto schema formatted by Wire, do not edit. 2 | // Source: 3 | 4 | message Person { 5 | required string name = 1; 6 | 7 | required int32 id = 2; 8 | 9 | - optional string email = 3; 10 | + optional string photo = 3; 11 | } 12 | -------------------------------------------------------------------------------- /src/test/resources/schemas/with_subjects.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "HelloWorld", 4 | "namespace": "dev.domnikl.schema_registry_gitops", 5 | "fields": [ 6 | { 7 | "name": "greeting", 8 | "type": "string" 9 | } 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /src/test/resources/schemas/with_subjects_and_references.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "HelloWorld", 4 | "namespace": "dev.domnikl.schema_registry_gitops", 5 | "fields": [ 6 | { 7 | "name": "greeting", 8 | "type": "dev.domnikl.schema_registry_gitops.foo" 9 | } 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /src/test/resources/subject_name_is_missing.yml: -------------------------------------------------------------------------------- 1 | subjects: 2 | - file: with_subjects.avsc 3 | -------------------------------------------------------------------------------- /src/test/resources/version.txt: -------------------------------------------------------------------------------- 1 | test -------------------------------------------------------------------------------- /src/test/resources/with_inline_schema.yml: -------------------------------------------------------------------------------- 1 | subjects: 2 | - name: foo 3 | compatibility: BACKWARD 4 | schema: '{ 5 | "type": "record", 6 | "name": "HelloWorld", 7 | "namespace": "dev.domnikl.schema_registry_gitops", 8 | "fields": [ 9 | { 10 | "name": "greeting", 11 | "type": "string" 12 | } 13 | ] 14 | }' 15 | -------------------------------------------------------------------------------- /src/test/resources/with_subjects.yml: -------------------------------------------------------------------------------- 1 | subjects: 2 | - name: foo 3 | file: with_subjects.avsc 4 | -------------------------------------------------------------------------------- /src/test/resources/with_subjects_and_compatibility.yml: -------------------------------------------------------------------------------- 1 | compatibility: FULL 2 | subjects: 3 | - name: foo 4 | compatibility: FORWARD 5 | file: with_subjects.avsc 6 | -------------------------------------------------------------------------------- /src/test/resources/with_subjects_and_references.yml: -------------------------------------------------------------------------------- 1 | subjects: 2 | - name: bar 3 | file: with_subjects_and_references.avsc 4 | references: 5 | - name: dev.domnikl.schema_registry_gitops.foo 6 | subject: foo 7 | version: 1 8 | --------------------------------------------------------------------------------