├── .codeclimate.yml ├── .github └── workflows │ ├── gradle.yml │ └── release.yml ├── .gitignore ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── build.gradle ├── build.sh ├── codecov.yml ├── docker ├── config │ └── kafka_server_jaas.conf ├── docker-compose.yml └── properties │ ├── admin.properties │ └── gitops-user.properties ├── docs ├── .nojekyll ├── _coverpage.md ├── _sidebar.md ├── confluent-cloud.md ├── documentation.md ├── index.html ├── installation.md ├── permissions.md ├── quick-start.md ├── services.md └── specification.md ├── examples ├── confluent-cloud │ ├── README.md │ ├── services.yaml │ ├── state.yaml │ ├── topics.yaml │ └── users.yaml └── gitops-user │ ├── README.md │ └── state.yaml ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle ├── src ├── main │ └── java │ │ └── com │ │ └── devshawn │ │ └── kafka │ │ └── gitops │ │ ├── MainCommand.java │ │ ├── StateManager.java │ │ ├── cli │ │ ├── AccountCommand.java │ │ ├── ApplyCommand.java │ │ ├── PlanCommand.java │ │ └── ValidateCommand.java │ │ ├── config │ │ ├── KafkaGitopsConfig.java │ │ ├── KafkaGitopsConfigLoader.java │ │ └── ManagerConfig.java │ │ ├── domain │ │ ├── confluent │ │ │ └── ServiceAccount.java │ │ ├── options │ │ │ └── GetAclOptions.java │ │ ├── plan │ │ │ ├── AclPlan.java │ │ │ ├── DesiredPlan.java │ │ │ ├── PlanOverview.java │ │ │ ├── TopicConfigPlan.java │ │ │ ├── TopicDetailsPlan.java │ │ │ └── TopicPlan.java │ │ └── state │ │ │ ├── AbstractService.java │ │ │ ├── AclDetails.java │ │ │ ├── CustomAclDetails.java │ │ │ ├── DesiredState.java │ │ │ ├── DesiredStateFile.java │ │ │ ├── ServiceDetails.java │ │ │ ├── TopicDetails.java │ │ │ ├── UserDetails.java │ │ │ ├── service │ │ │ ├── ApplicationService.java │ │ │ ├── KafkaConnectService.java │ │ │ ├── KafkaConnectStorageTopics.java │ │ │ ├── KafkaConnectorDetails.java │ │ │ └── KafkaStreamsService.java │ │ │ └── settings │ │ │ ├── Settings.java │ │ │ ├── SettingsCCloud.java │ │ │ ├── SettingsFiles.java │ │ │ ├── SettingsServices.java │ │ │ ├── SettingsServicesAcls.java │ │ │ ├── SettingsTopics.java │ │ │ ├── SettingsTopicsBlacklist.java │ │ │ └── SettingsTopicsDefaults.java │ │ ├── enums │ │ └── PlanAction.java │ │ ├── exception │ │ ├── ConfluentCloudException.java │ │ ├── InvalidAclDefinitionException.java │ │ ├── KafkaExecutionException.java │ │ ├── MissingConfigurationException.java │ │ ├── PlanIsUpToDateException.java │ │ ├── ReadPlanInputException.java │ │ ├── ServiceAccountNotFoundException.java │ │ ├── ValidationException.java │ │ └── WritePlanOutputException.java │ │ ├── manager │ │ ├── ApplyManager.java │ │ └── PlanManager.java │ │ ├── service │ │ ├── ConfluentCloudService.java │ │ ├── KafkaService.java │ │ ├── ParserService.java │ │ └── RoleService.java │ │ └── util │ │ ├── HelperUtil.java │ │ ├── LogUtil.java │ │ ├── PlanUtil.java │ │ └── StateUtil.java └── test │ ├── .gitkeep │ ├── groovy │ └── com │ │ └── devshawn │ │ └── kafka │ │ └── gitops │ │ ├── ApplyCommandIntegrationSpec.groovy │ │ ├── PlanCommandIntegrationSpec.groovy │ │ ├── TestUtils.groovy │ │ ├── ValidateCommandIntegrationSpec.groovy │ │ ├── config │ │ └── KafkaGitopsConfigLoaderSpec.groovy │ │ ├── domain │ │ └── state │ │ │ ├── AclDetailsSpec.groovy │ │ │ ├── CustomAclDetailsSpec.groovy │ │ │ ├── ServiceDetailsSpec.groovy │ │ │ └── service │ │ │ └── ApplicationServiceSpec.groovy │ │ └── exception │ │ └── PlanIsUpToDateExceptionSpec.groovy │ └── resources │ ├── command.properties │ └── plans │ ├── application-service-apply-output.txt │ ├── application-service-plan.json │ ├── application-service.yaml │ ├── custom-application-id-streams-apply-output.txt │ ├── custom-application-id-streams-plan.json │ ├── custom-application-id-streams.yaml │ ├── custom-group-id-application-apply-output.txt │ ├── custom-group-id-application-plan.json │ ├── custom-group-id-application.yaml │ ├── custom-group-id-connect-apply-output.txt │ ├── custom-group-id-connect-plan.json │ ├── custom-group-id-connect.yaml │ ├── custom-service-acls-apply-output.txt │ ├── custom-service-acls-plan.json │ ├── custom-service-acls.yaml │ ├── custom-storage-topic-apply-output.txt │ ├── custom-storage-topic-plan.json │ ├── custom-storage-topic.yaml │ ├── custom-storage-topics-apply-output.txt │ ├── custom-storage-topics-plan.json │ ├── custom-storage-topics.yaml │ ├── custom-user-acls-apply-output.txt │ ├── custom-user-acls-plan.json │ ├── custom-user-acls.yaml │ ├── default-replication-multiple-plan.json │ ├── default-replication-multiple.yaml │ ├── default-replication-plan.json │ ├── default-replication.yaml │ ├── describe-topic-acl-disabled-plan.json │ ├── describe-topic-acl-disabled.yaml │ ├── describe-topic-acl-enabled-plan.json │ ├── describe-topic-acl-enabled.yaml │ ├── invalid-custom-service-acls-1-validate-output.txt │ ├── invalid-custom-service-acls-1.yaml │ ├── invalid-custom-service-acls-2-validate-output.txt │ ├── invalid-custom-service-acls-2.yaml │ ├── invalid-custom-user-acls-1-validate-output.txt │ ├── invalid-custom-user-acls-1.yaml │ ├── invalid-custom-user-acls-2-validate-output.txt │ ├── invalid-custom-user-acls-2.yaml │ ├── invalid-default-replication-1-output.txt │ ├── invalid-default-replication-1.yaml │ ├── invalid-default-replication-2-output.txt │ ├── invalid-default-replication-2.yaml │ ├── invalid-format-output.txt │ ├── invalid-format.yaml │ ├── invalid-missing-principal-output.txt │ ├── invalid-missing-principal.yaml │ ├── invalid-missing-user-principal-output.txt │ ├── invalid-missing-user-principal.yaml │ ├── invalid-plan-output.txt │ ├── invalid-plan.json │ ├── invalid-storage-topics-output.txt │ ├── invalid-storage-topics.yaml │ ├── invalid-topic-output.txt │ ├── invalid-topic-remove-partitions-output.txt │ ├── invalid-topic-remove-partitions.yaml │ ├── invalid-topic.yaml │ ├── kafka-connect-service-apply-output.txt │ ├── kafka-connect-service-plan.json │ ├── kafka-connect-service.yaml │ ├── kafka-streams-service-apply-output.txt │ ├── kafka-streams-service-plan.json │ ├── kafka-streams-service.yaml │ ├── multi-file-apply-output.txt │ ├── multi-file-plan.json │ ├── multi-file-services.yaml │ ├── multi-file-topics.yaml │ ├── multi-file-users.yaml │ ├── multi-file.yaml │ ├── no-changes-apply-output.txt │ ├── no-changes-include-unchanged-apply-output.txt │ ├── no-changes-include-unchanged-plan.json │ ├── no-changes-output.txt │ ├── no-changes-plan.json │ ├── no-changes.yaml │ ├── null-file-output.txt │ ├── read-input-exception-output.txt │ ├── seed-acl-exists-apply-output.txt │ ├── seed-acl-exists-plan.json │ ├── seed-acl-exists.yaml │ ├── seed-basic-include-unchanged-plan.json │ ├── seed-basic-plan.json │ ├── seed-basic.yaml │ ├── seed-blacklist-topics-plan.json │ ├── seed-blacklist-topics.yaml │ ├── seed-topic-add-partitions-apply-output.txt │ ├── seed-topic-add-partitions-plan.json │ ├── seed-topic-add-partitions.yaml │ ├── seed-topic-add-replicas-apply-output.txt │ ├── seed-topic-add-replicas-plan.json │ ├── seed-topic-add-replicas.json │ ├── seed-topic-add-replicas.yaml │ ├── seed-topic-modification-2-plan.json │ ├── seed-topic-modification-2.yaml │ ├── seed-topic-modification-3-apply-output.txt │ ├── seed-topic-modification-3-no-delete-apply-output.txt │ ├── seed-topic-modification-3-plan.json │ ├── seed-topic-modification-3.yaml │ ├── seed-topic-modification-apply-output.txt │ ├── seed-topic-modification-no-delete-apply-output.txt │ ├── seed-topic-modification-no-delete-plan.json │ ├── seed-topic-modification-no-delete.yaml │ ├── seed-topic-modification-plan.json │ ├── seed-topic-modification.yaml │ ├── seed-topic-remove-replicas-plan.json │ ├── seed-topic-remove-replicas.yaml │ ├── simple-apply-output.txt │ ├── simple-plan.json │ ├── simple-users-apply-output.txt │ ├── simple-users-plan.json │ ├── simple-users.yaml │ ├── simple.yaml │ ├── skip-acls-apply-apply-output.txt │ ├── skip-acls-apply-plan.json │ ├── skip-acls-plan.json │ ├── skip-acls.yaml │ ├── topics-and-services-plan.json │ ├── topics-and-services.yaml │ ├── unrecognized-property-output.txt │ └── unrecognized-property.yaml └── stub.sh /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | checks: 3 | method-count: 4 | config: 5 | threshold: 40 6 | similar-code: 7 | config: 8 | threshold: 60 -------------------------------------------------------------------------------- /.github/workflows/gradle.yml: -------------------------------------------------------------------------------- 1 | name: Java CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - '**' 7 | tags-ignore: 8 | - '**' 9 | 10 | jobs: 11 | build: 12 | 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v1 17 | - name: Set up JDK 1.8 18 | uses: actions/setup-java@v1 19 | with: 20 | java-version: 1.8 21 | 22 | - name: Start docker compose for integration tests 23 | run: docker-compose -f docker/docker-compose.yml up -d 24 | 25 | - name: Build with Gradle 26 | run: ./gradlew --info build jacocoTestReport 27 | 28 | - name: Upload coverage to Codecov 29 | uses: codecov/codecov-action@v1 30 | with: 31 | file: ./build/reports/jacoco/report.xml 32 | 33 | - name: Upload build artifact 34 | uses: actions/upload-artifact@v1 35 | with: 36 | name: kafka-gitops.jar 37 | path: build/libs/kafka-gitops-all.jar 38 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | tags: 4 | - '*' 5 | 6 | name: Release 7 | 8 | jobs: 9 | build: 10 | name: Build Release 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v1 14 | - name: Set up JDK 1.8 15 | uses: actions/setup-java@v1 16 | with: 17 | java-version: 1.8 18 | 19 | - name: Build with Gradle 20 | run: ./gradlew buildRelease 21 | 22 | - name: Get the version 23 | id: get_version 24 | run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//} 25 | 26 | - name: Build & Push Docker Image 27 | uses: docker/build-push-action@v1 28 | with: 29 | image: devshawn/kafka-gitops 30 | tags: ${{ steps.get_version.outputs.VERSION }},latest 31 | registry: docker.io 32 | username: ${{ secrets.DOCKER_USERNAME }} 33 | password: ${{ secrets.DOCKER_PASSWORD }} 34 | push: ${{ startsWith(github.ref, 'refs/tags/') }} 35 | 36 | - name: Create Release 37 | id: create_release 38 | uses: actions/create-release@v1.0.0 39 | env: 40 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 41 | with: 42 | tag_name: ${{ github.ref }} 43 | release_name: Release ${{ github.ref }} 44 | draft: true 45 | prerelease: false 46 | 47 | - name: Upload Release Asset 48 | id: upload-release-asset 49 | uses: actions/upload-release-asset@v1.0.1 50 | env: 51 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 52 | with: 53 | upload_url: ${{ steps.create_release.outputs.upload_url }} 54 | asset_path: ./build/distributions/kafka-gitops.zip 55 | asset_name: kafka-gitops.zip 56 | asset_content_type: application/zip 57 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .classpath 2 | .factorypath 3 | .project 4 | .settings/ 5 | bin/ 6 | .idea/ 7 | *.iml 8 | .gradle/ 9 | build/ 10 | out/ 11 | docker/data/ 12 | state.yaml 13 | plan.json 14 | test.py 15 | /generated/ 16 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Contributions are very welcome! Any existing issues labeled ["help wanted"](https://github.com/devshawn/kafka-gitops/labels/help%20wanted) or ["good first issue"](https://github.com/devshawn/kafka-gitops/labels/good%20first%20issue) are free to be pursued. 4 | 5 | ## Feature Requests & Bug Reports 6 | For feature requests and bug reports, [submit an issue][issues]. 7 | 8 | ## Pull Requests 9 | The preferred way to contribute is to fork the [main repository][repository] on GitHub. 10 | 11 | 1. Discuss your proposed change in a GitHub issue first before spending time and implementing a feature or fix. 12 | 13 | 2. Ensure all changes are relevant to the pull request. Keep pull requests as small and to-the-point as possible. 14 | 15 | 3. Add & modify tests as necessary. Also, ensure the code matches the existing style. 16 | 17 | 4. Once changes are completed, open a pull request for review against the master branch. 18 | 19 | 20 | [repository]: https://github.com/devshawn/kafka-gitops 21 | [issues]: https://github.com/devshawn/kafka-gitops/issues 22 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-jre-slim 2 | 3 | RUN apt-get update && apt-get --yes upgrade && \ 4 | apt-get install -y python3 python3-pip curl && \ 5 | rm -rf /var/lib/apt/lists/* 6 | 7 | COPY ./build/output/kafka-gitops /usr/local/bin/kafka-gitops -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | id 'groovy' 4 | id 'application' 5 | id 'idea' 6 | id 'jacoco' 7 | id 'org.inferred.processors' version '1.2.10' 8 | id "net.ltgt.apt" version "0.21" 9 | id 'com.github.johnrengelman.shadow' version '4.0.4' 10 | } 11 | 12 | apply plugin: 'net.ltgt.apt-idea' 13 | 14 | group 'com.devshawn' 15 | 16 | mainClassName = 'com.devshawn.kafka.gitops.MainCommand' 17 | 18 | sourceCompatibility = 1.8 19 | 20 | repositories { 21 | mavenCentral() 22 | } 23 | 24 | dependencies { 25 | compile group: 'org.apache.kafka', name: 'kafka-clients', version: '2.4.0' 26 | compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.1' 27 | compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.8" 28 | compile "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.10.2" 29 | compile 'info.picocli:picocli:4.1.4' 30 | 31 | compile 'org.slf4j:slf4j-api:1.7.30' 32 | compile group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3' 33 | compile group: 'ch.qos.logback', name: 'logback-core', version: '1.2.3' 34 | 35 | processor 'org.inferred:freebuilder:2.5.0' 36 | 37 | testCompile group: 'junit', name: 'junit', version: '4.12' 38 | testCompile group: 'org.codehaus.groovy', name: 'groovy-all', version: '2.4.4' 39 | testCompile group: 'org.spockframework', name: 'spock-core', version: '1.0-groovy-2.4' 40 | testCompile group: 'cglib', name: 'cglib-nodep', version: '2.2' 41 | testCompile group: 'com.github.stefanbirkner', name: 'system-rules', version: '1.19.0' 42 | testCompile group: 'org.skyscreamer', name: 'jsonassert', version: '1.5.0' 43 | } 44 | 45 | jacocoTestReport { 46 | reports { 47 | xml.enabled = true 48 | html.enabled = true 49 | xml.destination "${buildDir}/reports/jacoco/report.xml" 50 | } 51 | 52 | afterEvaluate { 53 | classDirectories = files(classDirectories.files.collect { 54 | fileTree(dir: it, 55 | excludes: [ 56 | '**/*_Builder*/**', 57 | ]) 58 | }) 59 | } 60 | } 61 | 62 | jar { 63 | manifest { 64 | attributes( 65 | 'Class-Path': configurations.compile.collect { it.getName() }.join(' '), 66 | 'Main-Class': 'com.devshawn.kafka.gitops.MainCommand' 67 | ) 68 | } 69 | } 70 | 71 | task buildExecutableJar(type: Exec) { 72 | commandLine "sh", "build.sh" 73 | } 74 | 75 | task buildRelease(type: Zip, group: "build") { 76 | from("$buildDir/output") 77 | } 78 | 79 | buildRelease.dependsOn buildExecutableJar 80 | buildExecutableJar.dependsOn shadowJar -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | mkdir -p build/output 4 | 5 | cat stub.sh ./build/libs/kafka-gitops-all.jar > build/output/kafka-gitops 6 | chmod +x build/output/kafka-gitops -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | patch: 4 | default: 5 | enabled: no -------------------------------------------------------------------------------- /docker/config/kafka_server_jaas.conf: -------------------------------------------------------------------------------- 1 | KafkaServer { 2 | org.apache.kafka.common.security.plain.PlainLoginModule required 3 | username="kafka" 4 | password="kafka-secret" 5 | user_kafka="kafka-secret" 6 | user_test="test-secret" 7 | user_gitops-user="gitops-secret"; 8 | }; 9 | 10 | KafkaClient { 11 | org.apache.kafka.common.security.plain.PlainLoginModule required 12 | username="test" 13 | password="test-secret"; 14 | }; -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | 3 | services: 4 | zoo1: 5 | image: zookeeper:3.4.9 6 | hostname: zoo1 7 | ports: 8 | - "2181:2181" 9 | environment: 10 | ZOO_MY_ID: 1 11 | ZOO_PORT: 2181 12 | ZOO_SERVERS: server.1=zoo1:2888:3888 13 | volumes: 14 | - ./data/zoo1/data:/data 15 | - ./data/zoo1/datalog:/datalog 16 | 17 | kafka1: 18 | image: confluentinc/cp-kafka:5.5.3 19 | hostname: kafka1 20 | ports: 21 | - "9092:9092" 22 | environment: 23 | KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 24 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:SASL_PLAINTEXT,LISTENER_DOCKER_EXTERNAL:SASL_PLAINTEXT 25 | KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL 26 | KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" 27 | KAFKA_BROKER_ID: 1 28 | KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf" 29 | KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" 30 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 31 | KAFKA_SASL_ENABLED_MECHANISMS: PLAIN 32 | KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN 33 | ZOOKEEPER_SASL_ENABLED: "false" 34 | KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.auth.SimpleAclAuthorizer" 35 | KAFKA_SUPER_USERS: "User:test;User:kafka" 36 | KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" 37 | volumes: 38 | - ./data/kafka1/data:/var/lib/kafka/data 39 | - ./config/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf 40 | depends_on: 41 | - zoo1 42 | 43 | kafka2: 44 | image: confluentinc/cp-kafka:5.5.3 45 | hostname: kafka2 46 | ports: 47 | - "9093:9093" 48 | environment: 49 | KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093 50 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:SASL_PLAINTEXT,LISTENER_DOCKER_EXTERNAL:SASL_PLAINTEXT 51 | KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL 52 | KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" 53 | KAFKA_BROKER_ID: 2 54 | KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf" 55 | KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" 56 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 57 | KAFKA_SASL_ENABLED_MECHANISMS: PLAIN 58 | KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN 59 | ZOOKEEPER_SASL_ENABLED: "false" 60 | KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.auth.SimpleAclAuthorizer" 61 | KAFKA_SUPER_USERS: "User:test;User:kafka" 62 | KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" 63 | volumes: 64 | - ./data/kafka2/data:/var/lib/kafka/data 65 | - ./config/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf 66 | depends_on: 67 | - zoo1 68 | 69 | kafka3: 70 | image: confluentinc/cp-kafka:5.5.3 71 | hostname: kafka3 72 | ports: 73 | - "9094:9094" 74 | environment: 75 | KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094 76 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:SASL_PLAINTEXT,LISTENER_DOCKER_EXTERNAL:SASL_PLAINTEXT 77 | KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL 78 | KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" 79 | KAFKA_BROKER_ID: 3 80 | KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf" 81 | KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" 82 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 83 | KAFKA_SASL_ENABLED_MECHANISMS: PLAIN 84 | KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN 85 | ZOOKEEPER_SASL_ENABLED: "false" 86 | KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.auth.SimpleAclAuthorizer" 87 | KAFKA_SUPER_USERS: "User:test;User:kafka" 88 | KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" 89 | volumes: 90 | - ./data/kafka3/data:/var/lib/kafka/data 91 | - ./config/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf 92 | depends_on: 93 | - zoo1 94 | 95 | -------------------------------------------------------------------------------- /docker/properties/admin.properties: -------------------------------------------------------------------------------- 1 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="test" password="test-secret"; 2 | sasl.mechanism=PLAIN 3 | security.protocol=SASL_PLAINTEXT -------------------------------------------------------------------------------- /docker/properties/gitops-user.properties: -------------------------------------------------------------------------------- 1 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="gitops-user" password="gitops-secret"; 2 | sasl.mechanism=PLAIN 3 | security.protocol=SASL_PLAINTEXT -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devshawn/kafka-gitops/8f3afc93f386ce54e6974194375bac8db6a6a354/docs/.nojekyll -------------------------------------------------------------------------------- /docs/_coverpage.md: -------------------------------------------------------------------------------- 1 | 2 | # kafka-gitops 3 | 4 | > GitOps for Apache Kafka 5 | 6 | - Automated Topic & ACL Management 7 | - Manage topics, services, and ACLs with desired state files 8 | 9 | [Documentation](/documentation.md) 10 | [GitHub](https://github.com/devshawn/kafka-gitops) -------------------------------------------------------------------------------- /docs/_sidebar.md: -------------------------------------------------------------------------------- 1 | - **Getting Started** 2 | - [Introduction](/documentation.md) 3 | - [Installation](/installation.md) 4 | - [Quick Start](/quick-start.md) 5 | - [Services](/services.md) 6 | - [Confluent Cloud](/confluent-cloud.md) 7 | - [Permissions](/permissions.md) 8 | - [Specification](/specification.md) 9 | - **Links** 10 | - [Contributing](https://github.com/devshawn/kafka-gitops/blob/master/CONTRIBUTING.md) 11 | - [GitHub](https://github.com/devshawn/kafka-gitops) -------------------------------------------------------------------------------- /docs/confluent-cloud.md: -------------------------------------------------------------------------------- 1 | # Confluent Cloud 2 | 3 | This tool was designed to work with Confluent Cloud. It can manage service accounts, topics, and ACLs for Confluent Cloud clusters. 4 | 5 | ## Getting Started 6 | 7 | Ensure you have installed `kafka-gitops` or are using the `kafka-gitops` docker image as described in the [installation][installation] instructions. 8 | 9 | You must have the `ccloud` command line tools installed if you wish to auto-populate the `principal` fields on services. 10 | 11 | ## Desired State File 12 | 13 | Create a basic desired state file, `state.yaml`, such as: 14 | 15 | ```yaml 16 | settings: 17 | ccloud: 18 | enabled: true 19 | 20 | topics: 21 | test-topic: 22 | partitions: 6 23 | replication: 3 24 | 25 | services: 26 | test-service: 27 | type: application 28 | produces: 29 | - test-topic 30 | ``` 31 | 32 | To give an overview, throughout this guide, this will create: 33 | 34 | - A topic named `test-topic` 35 | - A service account named `test-service` 36 | - An `WRITE` ACL for topic `test-topic` tied to the service account `test-service` 37 | 38 | ## Configuration 39 | 40 | To use `kafka-gitops` with Confluent Cloud, you'll need to set a few environment variables. 41 | 42 | * `KAFKA_BOOTSTRAP_SERVERS`: Your Confluent Cloud cluster URL 43 | * `KAFKA_SASL_JAAS_USERNAME`: Your Confluent Cloud API key 44 | * `KAFKA_SASL_JAAS_PASSWORD`: Your Confluent Cloud API secret 45 | * `KAFKA_SECURITY_PROTOCOL`: `SASL_SSL` 46 | * `KAFKA_SASL_MECHANISM`: `PLAIN` 47 | * `KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM`: `HTTPS` 48 | 49 | Additionally, you'll need to login to the `ccloud` tool. You can automate this by setting the following environment variables: 50 | 51 | * `XX_CCLOUD_EMAIL`: Your Confluent Cloud administrator email 52 | * `XX_CCLOUD_PASSWORD`: Your Confluent Cloud administrator password 53 | 54 | Then, you can run `ccloud login` and it will run without a prompt. This is great for CI builds. 55 | 56 | You can optionally specify a path to a `ccloud` executable: 57 | 58 | * `CCLOUD_EXECUTABLE_PATH`: `/full/path/to/ccloud` 59 | 60 | Otherwise, `ccloud` must be on your path. 61 | 62 | ## Validate 63 | 64 | First, validate your state file is correct by running: 65 | 66 | ```bash 67 | kafka-gitops -f state.yaml validate 68 | ``` 69 | 70 | An example success message would look like: 71 | 72 | ```text 73 | [VALID] Successfully validated the desired state file. 74 | ``` 75 | 76 | ## Accounts 77 | 78 | Before generating an execution plan, you will need to create the service accounts. This can be done by running: 79 | 80 | ```bash 81 | kafka-gitops -f state.yaml account 82 | ``` 83 | 84 | This currently only creates service accounts; it will not delete any. 85 | 86 | ## Plan 87 | 88 | We're now ready to generate a plan to execute against the cluster. By using the plan command, we are **NOT** changing the cluster. 89 | 90 | Generate a plan by running: 91 | 92 | ```bash 93 | kafka-gitops -f state.yaml plan -o plan.json 94 | ``` 95 | 96 | This will generate an execution plan to run against the cluster and save the output to `plan.json`. 97 | 98 | The command will also pretty-print what changes it wants to make to the cluster. 99 | 100 | ## Apply 101 | 102 | To execute a plan against the cluster, we use the apply command. 103 | 104 | !> **WARNING**: This will apply changes to the cluster. This can be potentially destructive if you do not have all topics and ACLs defined. 105 | 106 | By default, the apply command will generate a plan and then apply it. For most situations, **you should output a plan file from the plan command and pass it to the apply command**. 107 | 108 | Example: 109 | 110 | ```bash 111 | kafka-gitops -f state.yaml apply -p plan.json 112 | ``` 113 | 114 | Congrats! You're now using `kafka-gitops` to manage your Confluent Cloud cluster. Once you've practiced locally, you should commit your state file to a repository and create CI/CD pipelines to create plans and execute them against the cluster. 115 | 116 | Welcome to GitOps! 117 | 118 | [installation]: /installation.md 119 | -------------------------------------------------------------------------------- /docs/documentation.md: -------------------------------------------------------------------------------- 1 |
2 |

GitOps for Apache Kafka

3 | 4 | Manage Apache Kafka topics, services, and ACLs through a desired state file. 5 |
6 | 7 | ## Overview 8 | 9 | Kafka GitOps is an Apache Kafka resources-as-code tool which allows you to automate the management of your Apache Kafka topics and ACLs from version controlled code. It allows you to define topics and services through the use of a desired state file, much like Terraform and other infrastructure-as-code tools. 10 | 11 | Topics and services get defined in a YAML file. When run, `kafka-gitops` compares your desired state to the actual state of the cluster and generates a plan to execute against the cluster. This will make your topics and ACLs match your desired state. 12 | 13 | This tool also generates the needed ACLs for each type of application. There is no need to manually create a bunch of ACLs for Kafka Connect, Kafka Streams, etc. By defining your services, `kafka-gitops` will build the necessary ACLs. 14 | 15 | ## Features 16 | 17 | - 🚀 **Built For CI/CD**: Made for CI/CD pipelines to automate the management of topics & ACLs. 18 | - 🔥 **Configuration as code**: Describe your desired state and manage it from a version-controlled declarative file. 19 | - 👍 **Easy to use**: Deep knowledge of Kafka administration or ACL management is **NOT** required. 20 | - ⚡️️ **Plan & Apply**: Generate and view a plan with or without executing it against your cluster. 21 | - 💻 **Portable**: Works across self-hosted clusters, managed clusters, and even Confluent Cloud clusters. 22 | - 🦄 **Idempotency**: Executing the same desired state file on an up-to-date cluster will yield the same result. 23 | - ☀️ **Continue from failures**: If a specific step fails during an apply, you can fix your desired state and re-run the command. You can execute `kafka-gitops` again without needing to rollback any partial successes. 24 | 25 | ## Getting Started 26 | 27 | Check out the **[Quick Start](/quick-start.md)** documentation to install `kafka-gitops` and get started. 28 | 29 | ## Ideology 30 | 31 | The idea behind this project is to manage Kafka topics and ACLs through desired state files. These files define what your cluster should look like and `kafka-gitops` modifies your cluster's actual state to match the desired state file. Typically, this looks like: 32 | 33 | - State files are stored in version control, such as a git repository. 34 | - Developers add, change, and remove topics & services from files and open a PR. 35 | - Operations teams valdiate and merge the PR. 36 | - A CI/CD pipeline generates a plan using `kafka-gitops`. 37 | - A human then validates the plan and applies the changes to the cluster. 38 | 39 | ## Contributing 40 | 41 | Contributions are very welcome. See [CONTRIBUTING.md][contributing] for details. 42 | 43 | ## License 44 | 45 | Copyright (c) 2020 Shawn Seymour. 46 | 47 | Licensed under the [Apache 2.0 license][license]. 48 | 49 | [contributing]: https://github.com/devshawn/kafka-gitops/blob/master/CONTRIBUTING.md 50 | [license]: https://github.com/devshawn/kafka-gitops/blob/master/LICENSE 51 | -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Kafka GitOps 6 | 7 | 8 | 9 | 10 | 11 | 12 |
13 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | Installing `kafka-gitops` requires either **Java** or **Docker**. 4 | 5 | ## Homebrew 6 | 7 | Install `kafka-gitops` with homebrew: 8 | 9 | ```bash 10 | brew tap devshawn/kafka-gitops 11 | brew install kafka-gitops 12 | ``` 13 | 14 | ## Local 15 | 16 | Install `kafka-gitops` by downloading the zip file from our [releases][releases] page. Move the `kafka-gitops` file to somewhere on your `$PATH`, such as `/usr/local/bin`. 17 | 18 | 19 | Ensure the command is working by running: 20 | 21 | ```bash 22 | kafka-gitops 23 | ``` 24 | 25 | You should see output similar to the following: 26 | 27 | ```text 28 | Usage: kafka-gitops [-hvV] [--no-delete] [-f=] [COMMAND] 29 | Manage Kafka resources with a desired state file. 30 | -f, --file= Specify the desired state file. 31 | -h, --help Display this help message. 32 | --no-delete Disable the ability to delete resources. 33 | -v, --verbose Show more detail during execution. 34 | -V, --version Print the current version of this tool. 35 | Commands: 36 | account Create Confluent Cloud service accounts. 37 | apply Apply changes to Kafka resources. 38 | plan Generate an execution plan of changes to Kafka resources. 39 | validate Validates the desired state file. 40 | ``` 41 | 42 | ## Docker 43 | 44 | We provide a public docker image: [devshawn/kafka-gitops][docker]. 45 | 46 | [releases]: https://github.com/devshawn/kafka-gitops/releases 47 | [docker]: https://hub.docker.com/r/devshawn/kafka-gitops -------------------------------------------------------------------------------- /docs/permissions.md: -------------------------------------------------------------------------------- 1 | # Permissions 2 | 3 | When running against a secured Kafka cluster, `kafka-gitops` needs to be authorized to perform actions against the cluster. This can either be a super user defined by the Kafka cluster or a custom user with specific permissions. 4 | 5 | ## Example 6 | 7 | For the purposes of this example, we'll assume we have a user principal named `gitops-user`. 8 | 9 | Full usage of kafka-gitops means you are managing topics, services, ACLs, and users. If you plan to make use of our ACL management features, the `gitops-user` principal must have the ability to create and manage ACLs. 10 | 11 | If you do not want to use a super user, you can create a `gitops-user` principal and a current super user can make them an *ACL Administrator*. An ACL administrator has the `ALTER --cluster` access control entry. This entry allows the user to create and delete ACLs for the given cluster. 12 | 13 | !> **Caution**: An ACL administrator can then create ACLs for any other principal, including themselves. 14 | 15 | ### Manually Add ACLs 16 | Add the alter cluster ACL to the `gitops-user` principal: 17 | 18 | ```bash 19 | kafka-acls --bootstrap-server localhost:9092 --command-config admin.properties \ 20 | --add --allow-principal User:gitops-user \ 21 | --operation ALTER --cluster 22 | ``` 23 | 24 | Add the ACLs needed to manage topics to the `gitops-user` principal: 25 | 26 | ```bash 27 | kafka-acls --bootstrap-server localhost:9092 --command-config admin.properties --add \ 28 | --allow-principal User:gitops-user --operation Create --operation Delete \ 29 | --operation DescribeConfigs --operation AlterConfigs --operation Alter \ 30 | --operation Describe --topic '*' 31 | ``` 32 | 33 | The above configs allow the `gitops-user` to manage ACLs, topics, and topic configurations. 34 | 35 | ### State File Definition 36 | You can also create the ACLs using kafka-gitops. Run it once with super admin credentials using the state file below, and then switch to using your `gitops-user` credentials. 37 | 38 | ```yaml 39 | users: 40 | gitops-user: 41 | principal: User:gitops-user 42 | 43 | customUserAcls: 44 | gitops-user: 45 | alter-cluster: 46 | name: kafka-cluster 47 | type: CLUSTER 48 | pattern: LITERAL 49 | host: "*" 50 | operation: ALTER 51 | permission: ALLOW 52 | create-topics: 53 | name: "*" 54 | type: TOPIC 55 | pattern: LITERAL 56 | host: "*" 57 | operation: CREATE 58 | permission: ALLOW 59 | alter-topics: 60 | name: "*" 61 | type: TOPIC 62 | pattern: LITERAL 63 | host: "*" 64 | operation: ALTER 65 | permission: ALLOW 66 | describe-topics: 67 | name: "*" 68 | type: TOPIC 69 | pattern: LITERAL 70 | host: "*" 71 | operation: DESCRIBE 72 | permission: ALLOW 73 | delete-topics: 74 | name: "*" 75 | type: TOPIC 76 | pattern: LITERAL 77 | host: "*" 78 | operation: DELETE 79 | permission: ALLOW 80 | describe-topic-configs: 81 | name: "*" 82 | type: TOPIC 83 | pattern: LITERAL 84 | host: "*" 85 | operation: DESCRIBE_CONFIGS 86 | permission: ALLOW 87 | alter-topic-configs: 88 | name: "*" 89 | type: TOPIC 90 | pattern: LITERAL 91 | host: "*" 92 | operation: ALTER_CONFIGS 93 | permission: ALLOW 94 | ``` 95 | 96 | -------------------------------------------------------------------------------- /examples/confluent-cloud/README.md: -------------------------------------------------------------------------------- 1 | # Confluent Cloud Example 2 | 3 | This is a basic example of using `kafka-gitops` with Confluent Cloud. 4 | 5 | ## Files 6 | - `services.yaml` defines service accounts + generates ACLs 7 | - `topics.yaml` defines topics and their configs 8 | - `users.yaml` defines "user" service accounts and roles (prefixed with `user-{name}`) 9 | 10 | ## Configuration 11 | 12 | The following environment variables need to be set locally or in the build job: 13 | 14 | ```bash 15 | # For service account creation / listing 16 | export XX_CCLOUD_EMAIL="Your Confluent Cloud email address" 17 | export XX_CCLOUD_PASSWORD="Your Confluent Cloud password" 18 | 19 | # For executing against the cluster 20 | export KAFKA_BOOTSTRAP_SERVERS="Your Confluent Cloud cluster URL" 21 | export KAFKA_SASL_JAAS_USERNAME"Your Confluent Cloud API key" 22 | export KAFKA_SASL_JAAS_PASSWORD="Your Confluent Cloud API secret" 23 | export KAFKA_SECURITY_PROTOCOL="SASL_SSL" 24 | export KAFKA_SASL_MECHANISM="PLAIN" 25 | export KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM="HTTPS" 26 | 27 | # For colored output 28 | export CLICOLOR_FORCE="true" 29 | ``` 30 | 31 | ## Service Accounts 32 | 33 | Once defining services and users, you can generate service accounts. 34 | 35 | **NOTE**: Before running `accounts` or `plan`, ensure you are logged in to Confluent Cloud. Use `ccloud login`. 36 | 37 | Create accounts: 38 | 39 | ```bash 40 | kafka-gitops -f state.yaml accounts 41 | ``` 42 | 43 | ## Plan 44 | 45 | Generate an execution plan against your Confluent Cloud cluster as so: 46 | 47 | ```bash 48 | kafka-gitops -f state.yaml plan -o plan.json 49 | ``` 50 | 51 | To disable deletes, you can use: 52 | 53 | ```bash 54 | kafka-gitops -f state.yaml --no-delete plan -o plan.json 55 | ``` 56 | 57 | This will output a plan file as well as pretty print the plan to standard out. 58 | 59 | ## Apply 60 | 61 | Apply the plan against your Confluent Cloud cluster as so: 62 | 63 | ```bash 64 | kafka-gitops -f state.yaml apply -p plan.json 65 | ``` 66 | 67 | 68 | To disable deletes, you can use: 69 | 70 | ```bash 71 | kafka-gitops -f state.yaml --no-delete apply -p plan.json 72 | ``` -------------------------------------------------------------------------------- /examples/confluent-cloud/services.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | my-test-service: 3 | type: application 4 | consumes: 5 | - my-test-topic -------------------------------------------------------------------------------- /examples/confluent-cloud/state.yaml: -------------------------------------------------------------------------------- 1 | settings: 2 | ccloud: 3 | enabled: true 4 | files: 5 | topics: topics.yaml 6 | services: services.yaml 7 | users: users.yaml 8 | topics: 9 | blacklist: 10 | prefixed: 11 | - _confluent -------------------------------------------------------------------------------- /examples/confluent-cloud/topics.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | my-test-topic: 3 | partitions: 6 4 | replication: 3 -------------------------------------------------------------------------------- /examples/confluent-cloud/users.yaml: -------------------------------------------------------------------------------- 1 | users: 2 | john-doe: 3 | roles: 4 | - operator -------------------------------------------------------------------------------- /examples/gitops-user/README.md: -------------------------------------------------------------------------------- 1 | # GitOps User Example 2 | 3 | This is a basic example of using `kafka-gitops` to generate ACLs needed for a kafka-gitops user principal within Kafka. 4 | 5 | -------------------------------------------------------------------------------- /examples/gitops-user/state.yaml: -------------------------------------------------------------------------------- 1 | users: 2 | gitops-user: 3 | principal: User:gitops-user 4 | roles: [] 5 | 6 | customUserAcls: 7 | gitops-user: 8 | alter-cluster: 9 | name: kafka-cluster 10 | type: CLUSTER 11 | pattern: LITERAL 12 | host: "*" 13 | operation: ALTER 14 | permission: ALLOW 15 | create-topics: 16 | name: "*" 17 | type: TOPIC 18 | pattern: LITERAL 19 | host: "*" 20 | operation: CREATE 21 | permission: ALLOW 22 | alter-topics: 23 | name: "*" 24 | type: TOPIC 25 | pattern: LITERAL 26 | host: "*" 27 | operation: ALTER 28 | permission: ALLOW 29 | describe-topics: 30 | name: "*" 31 | type: TOPIC 32 | pattern: LITERAL 33 | host: "*" 34 | operation: DESCRIBE 35 | permission: ALLOW 36 | delete-topics: 37 | name: "*" 38 | type: TOPIC 39 | pattern: LITERAL 40 | host: "*" 41 | operation: DELETE 42 | permission: ALLOW 43 | describe-topic-configs: 44 | name: "*" 45 | type: TOPIC 46 | pattern: LITERAL 47 | host: "*" 48 | operation: DESCRIBE_CONFIGS 49 | permission: ALLOW 50 | alter-topic-configs: 51 | name: "*" 52 | type: TOPIC 53 | pattern: LITERAL 54 | host: "*" 55 | operation: ALTER_CONFIGS 56 | permission: ALLOW 57 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devshawn/kafka-gitops/8f3afc93f386ce54e6974194375bac8db6a6a354/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.10-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-gitops' 2 | 3 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/MainCommand.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops; 2 | 3 | import com.devshawn.kafka.gitops.cli.AccountCommand; 4 | import com.devshawn.kafka.gitops.cli.ApplyCommand; 5 | import com.devshawn.kafka.gitops.cli.PlanCommand; 6 | import com.devshawn.kafka.gitops.cli.ValidateCommand; 7 | import picocli.CommandLine; 8 | import picocli.CommandLine.Command; 9 | import picocli.CommandLine.Option; 10 | 11 | import java.io.File; 12 | import java.util.concurrent.Callable; 13 | 14 | @Command(name = "kafka-gitops", 15 | version = "0.2.15", 16 | exitCodeOnInvalidInput = 0, 17 | subcommands = { 18 | AccountCommand.class, 19 | ApplyCommand.class, 20 | PlanCommand.class, 21 | ValidateCommand.class 22 | }, 23 | description = "Manage Kafka resources with a desired state file.") 24 | public class MainCommand implements Callable { 25 | 26 | @Option(names = {"-c", "--command-config"}, paramLabel = "", 27 | description = "Command config properties file.") 28 | private File configFile; 29 | 30 | @Option(names = {"-f", "--file"}, paramLabel = "", 31 | description = "Specify the desired state file.", defaultValue = "state.yaml") 32 | private File stateFile; 33 | 34 | @Option(names = {"--no-delete"}, description = "Disable the ability to delete resources.") 35 | private boolean deleteDisabled = false; 36 | 37 | @Option(names = {"--skip-acls"}, description = "Do not take ACLs into account during plans or applies.") 38 | private boolean skipAcls = false; 39 | 40 | @Option(names = {"-h", "--help"}, usageHelp = true, description = "Display this help message.") 41 | private boolean helpRequested = false; 42 | 43 | @Option(names = {"-v", "--verbose"}, description = "Show more detail during execution.") 44 | private boolean verboseRequested = false; 45 | 46 | @Option(names = {"-V", "--version"}, versionHelp = true, description = "Print the current version of this tool.") 47 | private boolean versionRequested = false; 48 | 49 | @CommandLine.Spec 50 | CommandLine.Model.CommandSpec spec; 51 | 52 | @Override 53 | public Integer call() { 54 | CommandLine commandLine = spec.commandLine(); 55 | if (commandLine.isVersionHelpRequested()) { 56 | spec.commandLine().printVersionHelp(commandLine.getOut()); 57 | return 0; 58 | } 59 | 60 | throw new CommandLine.ParameterException(spec.commandLine(), ""); 61 | } 62 | 63 | public boolean isVerboseRequested() { 64 | return verboseRequested; 65 | } 66 | 67 | public File getConfigFile() { 68 | return configFile; 69 | } 70 | 71 | public File getStateFile() { 72 | return stateFile; 73 | } 74 | 75 | public boolean isDeleteDisabled() { 76 | return deleteDisabled; 77 | } 78 | 79 | public boolean areAclsDisabled() { 80 | return skipAcls; 81 | } 82 | 83 | public static void main(String[] args) { 84 | int exitCode = new CommandLine(new MainCommand()).execute(args); 85 | System.exit(exitCode); 86 | } 87 | } -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/cli/AccountCommand.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.cli; 2 | 3 | import com.devshawn.kafka.gitops.MainCommand; 4 | import com.devshawn.kafka.gitops.StateManager; 5 | import com.devshawn.kafka.gitops.config.ManagerConfig; 6 | import com.devshawn.kafka.gitops.exception.ConfluentCloudException; 7 | import com.devshawn.kafka.gitops.exception.KafkaExecutionException; 8 | import com.devshawn.kafka.gitops.exception.MissingConfigurationException; 9 | import com.devshawn.kafka.gitops.exception.ValidationException; 10 | import com.devshawn.kafka.gitops.exception.WritePlanOutputException; 11 | import com.devshawn.kafka.gitops.service.ParserService; 12 | import com.devshawn.kafka.gitops.util.LogUtil; 13 | import picocli.CommandLine; 14 | 15 | import java.util.concurrent.Callable; 16 | 17 | @CommandLine.Command(name = "account", description = "Create Confluent Cloud service accounts.") 18 | public class AccountCommand implements Callable { 19 | 20 | @CommandLine.ParentCommand 21 | private MainCommand parent; 22 | 23 | @Override 24 | public Integer call() { 25 | try { 26 | System.out.println("Creating service accounts...\n"); 27 | ParserService parserService = new ParserService(parent.getStateFile()); 28 | StateManager stateManager = new StateManager(generateStateManagerConfig(), parserService); 29 | stateManager.createServiceAccounts(); 30 | return 0; 31 | } catch (MissingConfigurationException | ConfluentCloudException ex) { 32 | LogUtil.printSimpleError(ex.getMessage()); 33 | } catch (ValidationException ex) { 34 | LogUtil.printValidationResult(ex.getMessage(), false); 35 | } catch (KafkaExecutionException ex) { 36 | LogUtil.printKafkaExecutionError(ex); 37 | } catch (WritePlanOutputException ex) { 38 | LogUtil.printPlanOutputError(ex); 39 | } 40 | return 2; 41 | } 42 | 43 | private ManagerConfig generateStateManagerConfig() { 44 | return new ManagerConfig.Builder() 45 | .setVerboseRequested(parent.isVerboseRequested()) 46 | .setDeleteDisabled(parent.isDeleteDisabled()) 47 | .setIncludeUnchangedEnabled(false) 48 | .setSkipAclsDisabled(parent.areAclsDisabled()) 49 | .setConfigFile(parent.getConfigFile()) 50 | .setStateFile(parent.getStateFile()) 51 | .build(); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/cli/ApplyCommand.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.cli; 2 | 3 | import com.devshawn.kafka.gitops.MainCommand; 4 | import com.devshawn.kafka.gitops.StateManager; 5 | import com.devshawn.kafka.gitops.config.ManagerConfig; 6 | import com.devshawn.kafka.gitops.domain.plan.DesiredPlan; 7 | import com.devshawn.kafka.gitops.exception.KafkaExecutionException; 8 | import com.devshawn.kafka.gitops.exception.MissingConfigurationException; 9 | import com.devshawn.kafka.gitops.exception.PlanIsUpToDateException; 10 | import com.devshawn.kafka.gitops.exception.ReadPlanInputException; 11 | import com.devshawn.kafka.gitops.exception.ValidationException; 12 | import com.devshawn.kafka.gitops.service.ParserService; 13 | import com.devshawn.kafka.gitops.util.LogUtil; 14 | import com.devshawn.kafka.gitops.util.PlanUtil; 15 | import picocli.CommandLine; 16 | 17 | import java.io.File; 18 | import java.util.concurrent.Callable; 19 | 20 | @CommandLine.Command(name = "apply", description = "Apply changes to Kafka resources.") 21 | public class ApplyCommand implements Callable { 22 | 23 | @CommandLine.Option(names = {"-p", "--plan"}, paramLabel = "", 24 | description = "Specify the plan file to use.") 25 | private File planFile; 26 | 27 | @CommandLine.ParentCommand 28 | private MainCommand parent; 29 | 30 | @Override 31 | public Integer call() { 32 | try { 33 | System.out.println("Executing apply...\n"); 34 | ParserService parserService = new ParserService(parent.getStateFile()); 35 | StateManager stateManager = new StateManager(generateStateManagerConfig(), parserService); 36 | DesiredPlan desiredPlan = stateManager.apply(); 37 | LogUtil.printApplyOverview(PlanUtil.getOverview(desiredPlan, parent.isDeleteDisabled(), parent.areAclsDisabled())); 38 | return 0; 39 | } catch (PlanIsUpToDateException ex) { 40 | LogUtil.printNoChangesMessage(); 41 | return 0; 42 | } catch (MissingConfigurationException | ReadPlanInputException ex) { 43 | LogUtil.printGenericError(ex, true); 44 | } catch (ValidationException ex) { 45 | LogUtil.printValidationResult(ex.getMessage(), false); 46 | } catch (KafkaExecutionException ex) { 47 | LogUtil.printKafkaExecutionError(ex, true); 48 | } 49 | return 2; 50 | } 51 | 52 | private ManagerConfig generateStateManagerConfig() { 53 | return new ManagerConfig.Builder() 54 | .setVerboseRequested(parent.isVerboseRequested()) 55 | .setDeleteDisabled(parent.isDeleteDisabled()) 56 | .setIncludeUnchangedEnabled(false) 57 | .setSkipAclsDisabled(parent.areAclsDisabled()) 58 | .setConfigFile(parent.getConfigFile()) 59 | .setStateFile(parent.getStateFile()) 60 | .setNullablePlanFile(planFile) 61 | .build(); 62 | } 63 | } -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/cli/PlanCommand.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.cli; 2 | 3 | import com.devshawn.kafka.gitops.MainCommand; 4 | import com.devshawn.kafka.gitops.StateManager; 5 | import com.devshawn.kafka.gitops.config.ManagerConfig; 6 | import com.devshawn.kafka.gitops.domain.plan.DesiredPlan; 7 | import com.devshawn.kafka.gitops.exception.KafkaExecutionException; 8 | import com.devshawn.kafka.gitops.exception.MissingConfigurationException; 9 | import com.devshawn.kafka.gitops.exception.PlanIsUpToDateException; 10 | import com.devshawn.kafka.gitops.exception.ValidationException; 11 | import com.devshawn.kafka.gitops.exception.WritePlanOutputException; 12 | import com.devshawn.kafka.gitops.service.ParserService; 13 | import com.devshawn.kafka.gitops.util.LogUtil; 14 | import picocli.CommandLine; 15 | 16 | import java.io.File; 17 | import java.util.concurrent.Callable; 18 | 19 | @CommandLine.Command(name = "plan", description = "Generate an execution plan of changes to Kafka resources.") 20 | public class PlanCommand implements Callable { 21 | 22 | @CommandLine.Option(names = {"-o", "--output"}, paramLabel = "", 23 | description = "Specify the output file for the plan.") 24 | private File outputFile; 25 | 26 | @CommandLine.Option(names = {"--include-unchanged"}, description = "Include unchanged resources in the plan file.") 27 | private boolean includeUnchanged = false; 28 | 29 | @CommandLine.ParentCommand 30 | private MainCommand parent; 31 | 32 | @Override 33 | public Integer call() { 34 | try { 35 | System.out.println("Generating execution plan...\n"); 36 | ParserService parserService = new ParserService(parent.getStateFile()); 37 | StateManager stateManager = new StateManager(generateStateManagerConfig(), parserService); 38 | DesiredPlan desiredPlan = stateManager.plan(); 39 | LogUtil.printPlan(desiredPlan, parent.isDeleteDisabled(), parent.areAclsDisabled()); 40 | return 0; 41 | } catch (PlanIsUpToDateException ex) { 42 | LogUtil.printNoChangesMessage(); 43 | return 0; 44 | } catch (MissingConfigurationException ex) { 45 | LogUtil.printGenericError(ex); 46 | } catch (ValidationException ex) { 47 | LogUtil.printValidationResult(ex.getMessage(), false); 48 | } catch (KafkaExecutionException ex) { 49 | LogUtil.printKafkaExecutionError(ex); 50 | } catch (WritePlanOutputException ex) { 51 | LogUtil.printPlanOutputError(ex); 52 | } 53 | return 2; 54 | } 55 | 56 | private ManagerConfig generateStateManagerConfig() { 57 | return new ManagerConfig.Builder() 58 | .setVerboseRequested(parent.isVerboseRequested()) 59 | .setDeleteDisabled(parent.isDeleteDisabled()) 60 | .setIncludeUnchangedEnabled(includeUnchanged) 61 | .setConfigFile(parent.getConfigFile()) 62 | .setStateFile(parent.getStateFile()) 63 | .setSkipAclsDisabled(parent.areAclsDisabled()) 64 | .setNullablePlanFile(outputFile) 65 | .build(); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/cli/ValidateCommand.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.cli; 2 | 3 | import com.devshawn.kafka.gitops.MainCommand; 4 | import com.devshawn.kafka.gitops.StateManager; 5 | import com.devshawn.kafka.gitops.config.ManagerConfig; 6 | import com.devshawn.kafka.gitops.exception.ValidationException; 7 | import com.devshawn.kafka.gitops.service.ParserService; 8 | import com.devshawn.kafka.gitops.util.LogUtil; 9 | import picocli.CommandLine; 10 | import picocli.CommandLine.Command; 11 | 12 | import java.util.concurrent.Callable; 13 | 14 | 15 | @Command(name = "validate", description = "Validates the desired state file.") 16 | public class ValidateCommand implements Callable { 17 | 18 | @CommandLine.ParentCommand 19 | private MainCommand parent; 20 | 21 | @Override 22 | public Integer call() { 23 | try { 24 | ParserService parserService = new ParserService(parent.getStateFile()); 25 | StateManager stateManager = new StateManager(generateStateManagerConfig(), parserService); 26 | stateManager.getAndValidateStateFile(); 27 | LogUtil.printValidationResult("Successfully validated the desired state file.", true); 28 | return 0; 29 | } catch (ValidationException ex) { 30 | LogUtil.printValidationResult(ex.getMessage(), false); 31 | } 32 | return 2; 33 | } 34 | 35 | private ManagerConfig generateStateManagerConfig() { 36 | return new ManagerConfig.Builder() 37 | .setVerboseRequested(parent.isVerboseRequested()) 38 | .setDeleteDisabled(parent.isDeleteDisabled()) 39 | .setIncludeUnchangedEnabled(false) 40 | .setSkipAclsDisabled(parent.areAclsDisabled()) 41 | .setConfigFile(parent.getConfigFile()) 42 | .setStateFile(parent.getStateFile()) 43 | .build(); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/config/KafkaGitopsConfig.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.config; 2 | 3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 4 | import org.inferred.freebuilder.FreeBuilder; 5 | 6 | import java.util.Map; 7 | 8 | @FreeBuilder 9 | @JsonDeserialize(builder = KafkaGitopsConfig.Builder.class) 10 | public interface KafkaGitopsConfig { 11 | 12 | Map getConfig(); 13 | 14 | class Builder extends KafkaGitopsConfig_Builder { 15 | 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/config/ManagerConfig.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.config; 2 | 3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 4 | import org.inferred.freebuilder.FreeBuilder; 5 | 6 | import java.io.File; 7 | import java.util.Optional; 8 | 9 | @FreeBuilder 10 | @JsonDeserialize(builder = ManagerConfig.Builder.class) 11 | public interface ManagerConfig { 12 | 13 | boolean isVerboseRequested(); 14 | 15 | boolean isDeleteDisabled(); 16 | 17 | boolean isIncludeUnchangedEnabled(); 18 | 19 | boolean isSkipAclsDisabled(); 20 | 21 | File getConfigFile(); 22 | 23 | File getStateFile(); 24 | 25 | Optional getPlanFile(); 26 | 27 | class Builder extends ManagerConfig_Builder { 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/confluent/ServiceAccount.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.confluent; 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 4 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 5 | import org.inferred.freebuilder.FreeBuilder; 6 | 7 | @FreeBuilder 8 | @JsonIgnoreProperties(ignoreUnknown = true) 9 | @JsonDeserialize(builder = ServiceAccount.Builder.class) 10 | public interface ServiceAccount { 11 | 12 | String getId(); 13 | 14 | String getName(); 15 | 16 | class Builder extends ServiceAccount_Builder { 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/options/GetAclOptions.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.options; 2 | 3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 4 | import org.inferred.freebuilder.FreeBuilder; 5 | 6 | @FreeBuilder 7 | @JsonDeserialize(builder = GetAclOptions.Builder.class) 8 | public interface GetAclOptions { 9 | 10 | String getServiceName(); 11 | 12 | Boolean getDescribeAclEnabled(); 13 | 14 | class Builder extends GetAclOptions_Builder { 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/plan/AclPlan.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.plan; 2 | 3 | import com.devshawn.kafka.gitops.domain.state.AclDetails; 4 | import com.devshawn.kafka.gitops.enums.PlanAction; 5 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 6 | import org.inferred.freebuilder.FreeBuilder; 7 | 8 | @FreeBuilder 9 | @JsonDeserialize(builder = AclPlan.Builder.class) 10 | public interface AclPlan { 11 | 12 | String getName(); 13 | 14 | AclDetails getAclDetails(); 15 | 16 | PlanAction getAction(); 17 | 18 | class Builder extends AclPlan_Builder { 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/plan/DesiredPlan.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.plan; 2 | 3 | import com.devshawn.kafka.gitops.enums.PlanAction; 4 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 5 | import org.inferred.freebuilder.FreeBuilder; 6 | 7 | import java.util.List; 8 | 9 | @FreeBuilder 10 | @JsonDeserialize(builder = DesiredPlan.Builder.class) 11 | public interface DesiredPlan { 12 | 13 | List getTopicPlans(); 14 | 15 | List getAclPlans(); 16 | 17 | default DesiredPlan toChangesOnlyPlan() { 18 | DesiredPlan.Builder builder = new DesiredPlan.Builder(); 19 | getTopicPlans().stream().filter(it -> !it.getAction().equals(PlanAction.NO_CHANGE)).map(TopicPlan::toChangesOnlyPlan).forEach(builder::addTopicPlans); 20 | getAclPlans().stream().filter(it -> !it.getAction().equals(PlanAction.NO_CHANGE)).forEach(builder::addAclPlans); 21 | return builder.build(); 22 | } 23 | 24 | class Builder extends DesiredPlan_Builder { 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/plan/PlanOverview.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.plan; 2 | 3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 4 | import org.inferred.freebuilder.FreeBuilder; 5 | 6 | @FreeBuilder 7 | @JsonDeserialize(builder = PlanOverview.Builder.class) 8 | public interface PlanOverview { 9 | 10 | Long getAdd(); 11 | 12 | Long getRemove(); 13 | 14 | Long getUpdate(); 15 | 16 | Long getNoChange(); 17 | 18 | class Builder extends PlanOverview_Builder { 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/plan/TopicConfigPlan.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.plan; 2 | 3 | import com.devshawn.kafka.gitops.enums.PlanAction; 4 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 5 | import org.inferred.freebuilder.FreeBuilder; 6 | 7 | import java.util.Optional; 8 | 9 | @FreeBuilder 10 | @JsonDeserialize(builder = TopicConfigPlan.Builder.class) 11 | public interface TopicConfigPlan { 12 | 13 | String getKey(); 14 | 15 | Optional getValue(); 16 | 17 | Optional getPreviousValue(); 18 | 19 | PlanAction getAction(); 20 | 21 | class Builder extends TopicConfigPlan_Builder { 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/plan/TopicDetailsPlan.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.plan; 2 | 3 | import java.util.Optional; 4 | import org.inferred.freebuilder.FreeBuilder; 5 | import com.devshawn.kafka.gitops.enums.PlanAction; 6 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 7 | 8 | @FreeBuilder 9 | @JsonDeserialize(builder = TopicDetailsPlan.Builder.class) 10 | public interface TopicDetailsPlan { 11 | Optional getPartitions(); 12 | Optional getPreviousPartitions(); 13 | PlanAction getPartitionsAction(); 14 | 15 | Optional getReplication(); 16 | Optional getPreviousReplication(); 17 | PlanAction getReplicationAction(); 18 | 19 | public static Optional toChangesOnlyPlan(Optional topicDetailsPlan) { 20 | if(! topicDetailsPlan.isPresent()) { 21 | return topicDetailsPlan; 22 | } 23 | TopicDetailsPlan.Builder builder = new TopicDetailsPlan.Builder(); 24 | builder.setReplicationAction(topicDetailsPlan.get().getReplicationAction()); 25 | builder.setPartitionsAction(topicDetailsPlan.get().getPartitionsAction()); 26 | boolean nochanges = true; 27 | if ( topicDetailsPlan.get().getReplicationAction() != null && ! topicDetailsPlan.get().getReplicationAction().equals(PlanAction.NO_CHANGE)) { 28 | builder.setReplication(topicDetailsPlan.get().getReplication()); 29 | builder.setPreviousReplication(topicDetailsPlan.get().getPreviousReplication()); 30 | nochanges = false; 31 | } 32 | if (topicDetailsPlan.get().getPartitionsAction() != null && ! topicDetailsPlan.get().getPartitionsAction().equals(PlanAction.NO_CHANGE)) { 33 | builder.setPartitions(topicDetailsPlan.get().getPartitions()); 34 | builder.setPreviousPartitions(topicDetailsPlan.get().getPreviousPartitions()); 35 | nochanges = false; 36 | } 37 | if(nochanges) { 38 | return Optional.empty(); 39 | } 40 | return Optional.of(builder.build()); 41 | } 42 | class Builder extends TopicDetailsPlan_Builder { 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/plan/TopicPlan.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.plan; 2 | 3 | import com.devshawn.kafka.gitops.enums.PlanAction; 4 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 5 | import org.inferred.freebuilder.FreeBuilder; 6 | 7 | import java.util.List; 8 | import java.util.Optional; 9 | 10 | @FreeBuilder 11 | @JsonDeserialize(builder = TopicPlan.Builder.class) 12 | public interface TopicPlan { 13 | 14 | String getName(); 15 | 16 | PlanAction getAction(); 17 | 18 | Optional getTopicDetailsPlan(); 19 | 20 | List getTopicConfigPlans(); 21 | 22 | default TopicPlan toChangesOnlyPlan() { 23 | TopicPlan.Builder builder = new TopicPlan.Builder().setName(getName()).setAction(getAction()); 24 | 25 | builder.setTopicDetailsPlan(TopicDetailsPlan.toChangesOnlyPlan(getTopicDetailsPlan())); 26 | 27 | getTopicConfigPlans().stream().filter(it -> !it.getAction().equals(PlanAction.NO_CHANGE)).forEach(builder::addTopicConfigPlans); 28 | return builder.build(); 29 | } 30 | 31 | class Builder extends TopicPlan_Builder { 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/AbstractService.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state; 2 | 3 | import java.util.Optional; 4 | 5 | public abstract class AbstractService { 6 | 7 | public AclDetails.Builder generateReadAcl(String topic, Optional principal) { 8 | AclDetails.Builder builder = new AclDetails.Builder() 9 | .setHost("*") 10 | .setName(topic) 11 | .setOperation("READ") 12 | .setPermission("ALLOW") 13 | .setPattern("LITERAL") 14 | .setType("TOPIC"); 15 | 16 | principal.ifPresent(builder::setPrincipal); 17 | return builder; 18 | } 19 | 20 | public AclDetails.Builder generateWriteACL(String topic, Optional principal) { 21 | AclDetails.Builder builder = new AclDetails.Builder() 22 | .setHost("*") 23 | .setName(topic) 24 | .setOperation("WRITE") 25 | .setPermission("ALLOW") 26 | .setPattern("LITERAL") 27 | .setType("TOPIC"); 28 | 29 | principal.ifPresent(builder::setPrincipal); 30 | return builder; 31 | } 32 | 33 | public AclDetails.Builder generateDescribeAcl(String topic, Optional principal) { 34 | AclDetails.Builder builder = new AclDetails.Builder() 35 | .setHost("*") 36 | .setName(topic) 37 | .setOperation("DESCRIBE") 38 | .setPermission("ALLOW") 39 | .setPattern("LITERAL") 40 | .setType("TOPIC"); 41 | 42 | principal.ifPresent(builder::setPrincipal); 43 | return builder; 44 | } 45 | 46 | public AclDetails.Builder generatePrefixedTopicACL(String topic, Optional principal, String operation) { 47 | AclDetails.Builder builder = new AclDetails.Builder() 48 | .setHost("*") 49 | .setName(topic) 50 | .setOperation(operation) 51 | .setPermission("ALLOW") 52 | .setPattern("PREFIXED") 53 | .setType("TOPIC"); 54 | 55 | principal.ifPresent(builder::setPrincipal); 56 | return builder; 57 | } 58 | 59 | public AclDetails.Builder generateConsumerGroupAcl(String consumerGroupId, Optional principal, String operation) { 60 | AclDetails.Builder builder = new AclDetails.Builder() 61 | .setHost("*") 62 | .setName(consumerGroupId) 63 | .setOperation(operation) 64 | .setPermission("ALLOW") 65 | .setPattern("LITERAL") 66 | .setType("GROUP"); 67 | 68 | principal.ifPresent(builder::setPrincipal); 69 | return builder; 70 | } 71 | 72 | public AclDetails.Builder generateClusterAcl(Optional principal, String operation) { 73 | AclDetails.Builder builder = new AclDetails.Builder() 74 | .setHost("*") 75 | .setName("kafka-cluster") 76 | .setOperation(operation) 77 | .setPermission("ALLOW") 78 | .setPattern("LITERAL") 79 | .setType("CLUSTER"); 80 | 81 | principal.ifPresent(builder::setPrincipal); 82 | return builder; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/AclDetails.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state; 2 | 3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 4 | import org.apache.kafka.common.acl.AccessControlEntry; 5 | import org.apache.kafka.common.acl.AclBinding; 6 | import org.apache.kafka.common.acl.AclOperation; 7 | import org.apache.kafka.common.acl.AclPermissionType; 8 | import org.apache.kafka.common.resource.PatternType; 9 | import org.apache.kafka.common.resource.ResourcePattern; 10 | import org.apache.kafka.common.resource.ResourceType; 11 | import org.inferred.freebuilder.FreeBuilder; 12 | 13 | @FreeBuilder 14 | @JsonDeserialize(builder = AclDetails.Builder.class) 15 | public abstract class AclDetails { 16 | 17 | public abstract String getName(); 18 | 19 | public abstract String getType(); 20 | 21 | public abstract String getPattern(); 22 | 23 | public abstract String getPrincipal(); 24 | 25 | public abstract String getHost(); 26 | 27 | public abstract String getOperation(); 28 | 29 | public abstract String getPermission(); 30 | 31 | public static AclDetails fromAclBinding(AclBinding aclBinding) { 32 | AclDetails.Builder aclDetails = new AclDetails.Builder() 33 | .setName(aclBinding.pattern().name()) 34 | .setType(aclBinding.pattern().resourceType().name()) 35 | .setPattern(aclBinding.pattern().patternType().name()) 36 | .setPrincipal(aclBinding.entry().principal()) 37 | .setHost(aclBinding.entry().host()) 38 | .setOperation(aclBinding.entry().operation().name()) 39 | .setPermission(aclBinding.entry().permissionType().name()); 40 | return aclDetails.build(); 41 | } 42 | 43 | public static AclDetails.Builder fromCustomAclDetails(CustomAclDetails customAclDetails) { 44 | return new AclDetails.Builder() 45 | .setName(customAclDetails.getName()) 46 | .setType(customAclDetails.getType()) 47 | .setPattern(customAclDetails.getPattern()) 48 | .setHost(customAclDetails.getHost()) 49 | .setOperation(customAclDetails.getOperation()) 50 | .setPermission(customAclDetails.getPermission()); 51 | } 52 | 53 | public boolean equalsAclBinding(AclBinding aclBinding) { 54 | if (aclBinding.pattern().name().equals(getName()) 55 | && aclBinding.pattern().patternType().name().equals(getPattern()) 56 | && aclBinding.pattern().resourceType().name().equals(getType()) 57 | && aclBinding.entry().principal().equals(getPrincipal()) 58 | && aclBinding.entry().host().equals(getHost()) 59 | && aclBinding.entry().permissionType().name().equals(getPermission()) 60 | && aclBinding.entry().operation().name().equals(getOperation())) { 61 | return true; 62 | } 63 | return false; 64 | } 65 | 66 | public AclBinding toAclBinding() { 67 | return new AclBinding( 68 | new ResourcePattern(ResourceType.valueOf(getType()), getName(), PatternType.valueOf(getPattern())), 69 | new AccessControlEntry(getPrincipal(), getHost(), AclOperation.valueOf(getOperation()), AclPermissionType.valueOf(getPermission())) 70 | ); 71 | } 72 | 73 | public static class Builder extends AclDetails_Builder { 74 | public Builder() { 75 | setPermission(AclPermissionType.ALLOW.name()); 76 | setHost("*"); 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/CustomAclDetails.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state; 2 | 3 | import com.devshawn.kafka.gitops.exception.InvalidAclDefinitionException; 4 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 5 | import org.apache.kafka.common.acl.AclOperation; 6 | import org.apache.kafka.common.acl.AclPermissionType; 7 | import org.apache.kafka.common.resource.PatternType; 8 | import org.apache.kafka.common.resource.ResourceType; 9 | import org.inferred.freebuilder.FreeBuilder; 10 | 11 | import java.util.Arrays; 12 | import java.util.List; 13 | import java.util.Optional; 14 | import java.util.stream.Collectors; 15 | 16 | @FreeBuilder 17 | @JsonDeserialize(builder = CustomAclDetails.Builder.class) 18 | public abstract class CustomAclDetails { 19 | 20 | public abstract String getName(); 21 | 22 | public abstract String getType(); 23 | 24 | public abstract String getPattern(); 25 | 26 | public abstract Optional getPrincipal(); 27 | 28 | public abstract String getHost(); 29 | 30 | public abstract String getOperation(); 31 | 32 | public abstract String getPermission(); 33 | 34 | public void validate() { 35 | validateEnum(ResourceType.class, getType(), "type"); 36 | validateEnum(PatternType.class, getPattern(), "pattern"); 37 | validateEnum(AclOperation.class, getOperation(), "operation"); 38 | validateEnum(AclPermissionType.class, getPermission(), "permission"); 39 | } 40 | 41 | private > void validateEnum(Class enumData, String value, String field) { 42 | List allowedValues = Arrays.stream(enumData.getEnumConstants()).map(Enum::name).filter(it -> !it.equals("ANY") && !it.equals("UNKNOWN")).collect(Collectors.toList()); 43 | if (!allowedValues.contains(value)) { 44 | throw new InvalidAclDefinitionException(field, value, allowedValues); 45 | } 46 | } 47 | 48 | public static class Builder extends CustomAclDetails_Builder { 49 | public Builder() { 50 | setPermission(AclPermissionType.ALLOW.name()); 51 | setHost("*"); 52 | } 53 | } 54 | } -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredState.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state; 2 | 3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 4 | import org.inferred.freebuilder.FreeBuilder; 5 | 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | @FreeBuilder 10 | @JsonDeserialize(builder = DesiredState.Builder.class) 11 | public interface DesiredState { 12 | 13 | Map getTopics(); 14 | 15 | Map getAcls(); 16 | 17 | List getPrefixedTopicsToIgnore(); 18 | 19 | class Builder extends DesiredState_Builder { 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredStateFile.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state; 2 | 3 | 4 | import com.devshawn.kafka.gitops.domain.state.settings.Settings; 5 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 6 | import org.inferred.freebuilder.FreeBuilder; 7 | 8 | import java.util.Map; 9 | import java.util.Optional; 10 | 11 | @FreeBuilder 12 | @JsonDeserialize(builder = DesiredStateFile.Builder.class) 13 | public interface DesiredStateFile { 14 | 15 | Optional getSettings(); 16 | 17 | Map getServices(); 18 | 19 | Map getTopics(); 20 | 21 | Map getUsers(); 22 | 23 | Map> getCustomServiceAcls(); 24 | 25 | Map> getCustomUserAcls(); 26 | 27 | class Builder extends DesiredStateFile_Builder { 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/ServiceDetails.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state; 2 | 3 | import com.devshawn.kafka.gitops.domain.options.GetAclOptions; 4 | import com.devshawn.kafka.gitops.domain.state.service.ApplicationService; 5 | import com.devshawn.kafka.gitops.domain.state.service.KafkaConnectService; 6 | import com.devshawn.kafka.gitops.domain.state.service.KafkaStreamsService; 7 | import com.fasterxml.jackson.annotation.JsonSubTypes; 8 | import com.fasterxml.jackson.annotation.JsonTypeInfo; 9 | 10 | import java.util.List; 11 | 12 | @JsonSubTypes({ 13 | @JsonSubTypes.Type(value = ApplicationService.class, name = "application"), 14 | @JsonSubTypes.Type(value = KafkaConnectService.class, name = "kafka-connect"), 15 | @JsonSubTypes.Type(value = KafkaStreamsService.class, name = "kafka-streams") 16 | }) 17 | @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") 18 | public abstract class ServiceDetails extends AbstractService { 19 | 20 | public String type; 21 | 22 | public List getAcls(GetAclOptions options) { 23 | throw new UnsupportedOperationException("Method getAcls is not implemented."); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/TopicDetails.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state; 2 | 3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 4 | import org.inferred.freebuilder.FreeBuilder; 5 | 6 | import java.util.Map; 7 | import java.util.Optional; 8 | 9 | @FreeBuilder 10 | @JsonDeserialize(builder = TopicDetails.Builder.class) 11 | public interface TopicDetails { 12 | 13 | Integer getPartitions(); 14 | 15 | Optional getReplication(); 16 | 17 | Map getConfigs(); 18 | 19 | class Builder extends TopicDetails_Builder { 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/UserDetails.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state; 2 | 3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 4 | import org.inferred.freebuilder.FreeBuilder; 5 | 6 | import java.util.List; 7 | import java.util.Optional; 8 | 9 | @FreeBuilder 10 | @JsonDeserialize(builder = UserDetails.Builder.class) 11 | public interface UserDetails { 12 | 13 | Optional getPrincipal(); 14 | 15 | List getRoles(); 16 | 17 | class Builder extends UserDetails_Builder { 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/service/ApplicationService.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state.service; 2 | 3 | import com.devshawn.kafka.gitops.domain.options.GetAclOptions; 4 | import com.devshawn.kafka.gitops.domain.state.AclDetails; 5 | import com.devshawn.kafka.gitops.domain.state.ServiceDetails; 6 | import com.devshawn.kafka.gitops.util.HelperUtil; 7 | import com.fasterxml.jackson.annotation.JsonProperty; 8 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 9 | import org.inferred.freebuilder.FreeBuilder; 10 | 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | import java.util.Optional; 14 | 15 | @FreeBuilder 16 | @JsonDeserialize(builder = ApplicationService.Builder.class) 17 | public abstract class ApplicationService extends ServiceDetails { 18 | 19 | public abstract Optional getPrincipal(); 20 | 21 | @JsonProperty("group-id") 22 | public abstract Optional getGroupId(); 23 | 24 | public abstract List getProduces(); 25 | 26 | public abstract List getConsumes(); 27 | 28 | @Override 29 | public List getAcls(GetAclOptions options) { 30 | List acls = new ArrayList<>(); 31 | getProduces().forEach(topic -> acls.add(generateWriteACL(topic, getPrincipal()))); 32 | getConsumes().forEach(topic -> acls.add(generateReadAcl(topic, getPrincipal()))); 33 | 34 | if (options.getDescribeAclEnabled()) { 35 | List allTopics = HelperUtil.uniqueCombine(getConsumes(), getProduces()); 36 | allTopics.forEach(topic -> acls.add(generateDescribeAcl(topic, getPrincipal()))); 37 | } 38 | 39 | if (!getConsumes().isEmpty()) { 40 | String groupId = getGroupId().isPresent() ? getGroupId().get() : options.getServiceName(); 41 | acls.add(generateConsumerGroupAcl(groupId, getPrincipal(), "READ")); 42 | } 43 | return acls; 44 | } 45 | 46 | public static class Builder extends ApplicationService_Builder { 47 | 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/service/KafkaConnectService.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state.service; 2 | 3 | 4 | import com.devshawn.kafka.gitops.domain.options.GetAclOptions; 5 | import com.devshawn.kafka.gitops.domain.state.AclDetails; 6 | import com.devshawn.kafka.gitops.domain.state.ServiceDetails; 7 | import com.fasterxml.jackson.annotation.JsonProperty; 8 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 9 | import org.inferred.freebuilder.FreeBuilder; 10 | 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | import java.util.Map; 14 | import java.util.Optional; 15 | 16 | @FreeBuilder 17 | @JsonDeserialize(builder = KafkaConnectService.Builder.class) 18 | public abstract class KafkaConnectService extends ServiceDetails { 19 | 20 | @JsonProperty("group-id") 21 | public abstract Optional getGroupId(); 22 | 23 | public abstract Optional getPrincipal(); 24 | 25 | @JsonProperty("storage-topics") 26 | public abstract Optional getStorageTopics(); 27 | 28 | public abstract List getProduces(); 29 | 30 | public abstract Map getConnectors(); 31 | 32 | @Override 33 | public List getAcls(GetAclOptions options) { 34 | List acls = new ArrayList<>(); 35 | getProduces().forEach(topic -> acls.add(generateWriteACL(topic, getPrincipal()))); 36 | if (options.getDescribeAclEnabled()) { 37 | getProduces().forEach(topic -> acls.add(generateDescribeAcl(topic, getPrincipal()))); 38 | } 39 | acls.addAll(getConnectWorkerAcls(options)); 40 | return acls; 41 | } 42 | 43 | private List getConnectWorkerAcls(GetAclOptions options) { 44 | String groupId = getGroupId().isPresent() ? getGroupId().get() : options.getServiceName(); 45 | String configTopic = getConfigTopic(options.getServiceName()); 46 | String offsetTopic = getOffsetTopic(options.getServiceName()); 47 | String statusTopic = getStatusTopic(options.getServiceName()); 48 | 49 | List acls = new ArrayList<>(); 50 | acls.add(generateReadAcl(configTopic, getPrincipal())); 51 | acls.add(generateReadAcl(offsetTopic, getPrincipal())); 52 | acls.add(generateReadAcl(statusTopic, getPrincipal())); 53 | acls.add(generateWriteACL(configTopic, getPrincipal())); 54 | acls.add(generateWriteACL(offsetTopic, getPrincipal())); 55 | acls.add(generateWriteACL(statusTopic, getPrincipal())); 56 | acls.add(generateConsumerGroupAcl(groupId, getPrincipal(), "READ")); 57 | getConnectors().forEach((connectorName, connector) -> acls.addAll(connector.getAcls(connectorName, getPrincipal(), options))); 58 | return acls; 59 | } 60 | 61 | private String getConfigTopic(String serviceName) { 62 | if (getStorageTopics().isPresent() && getStorageTopics().get().getConfig().isPresent()) { 63 | return getStorageTopics().get().getConfig().get(); 64 | } 65 | return String.format("connect-configs-%s", serviceName); 66 | } 67 | 68 | private String getOffsetTopic(String serviceName) { 69 | if (getStorageTopics().isPresent() && getStorageTopics().get().getOffset().isPresent()) { 70 | return getStorageTopics().get().getOffset().get(); 71 | } 72 | return String.format("connect-offsets-%s", serviceName); 73 | } 74 | 75 | private String getStatusTopic(String serviceName) { 76 | if (getStorageTopics().isPresent() && getStorageTopics().get().getStatus().isPresent()) { 77 | return getStorageTopics().get().getStatus().get(); 78 | } 79 | return String.format("connect-status-%s", serviceName); 80 | } 81 | 82 | public static class Builder extends KafkaConnectService_Builder { 83 | 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/service/KafkaConnectStorageTopics.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state.service; 2 | 3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 4 | import org.inferred.freebuilder.FreeBuilder; 5 | 6 | import java.util.Optional; 7 | 8 | @FreeBuilder 9 | @JsonDeserialize(builder = KafkaConnectStorageTopics.Builder.class) 10 | public interface KafkaConnectStorageTopics { 11 | 12 | Optional getConfig(); 13 | 14 | Optional getOffset(); 15 | 16 | Optional getStatus(); 17 | 18 | class Builder extends KafkaConnectStorageTopics_Builder { 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/service/KafkaConnectorDetails.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state.service; 2 | 3 | import com.devshawn.kafka.gitops.domain.options.GetAclOptions; 4 | import com.devshawn.kafka.gitops.domain.state.AbstractService; 5 | import com.devshawn.kafka.gitops.domain.state.AclDetails; 6 | import com.devshawn.kafka.gitops.util.HelperUtil; 7 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 8 | import org.inferred.freebuilder.FreeBuilder; 9 | 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | import java.util.Optional; 13 | 14 | @FreeBuilder 15 | @JsonDeserialize(builder = KafkaConnectorDetails.Builder.class) 16 | public abstract class KafkaConnectorDetails extends AbstractService { 17 | 18 | public abstract List getProduces(); 19 | 20 | public abstract List getConsumes(); 21 | 22 | public List getAcls(String connectorName, Optional principal, GetAclOptions options) { 23 | List acls = new ArrayList<>(); 24 | getProduces().forEach(topic -> acls.add(generateWriteACL(topic, principal))); 25 | getConsumes().forEach(topic -> acls.add(generateReadAcl(topic, principal))); 26 | 27 | if (options.getDescribeAclEnabled()) { 28 | List allTopics = HelperUtil.uniqueCombine(getConsumes(), getProduces()); 29 | allTopics.forEach(topic -> acls.add(generateDescribeAcl(topic, principal))); 30 | } 31 | 32 | if (!getConsumes().isEmpty()) { 33 | acls.add(generateConsumerGroupAcl(String.format("connect-%s", connectorName), principal, "READ")); 34 | } 35 | return acls; 36 | } 37 | 38 | public static class Builder extends KafkaConnectorDetails_Builder { 39 | 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/service/KafkaStreamsService.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state.service; 2 | 3 | import com.devshawn.kafka.gitops.domain.options.GetAclOptions; 4 | import com.devshawn.kafka.gitops.domain.state.AclDetails; 5 | import com.devshawn.kafka.gitops.domain.state.ServiceDetails; 6 | import com.devshawn.kafka.gitops.util.HelperUtil; 7 | import com.fasterxml.jackson.annotation.JsonProperty; 8 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 9 | import org.inferred.freebuilder.FreeBuilder; 10 | 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | import java.util.Optional; 14 | 15 | @FreeBuilder 16 | @JsonDeserialize(builder = KafkaStreamsService.Builder.class) 17 | public abstract class KafkaStreamsService extends ServiceDetails { 18 | 19 | public abstract Optional getPrincipal(); 20 | 21 | @JsonProperty("application-id") 22 | public abstract Optional getApplicationId(); 23 | 24 | public abstract List getProduces(); 25 | 26 | public abstract List getConsumes(); 27 | 28 | @Override 29 | public List getAcls(GetAclOptions options) { 30 | List acls = new ArrayList<>(); 31 | getProduces().forEach(topic -> acls.add(generateWriteACL(topic, getPrincipal()))); 32 | getConsumes().forEach(topic -> acls.add(generateReadAcl(topic, getPrincipal()))); 33 | if (options.getDescribeAclEnabled()) { 34 | List allTopics = HelperUtil.uniqueCombine(getConsumes(), getProduces()); 35 | allTopics.forEach(topic -> acls.add(generateDescribeAcl(topic, getPrincipal()))); 36 | } 37 | acls.addAll(getInternalAcls(options.getServiceName())); 38 | return acls; 39 | } 40 | 41 | private List getInternalAcls(String serviceName) { 42 | String applicationId = getApplicationId().isPresent() ? getApplicationId().get() : serviceName; 43 | List acls = new ArrayList<>(); 44 | acls.add(generatePrefixedTopicACL(applicationId, getPrincipal(), "READ")); 45 | acls.add(generatePrefixedTopicACL(applicationId, getPrincipal(), "WRITE")); 46 | acls.add(generatePrefixedTopicACL(applicationId, getPrincipal(), "DESCRIBE")); 47 | acls.add(generatePrefixedTopicACL(applicationId, getPrincipal(), "DELETE")); 48 | acls.add(generatePrefixedTopicACL(applicationId, getPrincipal(), "CREATE")); 49 | acls.add(generatePrefixedTopicACL(applicationId, getPrincipal(), "ALTER")); 50 | acls.add(generatePrefixedTopicACL(applicationId, getPrincipal(), "ALTER_CONFIGS")); 51 | acls.add(generatePrefixedTopicACL(applicationId, getPrincipal(), "DESCRIBE_CONFIGS")); 52 | acls.add(generateConsumerGroupAcl(applicationId, getPrincipal(), "READ")); 53 | acls.add(generateConsumerGroupAcl(applicationId, getPrincipal(), "DESCRIBE")); 54 | acls.add(generateConsumerGroupAcl(applicationId, getPrincipal(), "DELETE")); 55 | acls.add(generateClusterAcl(getPrincipal(), "DESCRIBE_CONFIGS")); 56 | return acls; 57 | } 58 | 59 | public static class Builder extends KafkaStreamsService_Builder { 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/settings/Settings.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state.settings; 2 | 3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 4 | import org.inferred.freebuilder.FreeBuilder; 5 | 6 | import java.util.Optional; 7 | 8 | @FreeBuilder 9 | @JsonDeserialize(builder = Settings.Builder.class) 10 | public interface Settings { 11 | 12 | Optional getCcloud(); 13 | 14 | Optional getTopics(); 15 | 16 | Optional getServices(); 17 | 18 | Optional getFiles(); 19 | 20 | class Builder extends Settings_Builder { 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsCCloud.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state.settings; 2 | 3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 4 | import org.inferred.freebuilder.FreeBuilder; 5 | 6 | @FreeBuilder 7 | @JsonDeserialize(builder = SettingsCCloud.Builder.class) 8 | public interface SettingsCCloud { 9 | 10 | boolean isEnabled(); 11 | 12 | class Builder extends SettingsCCloud_Builder { 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsFiles.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state.settings; 2 | 3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 4 | import org.inferred.freebuilder.FreeBuilder; 5 | 6 | import java.util.Optional; 7 | 8 | @FreeBuilder 9 | @JsonDeserialize(builder = SettingsFiles.Builder.class) 10 | public interface SettingsFiles { 11 | 12 | Optional getServices(); 13 | 14 | Optional getTopics(); 15 | 16 | Optional getUsers(); 17 | 18 | class Builder extends SettingsFiles_Builder { 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsServices.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state.settings; 2 | 3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 4 | import org.inferred.freebuilder.FreeBuilder; 5 | 6 | import java.util.Optional; 7 | 8 | @FreeBuilder 9 | @JsonDeserialize(builder = SettingsServices.Builder.class) 10 | public interface SettingsServices { 11 | 12 | Optional getAcls(); 13 | 14 | class Builder extends SettingsServices_Builder { 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsServicesAcls.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state.settings; 2 | 3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 4 | import org.inferred.freebuilder.FreeBuilder; 5 | 6 | import java.util.Optional; 7 | 8 | @FreeBuilder 9 | @JsonDeserialize(builder = SettingsServicesAcls.Builder.class) 10 | public interface SettingsServicesAcls { 11 | 12 | Optional getDescribeTopicEnabled(); 13 | 14 | class Builder extends SettingsServicesAcls_Builder { 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsTopics.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state.settings; 2 | 3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 4 | import org.inferred.freebuilder.FreeBuilder; 5 | 6 | import java.util.Optional; 7 | 8 | @FreeBuilder 9 | @JsonDeserialize(builder = SettingsTopics.Builder.class) 10 | public interface SettingsTopics { 11 | 12 | Optional getDefaults(); 13 | 14 | Optional getBlacklist(); 15 | 16 | class Builder extends SettingsTopics_Builder { 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsTopicsBlacklist.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state.settings; 2 | 3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 4 | import org.inferred.freebuilder.FreeBuilder; 5 | 6 | import java.util.List; 7 | 8 | @FreeBuilder 9 | @JsonDeserialize(builder = SettingsTopicsBlacklist.Builder.class) 10 | public interface SettingsTopicsBlacklist { 11 | 12 | List getPrefixed(); 13 | 14 | class Builder extends SettingsTopicsBlacklist_Builder { 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/domain/state/settings/SettingsTopicsDefaults.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state.settings; 2 | 3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 4 | import org.inferred.freebuilder.FreeBuilder; 5 | 6 | import java.util.Optional; 7 | 8 | @FreeBuilder 9 | @JsonDeserialize(builder = SettingsTopicsDefaults.Builder.class) 10 | public interface SettingsTopicsDefaults { 11 | 12 | Optional getReplication(); 13 | 14 | class Builder extends SettingsTopicsDefaults_Builder { 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/enums/PlanAction.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.enums; 2 | 3 | public enum PlanAction { 4 | ADD, 5 | UPDATE, 6 | REMOVE, 7 | NO_CHANGE 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/exception/ConfluentCloudException.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.exception; 2 | 3 | public class ConfluentCloudException extends RuntimeException { 4 | 5 | public ConfluentCloudException(String message) { 6 | super(message); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/exception/InvalidAclDefinitionException.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.exception; 2 | 3 | import java.util.List; 4 | 5 | public class InvalidAclDefinitionException extends RuntimeException { 6 | 7 | private final String field; 8 | 9 | private final String value; 10 | 11 | private final List allowedValues; 12 | 13 | public InvalidAclDefinitionException(String field, String value, List allowedValues) { 14 | this.field = field; 15 | this.value = value; 16 | this.allowedValues = allowedValues; 17 | } 18 | 19 | public String getField() { 20 | return field; 21 | } 22 | 23 | public String getValue() { 24 | return value; 25 | } 26 | 27 | public List getAllowedValues() { 28 | return allowedValues; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/exception/KafkaExecutionException.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.exception; 2 | 3 | public class KafkaExecutionException extends RuntimeException { 4 | 5 | private final String exceptionMessage; 6 | 7 | public KafkaExecutionException(String message, String exceptionMessage) { 8 | super(message); 9 | this.exceptionMessage = exceptionMessage; 10 | } 11 | 12 | public String getExceptionMessage() { 13 | return exceptionMessage; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/exception/MissingConfigurationException.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.exception; 2 | 3 | public class MissingConfigurationException extends RuntimeException { 4 | 5 | public MissingConfigurationException(String environmentVariableName) { 6 | super(String.format("Missing required configuration: %s", environmentVariableName)); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/exception/PlanIsUpToDateException.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.exception; 2 | 3 | public class PlanIsUpToDateException extends RuntimeException { 4 | 5 | public PlanIsUpToDateException() { 6 | super("The current desired state file matches the actual state of the cluster."); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/exception/ReadPlanInputException.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.exception; 2 | 3 | public class ReadPlanInputException extends RuntimeException { 4 | 5 | public ReadPlanInputException() { 6 | super("Error reading execution plan from file: Please run the plan command again to generate a new plan file."); 7 | } 8 | 9 | public ReadPlanInputException(String message) { 10 | super(String.format("Error reading execution plan from file: %s", message)); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/exception/ServiceAccountNotFoundException.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.exception; 2 | 3 | public class ServiceAccountNotFoundException extends RuntimeException { 4 | 5 | public ServiceAccountNotFoundException(String service) { 6 | super(String.format("Service account not found for service: %s", service)); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/exception/ValidationException.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.exception; 2 | 3 | public class ValidationException extends RuntimeException { 4 | 5 | public ValidationException(String message) { 6 | super(message); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/exception/WritePlanOutputException.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.exception; 2 | 3 | public class WritePlanOutputException extends RuntimeException { 4 | 5 | public WritePlanOutputException(String exMessage) { 6 | super(String.format("Error writing execution plan to file: %s", exMessage)); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/service/ConfluentCloudService.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.service; 2 | 3 | import com.devshawn.kafka.gitops.domain.confluent.ServiceAccount; 4 | import com.devshawn.kafka.gitops.exception.ConfluentCloudException; 5 | import com.fasterxml.jackson.core.type.TypeReference; 6 | import com.fasterxml.jackson.databind.ObjectMapper; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.io.IOException; 10 | import java.util.List; 11 | 12 | public class ConfluentCloudService { 13 | 14 | private static org.slf4j.Logger log = LoggerFactory.getLogger(ConfluentCloudService.class); 15 | 16 | private final ObjectMapper objectMapper; 17 | private static final String ccloudExecutable; 18 | 19 | public ConfluentCloudService(ObjectMapper objectMapper) { 20 | this.objectMapper = objectMapper; 21 | } 22 | 23 | public List getServiceAccounts() { 24 | log.info("Fetching service account list from Confluent Cloud via ccloud tool."); 25 | try { 26 | String result = execCmd(new String[]{ccloudExecutable, "service-account", "list", "-o", "json"}); 27 | return objectMapper.readValue(result, new TypeReference>() { 28 | }); 29 | } catch (IOException ex) { 30 | log.info(ex.getMessage()); 31 | throw new ConfluentCloudException("There was an error listing Confluent Cloud service accounts. Are you logged in?"); 32 | } 33 | } 34 | 35 | public ServiceAccount createServiceAccount(String name, boolean isUser) { 36 | log.info("Creating service account {} in Confluent Cloud via ccloud tool.", name); 37 | try { 38 | String serviceName = isUser ? String.format("user-%s", name) : name; 39 | String description = isUser ? String.format("User: %s", name) : String.format("Service account: %s", name); 40 | String result = execCmd(new String[]{ccloudExecutable, "service-account", "create", serviceName, "--description", description, "-o", "json"}); 41 | return objectMapper.readValue(result, ServiceAccount.class); 42 | } catch (IOException ex) { 43 | throw new ConfluentCloudException(String.format("There was an error creating Confluent Cloud service account: %s.", name)); 44 | } 45 | } 46 | 47 | public static String execCmd(String[] cmd) throws java.io.IOException { 48 | try (java.util.Scanner s = new java.util.Scanner(Runtime.getRuntime().exec(cmd).getInputStream()).useDelimiter("\\A");) { 49 | return s.hasNext() ? s.next() : ""; 50 | } 51 | } 52 | 53 | static { 54 | ccloudExecutable = System.getenv("CCLOUD_EXECUTABLE_PATH") != null ? System.getenv("CCLOUD_EXECUTABLE_PATH") : "ccloud"; 55 | log.info("Using ccloud executable at: {}", ccloudExecutable); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/service/RoleService.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.service; 2 | 3 | import com.devshawn.kafka.gitops.domain.state.AclDetails; 4 | import com.devshawn.kafka.gitops.domain.state.ServiceDetails; 5 | import com.devshawn.kafka.gitops.exception.ValidationException; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import java.util.Optional; 10 | 11 | public class RoleService extends ServiceDetails { 12 | 13 | public List getAcls(String role, String principal) { 14 | switch (role.toLowerCase()) { 15 | case "reader": 16 | return getReaderAcls(principal); 17 | case "writer": 18 | return getWriterAcls(principal); 19 | case "operator": 20 | return getOperatorAcls(principal); 21 | default: 22 | throw new ValidationException(String.format("Role '%s' does not exist. Supported roles: 'reader', 'writer', 'operator'.", role)); 23 | } 24 | } 25 | 26 | private List getReaderAcls(String principal) { 27 | List acls = new ArrayList<>(); 28 | acls.add(generateReadAcl("*", Optional.of(principal))); 29 | acls.add(generateConsumerGroupAcl("*", Optional.of(principal), "READ")); 30 | return acls; 31 | } 32 | 33 | private List getWriterAcls(String principal) { 34 | List acls = new ArrayList<>(); 35 | acls.add(generateWriteACL("*", Optional.of(principal))); 36 | return acls; 37 | } 38 | 39 | private List getOperatorAcls(String principal) { 40 | List acls = new ArrayList<>(); 41 | acls.add(getClusterDescribeAcl(principal)); 42 | acls.add(getWildcardTopicAcl(principal, "DESCRIBE")); 43 | acls.add(getWildcardTopicAcl(principal, "DESCRIBE_CONFIGS")); 44 | acls.add(generateConsumerGroupAcl("*", Optional.of(principal), "READ")); 45 | acls.add(generateConsumerGroupAcl("*", Optional.of(principal), "DESCRIBE")); 46 | return acls; 47 | } 48 | 49 | private AclDetails.Builder getWildcardTopicAcl(String principal, String operation) { 50 | return new AclDetails.Builder() 51 | .setHost("*") 52 | .setType("TOPIC") 53 | .setPermission("ALLOW") 54 | .setPrincipal(principal) 55 | .setOperation(operation) 56 | .setPattern("LITERAL") 57 | .setName("*"); 58 | } 59 | 60 | private AclDetails.Builder getClusterDescribeAcl(String principal) { 61 | return new AclDetails.Builder() 62 | .setHost("*") 63 | .setType("CLUSTER") 64 | .setPermission("ALLOW") 65 | .setPrincipal(principal) 66 | .setOperation("DESCRIBE") 67 | .setPattern("LITERAL") 68 | .setName("kafka-cluster"); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/util/HelperUtil.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.util; 2 | 3 | import java.util.ArrayList; 4 | import java.util.LinkedHashSet; 5 | import java.util.List; 6 | import java.util.Set; 7 | 8 | public class HelperUtil { 9 | 10 | public static List uniqueCombine(List listOne, List listTwo) { 11 | Set set = new LinkedHashSet<>(listOne); 12 | set.addAll(listTwo); 13 | return new ArrayList<>(set); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/util/PlanUtil.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.util; 2 | 3 | import com.devshawn.kafka.gitops.domain.plan.DesiredPlan; 4 | import com.devshawn.kafka.gitops.domain.plan.PlanOverview; 5 | import com.devshawn.kafka.gitops.enums.PlanAction; 6 | 7 | import java.util.EnumMap; 8 | import java.util.EnumSet; 9 | 10 | public class PlanUtil { 11 | 12 | public static PlanOverview getOverview(DesiredPlan desiredPlan, boolean deleteDisabled, boolean skipAclsDisabled) { 13 | EnumMap map = getPlanActionMap(); 14 | desiredPlan.getTopicPlans().forEach(it -> addToMap(map, it.getAction(), deleteDisabled)); 15 | if(!skipAclsDisabled) { 16 | desiredPlan.getAclPlans().forEach(it -> addToMap(map, it.getAction(), deleteDisabled)); 17 | } 18 | return buildPlanOverview(map); 19 | } 20 | 21 | public static PlanOverview getTopicPlanOverview(DesiredPlan desiredPlan, boolean deleteDisabled) { 22 | EnumMap map = getPlanActionMap(); 23 | desiredPlan.getTopicPlans().forEach(it -> addToMap(map, it.getAction(), deleteDisabled)); 24 | return buildPlanOverview(map); 25 | } 26 | 27 | public static PlanOverview getAclPlanOverview(DesiredPlan desiredPlan, boolean deleteDisabled) { 28 | EnumMap map = getPlanActionMap(); 29 | desiredPlan.getAclPlans().forEach(it -> addToMap(map, it.getAction(), deleteDisabled)); 30 | return buildPlanOverview(map); 31 | } 32 | 33 | private static void addToMap(EnumMap map, PlanAction planAction, boolean deleteDisabled) { 34 | if (!(deleteDisabled && planAction == PlanAction.REMOVE)) { 35 | map.put(planAction, map.get(planAction) + 1); 36 | } 37 | } 38 | 39 | private static EnumMap getPlanActionMap() { 40 | EnumMap map = new EnumMap<>(PlanAction.class); 41 | EnumSet.allOf(PlanAction.class).forEach(it -> map.putIfAbsent(it, 0L)); 42 | return map; 43 | } 44 | 45 | private static PlanOverview buildPlanOverview(EnumMap map) { 46 | return new PlanOverview.Builder() 47 | .setAdd(map.get(PlanAction.ADD)) 48 | .setUpdate(map.get(PlanAction.UPDATE)) 49 | .setRemove(map.get(PlanAction.REMOVE)) 50 | .setNoChange(map.get(PlanAction.NO_CHANGE)) 51 | .build(); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/devshawn/kafka/gitops/util/StateUtil.java: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.util; 2 | 3 | import com.devshawn.kafka.gitops.domain.state.DesiredStateFile; 4 | 5 | import java.util.Optional; 6 | 7 | public class StateUtil { 8 | 9 | public static Optional fetchReplication(DesiredStateFile desiredStateFile) { 10 | if (desiredStateFile.getSettings().isPresent() && desiredStateFile.getSettings().get().getTopics().isPresent() 11 | && desiredStateFile.getSettings().get().getTopics().get().getDefaults().isPresent()) { 12 | return desiredStateFile.getSettings().get().getTopics().get().getDefaults().get().getReplication(); 13 | } 14 | return Optional.empty(); 15 | } 16 | 17 | public static boolean isDescribeTopicAclEnabled(DesiredStateFile desiredStateFile) { 18 | return desiredStateFile.getSettings().isPresent() && desiredStateFile.getSettings().get().getServices().isPresent() 19 | && desiredStateFile.getSettings().get().getServices().get().getAcls().isPresent() 20 | && desiredStateFile.getSettings().get().getServices().get().getAcls().get().getDescribeTopicEnabled().isPresent() 21 | && desiredStateFile.getSettings().get().getServices().get().getAcls().get().getDescribeTopicEnabled().get(); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/test/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devshawn/kafka-gitops/8f3afc93f386ce54e6974194375bac8db6a6a354/src/test/.gitkeep -------------------------------------------------------------------------------- /src/test/groovy/com/devshawn/kafka/gitops/ValidateCommandIntegrationSpec.groovy: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops 2 | 3 | 4 | import picocli.CommandLine 5 | import spock.lang.Specification 6 | import spock.lang.Unroll 7 | 8 | @Unroll 9 | class ValidateCommandIntegrationSpec extends Specification { 10 | 11 | void 'test validate state files - #planName'() { 12 | setup: 13 | ByteArrayOutputStream out = new ByteArrayOutputStream() 14 | PrintStream oldOut = System.out 15 | System.setOut(new PrintStream(out)) 16 | String file = TestUtils.getResourceFilePath("plans/${planName}.yaml") 17 | MainCommand mainCommand = new MainCommand() 18 | CommandLine cmd = new CommandLine(mainCommand) 19 | 20 | when: 21 | int exitCode = cmd.execute("-f", file, "validate") 22 | 23 | then: 24 | exitCode == 0 25 | out.toString() == "[VALID] Successfully validated the desired state file.\n" 26 | 27 | cleanup: 28 | System.setOut(oldOut) 29 | 30 | where: 31 | planName << [ 32 | "simple", 33 | "application-service", 34 | "kafka-connect-service", 35 | "kafka-streams-service", 36 | "topics-and-services", 37 | "multi-file" 38 | ] 39 | } 40 | 41 | void 'test invalid state files - #planName'() { 42 | setup: 43 | ByteArrayOutputStream out = new ByteArrayOutputStream() 44 | PrintStream oldOut = System.out 45 | System.setOut(new PrintStream(out)) 46 | String file = TestUtils.getResourceFilePath("plans/${planName}.yaml") 47 | String output = TestUtils.getResourceFileContent("plans/${planName}-validate-output.txt") 48 | MainCommand mainCommand = new MainCommand() 49 | CommandLine cmd = new CommandLine(mainCommand) 50 | 51 | when: 52 | int exitCode = cmd.execute("-f", file, "validate") 53 | 54 | then: 55 | exitCode == 2 56 | out.toString() == output 57 | 58 | cleanup: 59 | System.setOut(oldOut) 60 | 61 | where: 62 | planName << [ 63 | "invalid-custom-service-acls-1", 64 | "invalid-custom-service-acls-2", 65 | "invalid-custom-user-acls-1", 66 | "invalid-custom-user-acls-2" 67 | ] 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/test/groovy/com/devshawn/kafka/gitops/config/KafkaGitopsConfigLoaderSpec.groovy: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.config 2 | 3 | import org.apache.kafka.clients.CommonClientConfigs 4 | import org.apache.kafka.common.config.SaslConfigs 5 | import org.junit.ClassRule 6 | import org.junit.contrib.java.lang.system.EnvironmentVariables 7 | import spock.lang.Shared 8 | import spock.lang.Specification 9 | 10 | class KafkaGitopsConfigLoaderSpec extends Specification { 11 | 12 | @Shared 13 | @ClassRule 14 | EnvironmentVariables environmentVariables 15 | 16 | void setupSpec() { 17 | environmentVariables.set("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092") 18 | environmentVariables.set("KAFKA_SASL_MECHANISM", "PLAIN") 19 | environmentVariables.set("KAFKA_SECURITY_PROTOCOL", "SASL_PLAINTEXT") 20 | } 21 | 22 | void 'test username and password shortcut'() { 23 | setup: 24 | environmentVariables.set("KAFKA_SASL_JAAS_USERNAME", "test") 25 | environmentVariables.set("KAFKA_SASL_JAAS_PASSWORD", "test-secret") 26 | 27 | when: 28 | KafkaGitopsConfig config = KafkaGitopsConfigLoader.load() 29 | 30 | then: 31 | config 32 | config.config.get(SaslConfigs.SASL_JAAS_CONFIG) == "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"test\" password=\"test-secret\";" 33 | } 34 | 35 | void 'test escaping username and password shortcut'() { 36 | setup: 37 | environmentVariables.set("KAFKA_SASL_JAAS_USERNAME", "te\"st") 38 | environmentVariables.set("KAFKA_SASL_JAAS_PASSWORD", "te\"st-secr\"et") 39 | 40 | when: 41 | KafkaGitopsConfig config = KafkaGitopsConfigLoader.load() 42 | 43 | then: 44 | config 45 | config.config.get(SaslConfigs.SASL_JAAS_CONFIG) == "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"te\\\"st\" password=\"te\\\"st-secr\\\"et\";" 46 | } 47 | 48 | void 'test command config file'() { 49 | setup: 50 | File configFile = new File(getClass().getResource("/command.properties").toURI()) 51 | 52 | when: 53 | KafkaGitopsConfig config = KafkaGitopsConfigLoader.load(configFile) 54 | 55 | then: 56 | config.config.get(CommonClientConfigs.CLIENT_ID_CONFIG) == "kafka-gitops" 57 | config.config.get(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG) == "localhost:9092" 58 | config.config.get(SaslConfigs.SASL_MECHANISM) == "PLAIN" 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /src/test/groovy/com/devshawn/kafka/gitops/domain/state/AclDetailsSpec.groovy: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state 2 | 3 | import spock.lang.Specification 4 | import spock.lang.Unroll 5 | 6 | class AclDetailsSpec extends Specification { 7 | 8 | @Unroll 9 | void 'test fromCustomAclDetails - #name'() { 10 | setup: 11 | CustomAclDetails customAclDetails = new CustomAclDetails.Builder() 12 | .setName(name) 13 | .setType(type) 14 | .setPattern(pattern) 15 | .setHost(host) 16 | .setOperation(operation) 17 | .setPermission(permission) 18 | .setPrincipal(Optional.ofNullable(principal)) 19 | .build() 20 | 21 | when: 22 | AclDetails.Builder result = AclDetails.fromCustomAclDetails(customAclDetails) 23 | 24 | then: 25 | result 26 | result.name == name 27 | result.type == type 28 | result.pattern == pattern 29 | result.host == host 30 | result.operation == operation 31 | result.permission == permission 32 | 33 | when: 34 | result.principal == null 35 | 36 | then: 37 | thrown(IllegalStateException) 38 | 39 | where: 40 | name | type | pattern | host | operation | permission | principal 41 | "test" | "TOPIC" | "LITERAL" | "*" | "DESCRIBE" | "ALLOW" | null 42 | "topic-one" | "TOPIC" | "LITERAL" | "*" | "WRITE" | "DENY" | "super.admin" 43 | "another-name" | "GROUP" | "PREFIXED" | "localhost" | "READ" | "ALLOW" | "User:12345" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/test/groovy/com/devshawn/kafka/gitops/domain/state/ServiceDetailsSpec.groovy: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state 2 | 3 | import com.devshawn.kafka.gitops.domain.options.GetAclOptions 4 | import spock.lang.Specification 5 | 6 | class ServiceDetailsSpec extends Specification { 7 | 8 | void 'test default getAcls'() { 9 | setup: 10 | ServiceDetails serviceDetails = new ServiceDetails() {} 11 | 12 | when: 13 | serviceDetails.getAcls(new GetAclOptions.Builder().buildPartial()) 14 | 15 | then: 16 | UnsupportedOperationException ex = thrown(UnsupportedOperationException) 17 | ex.message == "Method getAcls is not implemented." 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/test/groovy/com/devshawn/kafka/gitops/domain/state/service/ApplicationServiceSpec.groovy: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.domain.state.service 2 | 3 | import com.devshawn.kafka.gitops.domain.options.GetAclOptions 4 | import com.devshawn.kafka.gitops.domain.state.AclDetails 5 | import spock.lang.Specification 6 | 7 | class ApplicationServiceSpec extends Specification { 8 | 9 | void 'test consumer and producer ACLs'() { 10 | setup: 11 | ApplicationService sut = new ApplicationService.Builder() 12 | .setPrincipal("principal") 13 | .addConsumes("consumer-topic") 14 | .addProduces("producer-topic") 15 | .build() 16 | 17 | when: 18 | List result = sut.getAcls(new GetAclOptions.Builder() 19 | .setServiceName("application-serivce").setDescribeAclEnabled(false).build()) 20 | 21 | then: 22 | result 23 | result.size() == 3 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/test/groovy/com/devshawn/kafka/gitops/exception/PlanIsUpToDateExceptionSpec.groovy: -------------------------------------------------------------------------------- 1 | package com.devshawn.kafka.gitops.exception 2 | 3 | import spock.lang.Specification 4 | 5 | class PlanIsUpToDateExceptionSpec extends Specification { 6 | 7 | void 'test PlanIsUpToDateException'() { 8 | when: 9 | PlanIsUpToDateException result = new PlanIsUpToDateException() 10 | 11 | then: 12 | result.message == "The current desired state file matches the actual state of the cluster." 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/test/resources/command.properties: -------------------------------------------------------------------------------- 1 | bootstrap.servers=commande.9092 2 | client.id=kafka-gitops -------------------------------------------------------------------------------- /src/test/resources/plans/application-service-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [CREATE] 4 | 5 | + [ACL] test-service-0 6 | + resource_name: test-topic 7 | + resource_type: TOPIC 8 | + resource_pattern: LITERAL 9 | + resource_principal: User:test 10 | + host: * 11 | + operation: WRITE 12 | + permission: ALLOW 13 | 14 | 15 | Successfully applied. 16 | 17 | Applying: [CREATE] 18 | 19 | + [ACL] test-service-1 20 | + resource_name: another-test-topic 21 | + resource_type: TOPIC 22 | + resource_pattern: LITERAL 23 | + resource_principal: User:test 24 | + host: * 25 | + operation: READ 26 | + permission: ALLOW 27 | 28 | 29 | Successfully applied. 30 | 31 | Applying: [CREATE] 32 | 33 | + [ACL] test-service-2 34 | + resource_name: test-service 35 | + resource_type: GROUP 36 | + resource_pattern: LITERAL 37 | + resource_principal: User:test 38 | + host: * 39 | + operation: READ 40 | + permission: ALLOW 41 | 42 | 43 | Successfully applied. 44 | 45 | [SUCCESS] Apply complete! Resources: 3 created, 0 updated, 0 deleted. 46 | -------------------------------------------------------------------------------- /src/test/resources/plans/application-service-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [], 3 | "aclPlans": [ 4 | { 5 | "name": "test-service-0", 6 | "aclDetails": { 7 | "name": "test-topic", 8 | "type": "TOPIC", 9 | "pattern": "LITERAL", 10 | "principal": "User:test", 11 | "host": "*", 12 | "operation": "WRITE", 13 | "permission": "ALLOW" 14 | }, 15 | "action": "ADD" 16 | }, 17 | { 18 | "name": "test-service-1", 19 | "aclDetails": { 20 | "name": "another-test-topic", 21 | "type": "TOPIC", 22 | "pattern": "LITERAL", 23 | "principal": "User:test", 24 | "host": "*", 25 | "operation": "READ", 26 | "permission": "ALLOW" 27 | }, 28 | "action": "ADD" 29 | }, 30 | { 31 | "name": "test-service-2", 32 | "aclDetails": { 33 | "name": "test-service", 34 | "type": "GROUP", 35 | "pattern": "LITERAL", 36 | "principal": "User:test", 37 | "host": "*", 38 | "operation": "READ", 39 | "permission": "ALLOW" 40 | }, 41 | "action": "ADD" 42 | } 43 | ] 44 | } -------------------------------------------------------------------------------- /src/test/resources/plans/application-service.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | test-service: 3 | type: application 4 | principal: User:test 5 | produces: 6 | - test-topic 7 | consumes: 8 | - another-test-topic -------------------------------------------------------------------------------- /src/test/resources/plans/custom-application-id-streams.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | streams-application: 3 | type: kafka-streams 4 | principal: User:test 5 | application-id: test-streams 6 | consumes: 7 | - my-streams-topic 8 | produces: 9 | - my-other-streams-topic 10 | -------------------------------------------------------------------------------- /src/test/resources/plans/custom-group-id-application-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [CREATE] 4 | 5 | + [ACL] test-service-0 6 | + resource_name: test-topic 7 | + resource_type: TOPIC 8 | + resource_pattern: LITERAL 9 | + resource_principal: User:test 10 | + host: * 11 | + operation: WRITE 12 | + permission: ALLOW 13 | 14 | 15 | Successfully applied. 16 | 17 | Applying: [CREATE] 18 | 19 | + [ACL] test-service-1 20 | + resource_name: another-test-topic 21 | + resource_type: TOPIC 22 | + resource_pattern: LITERAL 23 | + resource_principal: User:test 24 | + host: * 25 | + operation: READ 26 | + permission: ALLOW 27 | 28 | 29 | Successfully applied. 30 | 31 | Applying: [CREATE] 32 | 33 | + [ACL] test-service-2 34 | + resource_name: test-service-application 35 | + resource_type: GROUP 36 | + resource_pattern: LITERAL 37 | + resource_principal: User:test 38 | + host: * 39 | + operation: READ 40 | + permission: ALLOW 41 | 42 | 43 | Successfully applied. 44 | 45 | [SUCCESS] Apply complete! Resources: 3 created, 0 updated, 0 deleted. 46 | -------------------------------------------------------------------------------- /src/test/resources/plans/custom-group-id-application-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [], 3 | "aclPlans": [ 4 | { 5 | "name": "test-service-0", 6 | "aclDetails": { 7 | "name": "test-topic", 8 | "type": "TOPIC", 9 | "pattern": "LITERAL", 10 | "principal": "User:test", 11 | "host": "*", 12 | "operation": "WRITE", 13 | "permission": "ALLOW" 14 | }, 15 | "action": "ADD" 16 | }, 17 | { 18 | "name": "test-service-1", 19 | "aclDetails": { 20 | "name": "another-test-topic", 21 | "type": "TOPIC", 22 | "pattern": "LITERAL", 23 | "principal": "User:test", 24 | "host": "*", 25 | "operation": "READ", 26 | "permission": "ALLOW" 27 | }, 28 | "action": "ADD" 29 | }, 30 | { 31 | "name": "test-service-2", 32 | "aclDetails": { 33 | "name": "test-service-application", 34 | "type": "GROUP", 35 | "pattern": "LITERAL", 36 | "principal": "User:test", 37 | "host": "*", 38 | "operation": "READ", 39 | "permission": "ALLOW" 40 | }, 41 | "action": "ADD" 42 | } 43 | ] 44 | } -------------------------------------------------------------------------------- /src/test/resources/plans/custom-group-id-application.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | test-service: 3 | type: application 4 | group-id: test-service-application 5 | principal: User:test 6 | produces: 7 | - test-topic 8 | consumes: 9 | - another-test-topic -------------------------------------------------------------------------------- /src/test/resources/plans/custom-group-id-connect-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [CREATE] 4 | 5 | + [ACL] test-connect-cluster-0 6 | + resource_name: connect-configs-test-connect-cluster 7 | + resource_type: TOPIC 8 | + resource_pattern: LITERAL 9 | + resource_principal: User:test 10 | + host: * 11 | + operation: READ 12 | + permission: ALLOW 13 | 14 | 15 | Successfully applied. 16 | 17 | Applying: [CREATE] 18 | 19 | + [ACL] test-connect-cluster-1 20 | + resource_name: connect-offsets-test-connect-cluster 21 | + resource_type: TOPIC 22 | + resource_pattern: LITERAL 23 | + resource_principal: User:test 24 | + host: * 25 | + operation: READ 26 | + permission: ALLOW 27 | 28 | 29 | Successfully applied. 30 | 31 | Applying: [CREATE] 32 | 33 | + [ACL] test-connect-cluster-2 34 | + resource_name: connect-status-test-connect-cluster 35 | + resource_type: TOPIC 36 | + resource_pattern: LITERAL 37 | + resource_principal: User:test 38 | + host: * 39 | + operation: READ 40 | + permission: ALLOW 41 | 42 | 43 | Successfully applied. 44 | 45 | Applying: [CREATE] 46 | 47 | + [ACL] test-connect-cluster-3 48 | + resource_name: connect-configs-test-connect-cluster 49 | + resource_type: TOPIC 50 | + resource_pattern: LITERAL 51 | + resource_principal: User:test 52 | + host: * 53 | + operation: WRITE 54 | + permission: ALLOW 55 | 56 | 57 | Successfully applied. 58 | 59 | Applying: [CREATE] 60 | 61 | + [ACL] test-connect-cluster-4 62 | + resource_name: connect-offsets-test-connect-cluster 63 | + resource_type: TOPIC 64 | + resource_pattern: LITERAL 65 | + resource_principal: User:test 66 | + host: * 67 | + operation: WRITE 68 | + permission: ALLOW 69 | 70 | 71 | Successfully applied. 72 | 73 | Applying: [CREATE] 74 | 75 | + [ACL] test-connect-cluster-5 76 | + resource_name: connect-status-test-connect-cluster 77 | + resource_type: TOPIC 78 | + resource_pattern: LITERAL 79 | + resource_principal: User:test 80 | + host: * 81 | + operation: WRITE 82 | + permission: ALLOW 83 | 84 | 85 | Successfully applied. 86 | 87 | Applying: [CREATE] 88 | 89 | + [ACL] test-connect-cluster-6 90 | + resource_name: connect-cluster 91 | + resource_type: GROUP 92 | + resource_pattern: LITERAL 93 | + resource_principal: User:test 94 | + host: * 95 | + operation: READ 96 | + permission: ALLOW 97 | 98 | 99 | Successfully applied. 100 | 101 | Applying: [CREATE] 102 | 103 | + [ACL] test-connect-cluster-7 104 | + resource_name: production-topic 105 | + resource_type: TOPIC 106 | + resource_pattern: LITERAL 107 | + resource_principal: User:test 108 | + host: * 109 | + operation: WRITE 110 | + permission: ALLOW 111 | 112 | 113 | Successfully applied. 114 | 115 | Applying: [CREATE] 116 | 117 | + [ACL] test-connect-cluster-8 118 | + resource_name: consumption-topic 119 | + resource_type: TOPIC 120 | + resource_pattern: LITERAL 121 | + resource_principal: User:test 122 | + host: * 123 | + operation: READ 124 | + permission: ALLOW 125 | 126 | 127 | Successfully applied. 128 | 129 | Applying: [CREATE] 130 | 131 | + [ACL] test-connect-cluster-9 132 | + resource_name: connect-test-sink 133 | + resource_type: GROUP 134 | + resource_pattern: LITERAL 135 | + resource_principal: User:test 136 | + host: * 137 | + operation: READ 138 | + permission: ALLOW 139 | 140 | 141 | Successfully applied. 142 | 143 | [SUCCESS] Apply complete! Resources: 10 created, 0 updated, 0 deleted. 144 | -------------------------------------------------------------------------------- /src/test/resources/plans/custom-group-id-connect.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | test-connect-cluster: 3 | type: kafka-connect 4 | group-id: connect-cluster 5 | principal: User:test 6 | connectors: 7 | test-source: 8 | produces: 9 | - production-topic 10 | test-sink: 11 | consumes: 12 | - consumption-topic -------------------------------------------------------------------------------- /src/test/resources/plans/custom-service-acls-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [CREATE] 4 | 5 | + [ACL] test-service-0 6 | + resource_name: kafka. 7 | + resource_type: TOPIC 8 | + resource_pattern: PREFIXED 9 | + resource_principal: User:test 10 | + host: * 11 | + operation: READ 12 | + permission: ALLOW 13 | 14 | 15 | Successfully applied. 16 | 17 | [SUCCESS] Apply complete! Resources: 1 created, 0 updated, 0 deleted. 18 | -------------------------------------------------------------------------------- /src/test/resources/plans/custom-service-acls-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [], 3 | "aclPlans": [ 4 | { 5 | "name": "test-service-0", 6 | "aclDetails": { 7 | "name": "kafka.", 8 | "type": "TOPIC", 9 | "pattern": "PREFIXED", 10 | "principal": "User:test", 11 | "host": "*", 12 | "operation": "READ", 13 | "permission": "ALLOW" 14 | }, 15 | "action": "ADD" 16 | } 17 | ] 18 | } -------------------------------------------------------------------------------- /src/test/resources/plans/custom-service-acls.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | test-service: 3 | type: application 4 | principal: User:test 5 | 6 | customServiceAcls: 7 | test-service: 8 | read-all-kafka: 9 | name: kafka. 10 | type: TOPIC 11 | pattern: PREFIXED 12 | host: "*" 13 | principal: User:test 14 | operation: READ 15 | permission: ALLOW -------------------------------------------------------------------------------- /src/test/resources/plans/custom-storage-topic-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [CREATE] 4 | 5 | + [ACL] test-connect-cluster-0 6 | + resource_name: test-connect-cluster-configs 7 | + resource_type: TOPIC 8 | + resource_pattern: LITERAL 9 | + resource_principal: User:test 10 | + host: * 11 | + operation: READ 12 | + permission: ALLOW 13 | 14 | 15 | Successfully applied. 16 | 17 | Applying: [CREATE] 18 | 19 | + [ACL] test-connect-cluster-1 20 | + resource_name: connect-offsets-test-connect-cluster 21 | + resource_type: TOPIC 22 | + resource_pattern: LITERAL 23 | + resource_principal: User:test 24 | + host: * 25 | + operation: READ 26 | + permission: ALLOW 27 | 28 | 29 | Successfully applied. 30 | 31 | Applying: [CREATE] 32 | 33 | + [ACL] test-connect-cluster-2 34 | + resource_name: connect-status-test-connect-cluster 35 | + resource_type: TOPIC 36 | + resource_pattern: LITERAL 37 | + resource_principal: User:test 38 | + host: * 39 | + operation: READ 40 | + permission: ALLOW 41 | 42 | 43 | Successfully applied. 44 | 45 | Applying: [CREATE] 46 | 47 | + [ACL] test-connect-cluster-3 48 | + resource_name: test-connect-cluster-configs 49 | + resource_type: TOPIC 50 | + resource_pattern: LITERAL 51 | + resource_principal: User:test 52 | + host: * 53 | + operation: WRITE 54 | + permission: ALLOW 55 | 56 | 57 | Successfully applied. 58 | 59 | Applying: [CREATE] 60 | 61 | + [ACL] test-connect-cluster-4 62 | + resource_name: connect-offsets-test-connect-cluster 63 | + resource_type: TOPIC 64 | + resource_pattern: LITERAL 65 | + resource_principal: User:test 66 | + host: * 67 | + operation: WRITE 68 | + permission: ALLOW 69 | 70 | 71 | Successfully applied. 72 | 73 | Applying: [CREATE] 74 | 75 | + [ACL] test-connect-cluster-5 76 | + resource_name: connect-status-test-connect-cluster 77 | + resource_type: TOPIC 78 | + resource_pattern: LITERAL 79 | + resource_principal: User:test 80 | + host: * 81 | + operation: WRITE 82 | + permission: ALLOW 83 | 84 | 85 | Successfully applied. 86 | 87 | Applying: [CREATE] 88 | 89 | + [ACL] test-connect-cluster-6 90 | + resource_name: test-connect-cluster 91 | + resource_type: GROUP 92 | + resource_pattern: LITERAL 93 | + resource_principal: User:test 94 | + host: * 95 | + operation: READ 96 | + permission: ALLOW 97 | 98 | 99 | Successfully applied. 100 | 101 | Applying: [CREATE] 102 | 103 | + [ACL] test-connect-cluster-7 104 | + resource_name: production-topic 105 | + resource_type: TOPIC 106 | + resource_pattern: LITERAL 107 | + resource_principal: User:test 108 | + host: * 109 | + operation: WRITE 110 | + permission: ALLOW 111 | 112 | 113 | Successfully applied. 114 | 115 | Applying: [CREATE] 116 | 117 | + [ACL] test-connect-cluster-8 118 | + resource_name: consumption-topic 119 | + resource_type: TOPIC 120 | + resource_pattern: LITERAL 121 | + resource_principal: User:test 122 | + host: * 123 | + operation: READ 124 | + permission: ALLOW 125 | 126 | 127 | Successfully applied. 128 | 129 | Applying: [CREATE] 130 | 131 | + [ACL] test-connect-cluster-9 132 | + resource_name: connect-test-sink 133 | + resource_type: GROUP 134 | + resource_pattern: LITERAL 135 | + resource_principal: User:test 136 | + host: * 137 | + operation: READ 138 | + permission: ALLOW 139 | 140 | 141 | Successfully applied. 142 | 143 | [SUCCESS] Apply complete! Resources: 10 created, 0 updated, 0 deleted. 144 | -------------------------------------------------------------------------------- /src/test/resources/plans/custom-storage-topic.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | test-connect-cluster: 3 | type: kafka-connect 4 | principal: User:test 5 | storage-topics: 6 | config: test-connect-cluster-configs 7 | connectors: 8 | test-source: 9 | produces: 10 | - production-topic 11 | test-sink: 12 | consumes: 13 | - consumption-topic -------------------------------------------------------------------------------- /src/test/resources/plans/custom-storage-topics-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [CREATE] 4 | 5 | + [ACL] test-connect-cluster-0 6 | + resource_name: config-custom-topic 7 | + resource_type: TOPIC 8 | + resource_pattern: LITERAL 9 | + resource_principal: User:test 10 | + host: * 11 | + operation: READ 12 | + permission: ALLOW 13 | 14 | 15 | Successfully applied. 16 | 17 | Applying: [CREATE] 18 | 19 | + [ACL] test-connect-cluster-1 20 | + resource_name: offset-topic-custom 21 | + resource_type: TOPIC 22 | + resource_pattern: LITERAL 23 | + resource_principal: User:test 24 | + host: * 25 | + operation: READ 26 | + permission: ALLOW 27 | 28 | 29 | Successfully applied. 30 | 31 | Applying: [CREATE] 32 | 33 | + [ACL] test-connect-cluster-2 34 | + resource_name: custom-status-topic 35 | + resource_type: TOPIC 36 | + resource_pattern: LITERAL 37 | + resource_principal: User:test 38 | + host: * 39 | + operation: READ 40 | + permission: ALLOW 41 | 42 | 43 | Successfully applied. 44 | 45 | Applying: [CREATE] 46 | 47 | + [ACL] test-connect-cluster-3 48 | + resource_name: config-custom-topic 49 | + resource_type: TOPIC 50 | + resource_pattern: LITERAL 51 | + resource_principal: User:test 52 | + host: * 53 | + operation: WRITE 54 | + permission: ALLOW 55 | 56 | 57 | Successfully applied. 58 | 59 | Applying: [CREATE] 60 | 61 | + [ACL] test-connect-cluster-4 62 | + resource_name: offset-topic-custom 63 | + resource_type: TOPIC 64 | + resource_pattern: LITERAL 65 | + resource_principal: User:test 66 | + host: * 67 | + operation: WRITE 68 | + permission: ALLOW 69 | 70 | 71 | Successfully applied. 72 | 73 | Applying: [CREATE] 74 | 75 | + [ACL] test-connect-cluster-5 76 | + resource_name: custom-status-topic 77 | + resource_type: TOPIC 78 | + resource_pattern: LITERAL 79 | + resource_principal: User:test 80 | + host: * 81 | + operation: WRITE 82 | + permission: ALLOW 83 | 84 | 85 | Successfully applied. 86 | 87 | Applying: [CREATE] 88 | 89 | + [ACL] test-connect-cluster-6 90 | + resource_name: test-connect-cluster 91 | + resource_type: GROUP 92 | + resource_pattern: LITERAL 93 | + resource_principal: User:test 94 | + host: * 95 | + operation: READ 96 | + permission: ALLOW 97 | 98 | 99 | Successfully applied. 100 | 101 | Applying: [CREATE] 102 | 103 | + [ACL] test-connect-cluster-7 104 | + resource_name: production-topic 105 | + resource_type: TOPIC 106 | + resource_pattern: LITERAL 107 | + resource_principal: User:test 108 | + host: * 109 | + operation: WRITE 110 | + permission: ALLOW 111 | 112 | 113 | Successfully applied. 114 | 115 | Applying: [CREATE] 116 | 117 | + [ACL] test-connect-cluster-8 118 | + resource_name: consumption-topic 119 | + resource_type: TOPIC 120 | + resource_pattern: LITERAL 121 | + resource_principal: User:test 122 | + host: * 123 | + operation: READ 124 | + permission: ALLOW 125 | 126 | 127 | Successfully applied. 128 | 129 | Applying: [CREATE] 130 | 131 | + [ACL] test-connect-cluster-9 132 | + resource_name: connect-test-sink 133 | + resource_type: GROUP 134 | + resource_pattern: LITERAL 135 | + resource_principal: User:test 136 | + host: * 137 | + operation: READ 138 | + permission: ALLOW 139 | 140 | 141 | Successfully applied. 142 | 143 | [SUCCESS] Apply complete! Resources: 10 created, 0 updated, 0 deleted. 144 | -------------------------------------------------------------------------------- /src/test/resources/plans/custom-storage-topics.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | test-connect-cluster: 3 | type: kafka-connect 4 | principal: User:test 5 | storage-topics: 6 | config: config-custom-topic 7 | status: custom-status-topic 8 | offset: offset-topic-custom 9 | connectors: 10 | test-source: 11 | produces: 12 | - production-topic 13 | test-sink: 14 | consumes: 15 | - consumption-topic -------------------------------------------------------------------------------- /src/test/resources/plans/custom-user-acls-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [CREATE] 4 | 5 | + [ACL] test-user-0 6 | + resource_name: kafka. 7 | + resource_type: TOPIC 8 | + resource_pattern: PREFIXED 9 | + resource_principal: User:test 10 | + host: * 11 | + operation: READ 12 | + permission: ALLOW 13 | 14 | 15 | Successfully applied. 16 | 17 | [SUCCESS] Apply complete! Resources: 1 created, 0 updated, 0 deleted. 18 | -------------------------------------------------------------------------------- /src/test/resources/plans/custom-user-acls-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [], 3 | "aclPlans": [ 4 | { 5 | "name": "test-user-0", 6 | "aclDetails": { 7 | "name": "kafka.", 8 | "type": "TOPIC", 9 | "pattern": "PREFIXED", 10 | "principal": "User:test", 11 | "host": "*", 12 | "operation": "READ", 13 | "permission": "ALLOW" 14 | }, 15 | "action": "ADD" 16 | } 17 | ] 18 | } -------------------------------------------------------------------------------- /src/test/resources/plans/custom-user-acls.yaml: -------------------------------------------------------------------------------- 1 | users: 2 | test-user: 3 | principal: User:test 4 | 5 | customUserAcls: 6 | test-user: 7 | read-all-kafka: 8 | name: kafka. 9 | type: TOPIC 10 | pattern: PREFIXED 11 | host: "*" 12 | operation: READ 13 | permission: ALLOW 14 | -------------------------------------------------------------------------------- /src/test/resources/plans/default-replication-multiple-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "test-topic", 5 | "action": "ADD", 6 | "topicDetailsPlan": { 7 | "partitions": 6, 8 | "previousPartitions": null, 9 | "partitionsAction": "ADD", 10 | "replication": 3, 11 | "previousReplication": null, 12 | "replicationAction": "ADD" 13 | }, 14 | "topicConfigPlans": [] 15 | }, 16 | { 17 | "name": "another-topic", 18 | "action": "ADD", 19 | "topicDetailsPlan": { 20 | "partitions": 3, 21 | "previousPartitions": null, 22 | "partitionsAction": "ADD", 23 | "replication": 2, 24 | "previousReplication": null, 25 | "replicationAction": "ADD" 26 | }, 27 | "topicConfigPlans": [] 28 | }, 29 | { 30 | "name": "last-topic", 31 | "action": "ADD", 32 | "topicDetailsPlan": { 33 | "partitions": 3, 34 | "previousPartitions": null, 35 | "partitionsAction": "ADD", 36 | "replication": 4, 37 | "previousReplication": null, 38 | "replicationAction": "ADD" 39 | }, 40 | "topicConfigPlans": [] 41 | } 42 | ], 43 | "aclPlans": [] 44 | } -------------------------------------------------------------------------------- /src/test/resources/plans/default-replication-multiple.yaml: -------------------------------------------------------------------------------- 1 | settings: 2 | topics: 3 | defaults: 4 | replication: 3 5 | 6 | topics: 7 | test-topic: 8 | partitions: 6 9 | 10 | another-topic: 11 | partitions: 3 12 | replication: 2 13 | 14 | last-topic: 15 | partitions: 3 16 | replication: 4 17 | -------------------------------------------------------------------------------- /src/test/resources/plans/default-replication-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "test-topic", 5 | "action": "ADD", 6 | "topicDetailsPlan": { 7 | "partitions": 6, 8 | "previousPartitions": null, 9 | "partitionsAction": "ADD", 10 | "replication": 2, 11 | "previousReplication": null, 12 | "replicationAction": "ADD" 13 | }, 14 | "topicConfigPlans": [] 15 | } 16 | ], 17 | "aclPlans": [] 18 | } -------------------------------------------------------------------------------- /src/test/resources/plans/default-replication.yaml: -------------------------------------------------------------------------------- 1 | settings: 2 | topics: 3 | defaults: 4 | replication: 2 5 | 6 | topics: 7 | test-topic: 8 | partitions: 6 9 | -------------------------------------------------------------------------------- /src/test/resources/plans/describe-topic-acl-disabled.yaml: -------------------------------------------------------------------------------- 1 | settings: 2 | services: 3 | acls: 4 | describeTopicEnabled: false 5 | 6 | services: 7 | normal-application: 8 | type: application 9 | principal: User:test 10 | produces: 11 | - test-topic 12 | consumes: 13 | - first-topic 14 | 15 | streams-application: 16 | type: kafka-streams 17 | principal: User:test 18 | consumes: 19 | - test-topic 20 | produces: 21 | - another-test-topic 22 | 23 | kafka-connect-application: 24 | type: kafka-connect 25 | principal: User:test 26 | produces: 27 | - poison-topic 28 | connectors: 29 | test-source: 30 | produces: 31 | - another-test-topic 32 | test-sink: 33 | consumes: 34 | - test-topic 35 | -------------------------------------------------------------------------------- /src/test/resources/plans/describe-topic-acl-enabled.yaml: -------------------------------------------------------------------------------- 1 | settings: 2 | services: 3 | acls: 4 | describeTopicEnabled: true 5 | 6 | services: 7 | normal-application: 8 | type: application 9 | principal: User:test 10 | produces: 11 | - test-topic 12 | consumes: 13 | - first-topic 14 | 15 | streams-application: 16 | type: kafka-streams 17 | principal: User:test 18 | consumes: 19 | - test-topic 20 | produces: 21 | - another-test-topic 22 | 23 | kafka-connect-application: 24 | type: kafka-connect 25 | principal: User:test 26 | produces: 27 | - poison-topic 28 | connectors: 29 | test-source: 30 | produces: 31 | - another-test-topic 32 | test-sink: 33 | consumes: 34 | - test-topic 35 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-custom-service-acls-1-validate-output.txt: -------------------------------------------------------------------------------- 1 | [INVALID] Custom ACL definition for service 'test-service' is invalid for field 'pattern'. Allowed values: [MATCH, LITERAL, PREFIXED] 2 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-custom-service-acls-1.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | test-service: 3 | type: application 4 | principal: User:test 5 | 6 | customServiceAcls: 7 | test-service: 8 | read-all-kafka: 9 | name: kafka. 10 | type: TOPIC 11 | pattern: PREFIX 12 | host: "*" 13 | principal: User:test 14 | operation: READ 15 | permission: ALLOW -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-custom-service-acls-2-validate-output.txt: -------------------------------------------------------------------------------- 1 | [INVALID] Custom ACL definition for service 'other-service' is invalid for field 'operation'. Allowed values: [ALL, READ, WRITE, CREATE, DELETE, ALTER, DESCRIBE, CLUSTER_ACTION, DESCRIBE_CONFIGS, ALTER_CONFIGS, IDEMPOTENT_WRITE] 2 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-custom-service-acls-2.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | other-service: 3 | type: application 4 | principal: User:test 5 | 6 | customServiceAcls: 7 | other-service: 8 | read-all-kafka: 9 | name: kafka. 10 | type: TOPIC 11 | pattern: PREFIXED 12 | host: "*" 13 | principal: User:test 14 | operation: ANY 15 | permission: ALLOW -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-custom-user-acls-1-validate-output.txt: -------------------------------------------------------------------------------- 1 | [INVALID] Custom ACL definition for user 'test-user' is invalid for field 'type'. Allowed values: [TOPIC, GROUP, CLUSTER, TRANSACTIONAL_ID, DELEGATION_TOKEN] 2 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-custom-user-acls-1.yaml: -------------------------------------------------------------------------------- 1 | users: 2 | test-user: 3 | principal: User:test 4 | 5 | customUserAcls: 6 | test-user: 7 | read-all-kafka: 8 | name: kafka. 9 | type: UNKNOWN 10 | pattern: PREFIXED 11 | host: "*" 12 | operation: READ 13 | permission: ALLOW 14 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-custom-user-acls-2-validate-output.txt: -------------------------------------------------------------------------------- 1 | [INVALID] Custom ACL definition for user 'my-user' is invalid for field 'permission'. Allowed values: [DENY, ALLOW] 2 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-custom-user-acls-2.yaml: -------------------------------------------------------------------------------- 1 | users: 2 | my-user: 3 | principal: User:test 4 | 5 | customUserAcls: 6 | my-user: 7 | read-all-kafka: 8 | name: kafka. 9 | type: TOPIC 10 | pattern: PREFIXED 11 | host: "*" 12 | operation: READ 13 | permission: NONE 14 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-default-replication-1-output.txt: -------------------------------------------------------------------------------- 1 | Generating execution plan... 2 | 3 | [INVALID] The default replication factor must be a positive integer. 4 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-default-replication-1.yaml: -------------------------------------------------------------------------------- 1 | settings: 2 | topics: 3 | defaults: 4 | replication: -1 5 | 6 | topics: 7 | test-topic: 8 | partitions: 6 9 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-default-replication-2-output.txt: -------------------------------------------------------------------------------- 1 | Generating execution plan... 2 | 3 | [INVALID] Not set: [replication] in state file definition: topics -> test-topic 4 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-default-replication-2.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | test-topic: 3 | partitions: 6 4 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-format-output.txt: -------------------------------------------------------------------------------- 1 | Generating execution plan... 2 | 3 | [INVALID] Value 'a' is not a valid format for: [partitions] in state file definition: topics -> test-topic 4 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-format.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | test-topic: 3 | partitions: "a" 4 | replication: 3 5 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-missing-principal-output.txt: -------------------------------------------------------------------------------- 1 | Generating execution plan... 2 | 3 | [ERROR] Missing required configuration: Not set: [principal] for service: test-service 4 | 5 | [ERROR] An error has occurred during the planning process. No plan was created. 6 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-missing-principal.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | test-service: 3 | type: application 4 | produces: 5 | - test-topic -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-missing-user-principal-output.txt: -------------------------------------------------------------------------------- 1 | Generating execution plan... 2 | 3 | [ERROR] Missing required configuration: Missing principal for user test-user 4 | 5 | [ERROR] An error has occurred during the planning process. No plan was created. 6 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-missing-user-principal.yaml: -------------------------------------------------------------------------------- 1 | users: 2 | test-user: 3 | roles: 4 | - reader 5 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-plan-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | [ERROR] Error reading execution plan from file: Please run the plan command again to generate a new plan file. 4 | 5 | [ERROR] An error has occurred during the apply process. 6 | [ERROR] The apply process has stopped in place. There is no rollback. 7 | [ERROR] Fix the error, re-create a plan, and apply the new plan to continue. 8 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": {} 3 | } -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-storage-topics-output.txt: -------------------------------------------------------------------------------- 1 | Generating execution plan... 2 | 3 | [INVALID] Unrecognized field: [configs] in state file definition: services -> test-connect-cluster -> storage-topics 4 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-storage-topics.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | test-connect-cluster: 3 | type: kafka-connect 4 | principal: User:test 5 | storage-topics: 6 | configs: test-config-topic 7 | connectors: 8 | test-source: 9 | produces: 10 | - production-topic 11 | test-sink: 12 | consumes: 13 | - consumption-topic -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-topic-output.txt: -------------------------------------------------------------------------------- 1 | Generating execution plan... 2 | 3 | [INVALID] Not set: [partitions] in state file definition: topics -> test-topic 4 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-topic-remove-partitions-output.txt: -------------------------------------------------------------------------------- 1 | Generating execution plan... 2 | 3 | [INVALID] Removing the partition number is not supported by Apache Kafka (topic: topic-with-configs-2 (6 -> 2)) 4 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-topic-remove-partitions.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | delete-topic: 3 | partitions: 1 4 | replication: 2 5 | 6 | test-topic: 7 | partitions: 1 8 | replication: 2 9 | 10 | topic-with-configs-1: 11 | partitions: 3 12 | replication: 2 13 | configs: 14 | cleanup.policy: compact 15 | segment.bytes: 100000 16 | 17 | topic-with-configs-2: 18 | partitions: 2 19 | replication: 2 20 | configs: 21 | retention.ms: 60000 22 | -------------------------------------------------------------------------------- /src/test/resources/plans/invalid-topic.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | test-topic: 3 | replication: 2 -------------------------------------------------------------------------------- /src/test/resources/plans/kafka-connect-service-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [CREATE] 4 | 5 | + [ACL] test-connect-cluster-0 6 | + resource_name: connect-configs-test-connect-cluster 7 | + resource_type: TOPIC 8 | + resource_pattern: LITERAL 9 | + resource_principal: User:test 10 | + host: * 11 | + operation: READ 12 | + permission: ALLOW 13 | 14 | 15 | Successfully applied. 16 | 17 | Applying: [CREATE] 18 | 19 | + [ACL] test-connect-cluster-1 20 | + resource_name: connect-offsets-test-connect-cluster 21 | + resource_type: TOPIC 22 | + resource_pattern: LITERAL 23 | + resource_principal: User:test 24 | + host: * 25 | + operation: READ 26 | + permission: ALLOW 27 | 28 | 29 | Successfully applied. 30 | 31 | Applying: [CREATE] 32 | 33 | + [ACL] test-connect-cluster-2 34 | + resource_name: connect-status-test-connect-cluster 35 | + resource_type: TOPIC 36 | + resource_pattern: LITERAL 37 | + resource_principal: User:test 38 | + host: * 39 | + operation: READ 40 | + permission: ALLOW 41 | 42 | 43 | Successfully applied. 44 | 45 | Applying: [CREATE] 46 | 47 | + [ACL] test-connect-cluster-3 48 | + resource_name: connect-configs-test-connect-cluster 49 | + resource_type: TOPIC 50 | + resource_pattern: LITERAL 51 | + resource_principal: User:test 52 | + host: * 53 | + operation: WRITE 54 | + permission: ALLOW 55 | 56 | 57 | Successfully applied. 58 | 59 | Applying: [CREATE] 60 | 61 | + [ACL] test-connect-cluster-4 62 | + resource_name: connect-offsets-test-connect-cluster 63 | + resource_type: TOPIC 64 | + resource_pattern: LITERAL 65 | + resource_principal: User:test 66 | + host: * 67 | + operation: WRITE 68 | + permission: ALLOW 69 | 70 | 71 | Successfully applied. 72 | 73 | Applying: [CREATE] 74 | 75 | + [ACL] test-connect-cluster-5 76 | + resource_name: connect-status-test-connect-cluster 77 | + resource_type: TOPIC 78 | + resource_pattern: LITERAL 79 | + resource_principal: User:test 80 | + host: * 81 | + operation: WRITE 82 | + permission: ALLOW 83 | 84 | 85 | Successfully applied. 86 | 87 | Applying: [CREATE] 88 | 89 | + [ACL] test-connect-cluster-6 90 | + resource_name: test-connect-cluster 91 | + resource_type: GROUP 92 | + resource_pattern: LITERAL 93 | + resource_principal: User:test 94 | + host: * 95 | + operation: READ 96 | + permission: ALLOW 97 | 98 | 99 | Successfully applied. 100 | 101 | Applying: [CREATE] 102 | 103 | + [ACL] test-connect-cluster-7 104 | + resource_name: production-topic 105 | + resource_type: TOPIC 106 | + resource_pattern: LITERAL 107 | + resource_principal: User:test 108 | + host: * 109 | + operation: WRITE 110 | + permission: ALLOW 111 | 112 | 113 | Successfully applied. 114 | 115 | Applying: [CREATE] 116 | 117 | + [ACL] test-connect-cluster-8 118 | + resource_name: consumption-topic 119 | + resource_type: TOPIC 120 | + resource_pattern: LITERAL 121 | + resource_principal: User:test 122 | + host: * 123 | + operation: READ 124 | + permission: ALLOW 125 | 126 | 127 | Successfully applied. 128 | 129 | Applying: [CREATE] 130 | 131 | + [ACL] test-connect-cluster-9 132 | + resource_name: connect-test-sink 133 | + resource_type: GROUP 134 | + resource_pattern: LITERAL 135 | + resource_principal: User:test 136 | + host: * 137 | + operation: READ 138 | + permission: ALLOW 139 | 140 | 141 | Successfully applied. 142 | 143 | [SUCCESS] Apply complete! Resources: 10 created, 0 updated, 0 deleted. 144 | -------------------------------------------------------------------------------- /src/test/resources/plans/kafka-connect-service.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | test-connect-cluster: 3 | type: kafka-connect 4 | principal: User:test 5 | connectors: 6 | test-source: 7 | produces: 8 | - production-topic 9 | test-sink: 10 | consumes: 11 | - consumption-topic -------------------------------------------------------------------------------- /src/test/resources/plans/kafka-streams-service.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | streams-application: 3 | type: kafka-streams 4 | principal: User:test 5 | consumes: 6 | - my-streams-topic 7 | produces: 8 | - my-other-streams-topic 9 | - another-topic -------------------------------------------------------------------------------- /src/test/resources/plans/multi-file-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [CREATE] 4 | 5 | + [TOPIC] test-topic 6 | + partitions: 6 7 | + replication: 2 8 | 9 | 10 | Successfully applied. 11 | 12 | Applying: [CREATE] 13 | 14 | + [ACL] test-service-0 15 | + resource_name: test-topic 16 | + resource_type: TOPIC 17 | + resource_pattern: LITERAL 18 | + resource_principal: User:test 19 | + host: * 20 | + operation: WRITE 21 | + permission: ALLOW 22 | 23 | 24 | Successfully applied. 25 | 26 | Applying: [CREATE] 27 | 28 | + [ACL] test-service-1 29 | + resource_name: another-test-topic 30 | + resource_type: TOPIC 31 | + resource_pattern: LITERAL 32 | + resource_principal: User:test 33 | + host: * 34 | + operation: READ 35 | + permission: ALLOW 36 | 37 | 38 | Successfully applied. 39 | 40 | Applying: [CREATE] 41 | 42 | + [ACL] test-service-2 43 | + resource_name: test-service 44 | + resource_type: GROUP 45 | + resource_pattern: LITERAL 46 | + resource_principal: User:test 47 | + host: * 48 | + operation: READ 49 | + permission: ALLOW 50 | 51 | 52 | Successfully applied. 53 | 54 | [SUCCESS] Apply complete! Resources: 4 created, 0 updated, 0 deleted. 55 | -------------------------------------------------------------------------------- /src/test/resources/plans/multi-file-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "test-topic", 5 | "action": "ADD", 6 | "topicDetailsPlan": { 7 | "partitions": 6, 8 | "previousPartitions": null, 9 | "partitionsAction": "ADD", 10 | "replication": 2, 11 | "previousReplication": null, 12 | "replicationAction": "ADD" 13 | }, 14 | "topicConfigPlans": [] 15 | } 16 | ], 17 | "aclPlans": [ 18 | { 19 | "name": "test-service-0", 20 | "aclDetails": { 21 | "name": "test-topic", 22 | "type": "TOPIC", 23 | "pattern": "LITERAL", 24 | "principal": "User:test", 25 | "host": "*", 26 | "operation": "WRITE", 27 | "permission": "ALLOW" 28 | }, 29 | "action": "ADD" 30 | }, 31 | { 32 | "name": "test-service-1", 33 | "aclDetails": { 34 | "name": "another-test-topic", 35 | "type": "TOPIC", 36 | "pattern": "LITERAL", 37 | "principal": "User:test", 38 | "host": "*", 39 | "operation": "READ", 40 | "permission": "ALLOW" 41 | }, 42 | "action": "ADD" 43 | }, 44 | { 45 | "name": "test-service-2", 46 | "aclDetails": { 47 | "name": "test-service", 48 | "type": "GROUP", 49 | "pattern": "LITERAL", 50 | "principal": "User:test", 51 | "host": "*", 52 | "operation": "READ", 53 | "permission": "ALLOW" 54 | }, 55 | "action": "ADD" 56 | } 57 | ] 58 | } -------------------------------------------------------------------------------- /src/test/resources/plans/multi-file-services.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | test-service: 3 | type: application 4 | principal: User:test 5 | produces: 6 | - test-topic 7 | consumes: 8 | - another-test-topic -------------------------------------------------------------------------------- /src/test/resources/plans/multi-file-topics.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | test-topic: 3 | partitions: 6 4 | replication: 2 -------------------------------------------------------------------------------- /src/test/resources/plans/multi-file-users.yaml: -------------------------------------------------------------------------------- 1 | users: {} -------------------------------------------------------------------------------- /src/test/resources/plans/multi-file.yaml: -------------------------------------------------------------------------------- 1 | settings: 2 | files: 3 | services: multi-file-services.yaml 4 | topics: multi-file-topics.yaml 5 | users: multi-file-users.yaml 6 | -------------------------------------------------------------------------------- /src/test/resources/plans/no-changes-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | [SUCCESS] There are no necessary changes; the actual state matches the desired state. 4 | -------------------------------------------------------------------------------- /src/test/resources/plans/no-changes-include-unchanged-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | [SUCCESS] There are no necessary changes; the actual state matches the desired state. 4 | -------------------------------------------------------------------------------- /src/test/resources/plans/no-changes-include-unchanged-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "delete-topic", 5 | "action": "NO_CHANGE", 6 | "topicDetailsPlan": { 7 | "partitions": 1, 8 | "previousPartitions": null, 9 | "partitionsAction": "NO_CHANGE", 10 | "replication": 2, 11 | "previousReplication": null, 12 | "replicationAction": "NO_CHANGE" 13 | }, 14 | "topicConfigPlans": [] 15 | }, 16 | { 17 | "name": "test-topic", 18 | "action": "NO_CHANGE", 19 | "topicDetailsPlan": { 20 | "partitions": 1, 21 | "previousPartitions": null, 22 | "partitionsAction": "NO_CHANGE", 23 | "replication": 2, 24 | "previousReplication": null, 25 | "replicationAction": "NO_CHANGE" 26 | }, 27 | "topicConfigPlans": [] 28 | }, 29 | { 30 | "name": "topic-with-configs-1", 31 | "action": "NO_CHANGE", 32 | "topicDetailsPlan": { 33 | "partitions": 3, 34 | "previousPartitions": null, 35 | "partitionsAction": "NO_CHANGE", 36 | "replication": 2, 37 | "previousReplication": null, 38 | "replicationAction": "NO_CHANGE" 39 | }, 40 | "topicConfigPlans": [ 41 | { 42 | "key": "cleanup.policy", 43 | "value": "compact", 44 | "previousValue": null, 45 | "action": "NO_CHANGE" 46 | }, 47 | { 48 | "key": "segment.bytes", 49 | "value": "100000", 50 | "previousValue": null, 51 | "action": "NO_CHANGE" 52 | } 53 | ] 54 | }, 55 | { 56 | "name": "topic-with-configs-2", 57 | "action": "NO_CHANGE", 58 | "topicDetailsPlan": { 59 | "partitions": 6, 60 | "previousPartitions": null, 61 | "partitionsAction": "NO_CHANGE", 62 | "replication": 2, 63 | "previousReplication": null, 64 | "replicationAction": "NO_CHANGE" 65 | }, 66 | "topicConfigPlans": [ 67 | { 68 | "key": "retention.ms", 69 | "value": "60000", 70 | "previousValue": null, 71 | "action": "NO_CHANGE" 72 | } 73 | ] 74 | } 75 | ], 76 | "aclPlans": [ 77 | { 78 | "name": "test-service-0", 79 | "aclDetails": { 80 | "name": "test-topic", 81 | "type": "TOPIC", 82 | "pattern": "LITERAL", 83 | "principal": "User:test", 84 | "host": "*", 85 | "operation": "READ", 86 | "permission": "ALLOW" 87 | }, 88 | "action": "NO_CHANGE" 89 | } 90 | ] 91 | } -------------------------------------------------------------------------------- /src/test/resources/plans/no-changes-output.txt: -------------------------------------------------------------------------------- 1 | Generating execution plan... 2 | 3 | [SUCCESS] There are no necessary changes; the actual state matches the desired state. 4 | -------------------------------------------------------------------------------- /src/test/resources/plans/no-changes-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [], 3 | "aclPlans": [] 4 | } -------------------------------------------------------------------------------- /src/test/resources/plans/no-changes.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | delete-topic: 3 | partitions: 1 4 | replication: 2 5 | 6 | test-topic: 7 | partitions: 1 8 | replication: 2 9 | 10 | topic-with-configs-1: 11 | partitions: 3 12 | replication: 2 13 | configs: 14 | cleanup.policy: compact 15 | segment.bytes: 100000 16 | 17 | topic-with-configs-2: 18 | partitions: 6 19 | replication: 2 20 | configs: 21 | retention.ms: 60000 22 | 23 | services: 24 | test-service: 25 | principal: "User:test" 26 | type: application 27 | 28 | customServiceAcls: 29 | test-service: 30 | read: 31 | name: test-topic 32 | type: TOPIC 33 | pattern: LITERAL 34 | host: "*" 35 | principal: User:test 36 | operation: READ 37 | permission: ALLOW -------------------------------------------------------------------------------- /src/test/resources/plans/null-file-output.txt: -------------------------------------------------------------------------------- 1 | Generating execution plan... 2 | 3 | [INVALID] The specified state file could not be found. 4 | -------------------------------------------------------------------------------- /src/test/resources/plans/read-input-exception-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | [ERROR] Error reading execution plan from file: The specified plan file could not be found. 4 | 5 | [ERROR] An error has occurred during the apply process. 6 | [ERROR] The apply process has stopped in place. There is no rollback. 7 | [ERROR] Fix the error, re-create a plan, and apply the new plan to continue. 8 | -------------------------------------------------------------------------------- /src/test/resources/plans/seed-acl-exists-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [UPDATE] 4 | 5 | ~ [TOPIC] test-topic 6 | ~ configs: 7 | + retention.ms: 60000 8 | 9 | 10 | Successfully applied. 11 | 12 | Applying: [UPDATE] 13 | 14 | ~ [TOPIC] topic-with-configs-1 15 | ~ configs: 16 | - cleanup.policy (compact) 17 | - segment.bytes (100000) 18 | + retention.ms: 100000 19 | 20 | 21 | Successfully applied. 22 | 23 | Applying: [CREATE] 24 | 25 | + [ACL] test-service-1 26 | + resource_name: test-service 27 | + resource_type: GROUP 28 | + resource_pattern: LITERAL 29 | + resource_principal: User:test 30 | + host: * 31 | + operation: READ 32 | + permission: ALLOW 33 | 34 | 35 | Successfully applied. 36 | 37 | [SUCCESS] Apply complete! Resources: 1 created, 2 updated, 0 deleted. 38 | -------------------------------------------------------------------------------- /src/test/resources/plans/seed-acl-exists-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "test-topic", 5 | "action": "UPDATE", 6 | "topicDetailsPlan": null, 7 | "topicConfigPlans": [ 8 | { 9 | "key": "retention.ms", 10 | "value": "60000", 11 | "previousValue": null, 12 | "action": "ADD" 13 | } 14 | ] 15 | }, 16 | { 17 | "name": "topic-with-configs-1", 18 | "action": "UPDATE", 19 | "topicDetailsPlan": null, 20 | "topicConfigPlans": [ 21 | { 22 | "key": "cleanup.policy", 23 | "value": null, 24 | "previousValue": "compact", 25 | "action": "REMOVE" 26 | }, 27 | { 28 | "key": "segment.bytes", 29 | "value": null, 30 | "previousValue": "100000", 31 | "action": "REMOVE" 32 | }, 33 | { 34 | "key": "retention.ms", 35 | "value": "100000", 36 | "previousValue": null, 37 | "action": "ADD" 38 | } 39 | ] 40 | } 41 | ], 42 | "aclPlans": [ 43 | { 44 | "name": "test-service-1", 45 | "aclDetails": { 46 | "name": "test-service", 47 | "type": "GROUP", 48 | "pattern": "LITERAL", 49 | "principal": "User:test", 50 | "host": "*", 51 | "operation": "READ", 52 | "permission": "ALLOW" 53 | }, 54 | "action": "ADD" 55 | } 56 | ] 57 | } -------------------------------------------------------------------------------- /src/test/resources/plans/seed-acl-exists.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | test-topic: 3 | partitions: 1 4 | replication: 2 5 | configs: 6 | retention.ms: 60000 7 | 8 | topic-with-configs-1: 9 | partitions: 3 10 | replication: 2 11 | configs: 12 | retention.ms: 100000 13 | 14 | services: 15 | test-service: 16 | principal: "User:test" 17 | type: application 18 | consumes: 19 | - test-topic -------------------------------------------------------------------------------- /src/test/resources/plans/seed-basic-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "topic-with-configs-1", 5 | "action": "UPDATE", 6 | "topicDetailsPlan": null, 7 | "topicConfigPlans": [ 8 | { 9 | "key": "segment.bytes", 10 | "value": null, 11 | "previousValue": "100000", 12 | "action": "REMOVE" 13 | }, 14 | { 15 | "key": "retention.ms", 16 | "value": "400000", 17 | "previousValue": null, 18 | "action": "ADD" 19 | } 20 | ] 21 | }, 22 | { 23 | "name": "new-topic", 24 | "action": "ADD", 25 | "topicDetailsPlan": { 26 | "partitions": 3, 27 | "previousPartitions": null, 28 | "partitionsAction": "ADD", 29 | "replication": 2, 30 | "previousReplication": null, 31 | "replicationAction": "ADD" 32 | }, 33 | "topicConfigPlans": [] 34 | }, 35 | { 36 | "name": "test-topic", 37 | "action": "REMOVE", 38 | "topicDetailsPlan": null, 39 | "topicConfigPlans": [] 40 | } 41 | ], 42 | "aclPlans": [ 43 | { 44 | "name": "test-service-1", 45 | "aclDetails": { 46 | "name": "test-service", 47 | "type": "GROUP", 48 | "pattern": "LITERAL", 49 | "principal": "User:test", 50 | "host": "*", 51 | "operation": "READ", 52 | "permission": "ALLOW" 53 | }, 54 | "action": "ADD" 55 | }, 56 | { 57 | "name": "new-service-0", 58 | "aclDetails": { 59 | "name": "delete-topic", 60 | "type": "TOPIC", 61 | "pattern": "LITERAL", 62 | "principal": "User:new", 63 | "host": "*", 64 | "operation": "READ", 65 | "permission": "ALLOW" 66 | }, 67 | "action": "ADD" 68 | }, 69 | { 70 | "name": "new-service-1", 71 | "aclDetails": { 72 | "name": "new-service", 73 | "type": "GROUP", 74 | "pattern": "LITERAL", 75 | "principal": "User:new", 76 | "host": "*", 77 | "operation": "READ", 78 | "permission": "ALLOW" 79 | }, 80 | "action": "ADD" 81 | } 82 | ] 83 | } -------------------------------------------------------------------------------- /src/test/resources/plans/seed-basic.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | delete-topic: 3 | partitions: 1 4 | replication: 2 5 | 6 | topic-with-configs-1: 7 | partitions: 3 8 | replication: 2 9 | configs: 10 | cleanup.policy: compact 11 | retention.ms: 400000 12 | 13 | topic-with-configs-2: 14 | partitions: 6 15 | replication: 2 16 | configs: 17 | retention.ms: 60000 18 | 19 | new-topic: 20 | partitions: 3 21 | replication: 2 22 | 23 | services: 24 | test-service: 25 | principal: "User:test" 26 | type: application 27 | consumes: 28 | - test-topic 29 | 30 | new-service: 31 | principal: "User:new" 32 | type: application 33 | consumes: 34 | - delete-topic 35 | -------------------------------------------------------------------------------- /src/test/resources/plans/seed-blacklist-topics-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "new-topic", 5 | "action": "ADD", 6 | "topicDetailsPlan": { 7 | "partitions": 6, 8 | "previousPartitions": null, 9 | "partitionsAction": "ADD", 10 | "replication": 2, 11 | "previousReplication": null, 12 | "replicationAction": "ADD" 13 | }, 14 | "topicConfigPlans": [] 15 | }, 16 | { 17 | "name": "delete-topic", 18 | "action": "REMOVE", 19 | "topicDetailsPlan": null, 20 | "topicConfigPlans": [] 21 | }, 22 | { 23 | "name": "topic-with-configs-2", 24 | "action": "REMOVE", 25 | "topicDetailsPlan": null, 26 | "topicConfigPlans": [] 27 | }, 28 | { 29 | "name": "topic-with-configs-1", 30 | "action": "REMOVE", 31 | "topicDetailsPlan": null, 32 | "topicConfigPlans": [] 33 | } 34 | ], 35 | "aclPlans": [ 36 | { 37 | "name": "Unnamed ACL", 38 | "aclDetails": { 39 | "name": "test-topic", 40 | "type": "TOPIC", 41 | "pattern": "LITERAL", 42 | "principal": "User:test", 43 | "host": "*", 44 | "operation": "READ", 45 | "permission": "ALLOW" 46 | }, 47 | "action": "REMOVE" 48 | } 49 | ] 50 | } -------------------------------------------------------------------------------- /src/test/resources/plans/seed-blacklist-topics.yaml: -------------------------------------------------------------------------------- 1 | settings: 2 | topics: 3 | blacklist: 4 | prefixed: 5 | - test 6 | 7 | topics: 8 | new-topic: 9 | partitions: 6 10 | replication: 2 -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-add-partitions-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [UPDATE] 4 | 5 | ~ [TOPIC] topic-with-configs-1 6 | ~ partitions: 4 (3) 7 | 8 | 9 | Successfully applied. 10 | 11 | Applying: [UPDATE] 12 | 13 | ~ [TOPIC] topic-with-configs-2 14 | ~ partitions: 10 (6) 15 | 16 | 17 | Successfully applied. 18 | 19 | Applying: [DELETE] 20 | 21 | - [ACL] Unnamed ACL 22 | - resource_name: test-topic 23 | - resource_type: TOPIC 24 | - resource_pattern: LITERAL 25 | - resource_principal: User:test 26 | - host: * 27 | - operation: READ 28 | - permission: ALLOW 29 | 30 | 31 | Successfully applied. 32 | 33 | [SUCCESS] Apply complete! Resources: 0 created, 2 updated, 1 deleted. 34 | -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-add-partitions-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "topic-with-configs-1", 5 | "action": "UPDATE", 6 | "topicDetailsPlan": { 7 | "partitions": 4, 8 | "previousPartitions": 3, 9 | "partitionsAction": "UPDATE", 10 | "replication": null, 11 | "previousReplication": null, 12 | "replicationAction": "NO_CHANGE" 13 | }, 14 | "topicConfigPlans": [] 15 | }, 16 | { 17 | "name": "topic-with-configs-2", 18 | "action": "UPDATE", 19 | "topicDetailsPlan": { 20 | "partitions": 10, 21 | "previousPartitions": 6, 22 | "partitionsAction": "UPDATE", 23 | "replication": null, 24 | "previousReplication": null, 25 | "replicationAction": "NO_CHANGE" 26 | }, 27 | "topicConfigPlans": [] 28 | } 29 | ], 30 | "aclPlans": [ 31 | { 32 | "name": "Unnamed ACL", 33 | "aclDetails": { 34 | "name": "test-topic", 35 | "type": "TOPIC", 36 | "pattern": "LITERAL", 37 | "principal": "User:test", 38 | "host": "*", 39 | "operation": "READ", 40 | "permission": "ALLOW" 41 | }, 42 | "action": "REMOVE" 43 | } 44 | ] 45 | } -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-add-partitions.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | delete-topic: 3 | partitions: 1 4 | replication: 2 5 | 6 | test-topic: 7 | partitions: 1 8 | replication: 2 9 | 10 | topic-with-configs-1: 11 | partitions: 4 12 | replication: 2 13 | configs: 14 | cleanup.policy: compact 15 | segment.bytes: 100000 16 | 17 | topic-with-configs-2: 18 | partitions: 10 19 | replication: 2 20 | configs: 21 | retention.ms: 60000 22 | -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-add-replicas-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [UPDATE] 4 | 5 | ~ [TOPIC] topic-with-configs-1 6 | ~ replication: 3 (2) 7 | 8 | 9 | Successfully applied. 10 | 11 | Applying: [UPDATE] 12 | 13 | ~ [TOPIC] topic-with-configs-2 14 | ~ replication: 3 (2) 15 | 16 | 17 | Successfully applied. 18 | 19 | Applying: [DELETE] 20 | 21 | - [ACL] Unnamed ACL 22 | - resource_name: test-topic 23 | - resource_type: TOPIC 24 | - resource_pattern: LITERAL 25 | - resource_principal: User:test 26 | - host: * 27 | - operation: READ 28 | - permission: ALLOW 29 | 30 | 31 | Successfully applied. 32 | 33 | [SUCCESS] Apply complete! Resources: 0 created, 2 updated, 1 deleted. 34 | -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-add-replicas-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "topic-with-configs-1", 5 | "action": "UPDATE", 6 | "topicDetailsPlan": { 7 | "partitions": null, 8 | "previousPartitions": null, 9 | "partitionsAction": "NO_CHANGE", 10 | "replication": 3, 11 | "previousReplication": 2, 12 | "replicationAction": "UPDATE" 13 | }, 14 | "topicConfigPlans": [] 15 | }, 16 | { 17 | "name": "topic-with-configs-2", 18 | "action": "UPDATE", 19 | "topicDetailsPlan": { 20 | "partitions": null, 21 | "previousPartitions": null, 22 | "partitionsAction": "NO_CHANGE", 23 | "replication": 3, 24 | "previousReplication": 2, 25 | "replicationAction": "UPDATE" 26 | }, 27 | "topicConfigPlans": [] 28 | } 29 | ], 30 | "aclPlans": [ 31 | { 32 | "name": "Unnamed ACL", 33 | "aclDetails": { 34 | "name": "test-topic", 35 | "type": "TOPIC", 36 | "pattern": "LITERAL", 37 | "principal": "User:test", 38 | "host": "*", 39 | "operation": "READ", 40 | "permission": "ALLOW" 41 | }, 42 | "action": "REMOVE" 43 | } 44 | ] 45 | } -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-add-replicas.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "test-topic", 5 | "action": "UPDATE", 6 | "topicDetailsPlan": null, 7 | "topicConfigPlans": [ 8 | { 9 | "key": "retention.ms", 10 | "value": "60000", 11 | "previousValue": null, 12 | "action": "ADD" 13 | } 14 | ] 15 | }, 16 | { 17 | "name": "new-topic", 18 | "action": "ADD", 19 | "topicDetailsPlan": { 20 | "partitions": 6, 21 | "previousPartitions": null, 22 | "partitionsAction": "ADD", 23 | "replication": 2, 24 | "previousReplication": null, 25 | "replicationAction": "ADD" 26 | }, 27 | "topicConfigPlans": [] 28 | }, 29 | { 30 | "name": "topic-with-configs-1", 31 | "action": "UPDATE", 32 | "topicDetailsPlan": null, 33 | "topicConfigPlans": [ 34 | { 35 | "key": "segment.bytes", 36 | "value": null, 37 | "previousValue": null, 38 | "action": "REMOVE" 39 | }, 40 | { 41 | "key": "retention.ms", 42 | "value": "100000", 43 | "previousValue": null, 44 | "action": "ADD" 45 | } 46 | ] 47 | }, 48 | { 49 | "name": "topic-with-configs-2", 50 | "action": "UPDATE", 51 | "topicDetailsPlan": null, 52 | "topicConfigPlans": [ 53 | { 54 | "key": "retention.ms", 55 | "value": "100000", 56 | "previousValue": "60000", 57 | "action": "UPDATE" 58 | } 59 | ] 60 | }, 61 | { 62 | "name": "delete-topic", 63 | "action": "REMOVE", 64 | "topicDetailsPlan": null, 65 | "topicConfigPlans": [] 66 | } 67 | ], 68 | "aclPlans": [ 69 | { 70 | "name": "Unnamed ACL", 71 | "aclDetails": { 72 | "name": "test-topic", 73 | "type": "TOPIC", 74 | "pattern": "LITERAL", 75 | "principal": "User:test", 76 | "host": "*", 77 | "operation": "READ", 78 | "permission": "ALLOW" 79 | }, 80 | "action": "REMOVE" 81 | } 82 | ] 83 | } -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-add-replicas.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | delete-topic: 3 | partitions: 1 4 | replication: 2 5 | 6 | test-topic: 7 | partitions: 1 8 | replication: 2 9 | 10 | topic-with-configs-1: 11 | partitions: 3 12 | replication: 3 13 | configs: 14 | cleanup.policy: compact 15 | segment.bytes: 100000 16 | 17 | topic-with-configs-2: 18 | partitions: 6 19 | replication: 3 20 | configs: 21 | retention.ms: 60000 22 | -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-modification-2-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "test-topic", 5 | "action": "UPDATE", 6 | "topicDetailsPlan": null, 7 | "topicConfigPlans": [ 8 | { 9 | "key": "retention.ms", 10 | "value": "60000", 11 | "previousValue": null, 12 | "action": "ADD" 13 | } 14 | ] 15 | }, 16 | { 17 | "name": "new-topic", 18 | "action": "ADD", 19 | "topicDetailsPlan": { 20 | "partitions": 6, 21 | "previousPartitions": null, 22 | "partitionsAction": "ADD", 23 | "replication": 2, 24 | "previousReplication": null, 25 | "replicationAction": "ADD" 26 | }, 27 | "topicConfigPlans": [] 28 | }, 29 | { 30 | "name": "topic-with-configs-1", 31 | "action": "UPDATE", 32 | "topicDetailsPlan": null, 33 | "topicConfigPlans": [ 34 | { 35 | "key": "cleanup.policy", 36 | "value": null, 37 | "previousValue": "compact", 38 | "action": "REMOVE" 39 | }, 40 | { 41 | "key": "segment.bytes", 42 | "value": null, 43 | "previousValue": "100000", 44 | "action": "REMOVE" 45 | }, 46 | { 47 | "key": "retention.ms", 48 | "value": "100000", 49 | "previousValue": null, 50 | "action": "ADD" 51 | } 52 | ] 53 | }, 54 | { 55 | "name": "delete-topic", 56 | "action": "REMOVE", 57 | "topicDetailsPlan": null, 58 | "topicConfigPlans": [] 59 | }, 60 | { 61 | "name": "topic-with-configs-2", 62 | "action": "REMOVE", 63 | "topicDetailsPlan": null, 64 | "topicConfigPlans": [] 65 | } 66 | ], 67 | "aclPlans": [ 68 | { 69 | "name": "Unnamed ACL", 70 | "aclDetails": { 71 | "name": "test-topic", 72 | "type": "TOPIC", 73 | "pattern": "LITERAL", 74 | "principal": "User:test", 75 | "host": "*", 76 | "operation": "READ", 77 | "permission": "ALLOW" 78 | }, 79 | "action": "REMOVE" 80 | } 81 | ] 82 | } -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-modification-2.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | test-topic: 3 | partitions: 1 4 | replication: 2 5 | configs: 6 | retention.ms: 60000 7 | 8 | new-topic: 9 | partitions: 6 10 | replication: 2 11 | 12 | topic-with-configs-1: 13 | partitions: 3 14 | replication: 2 15 | configs: 16 | retention.ms: 100000 17 | -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-modification-3-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [UPDATE] 4 | 5 | ~ [TOPIC] test-topic 6 | ~ configs: 7 | + retention.ms: 60000 8 | 9 | 10 | Successfully applied. 11 | 12 | Applying: [CREATE] 13 | 14 | + [TOPIC] new-topic 15 | + partitions: 6 16 | + replication: 2 17 | 18 | 19 | Successfully applied. 20 | 21 | Applying: [UPDATE] 22 | 23 | ~ [TOPIC] topic-with-configs-1 24 | ~ configs: 25 | - segment.bytes (100000) 26 | + retention.ms: 100000 27 | 28 | 29 | Successfully applied. 30 | 31 | Applying: [UPDATE] 32 | 33 | ~ [TOPIC] topic-with-configs-2 34 | ~ configs: 35 | ~ retention.ms: 100000 ( 60000 ) 36 | 37 | 38 | Successfully applied. 39 | 40 | Applying: [DELETE] 41 | 42 | - [TOPIC] delete-topic 43 | 44 | 45 | Successfully applied. 46 | 47 | Applying: [DELETE] 48 | 49 | - [ACL] Unnamed ACL 50 | - resource_name: test-topic 51 | - resource_type: TOPIC 52 | - resource_pattern: LITERAL 53 | - resource_principal: User:test 54 | - host: * 55 | - operation: READ 56 | - permission: ALLOW 57 | 58 | 59 | Successfully applied. 60 | 61 | [SUCCESS] Apply complete! Resources: 1 created, 3 updated, 2 deleted. 62 | -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-modification-3-no-delete-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [UPDATE] 4 | 5 | ~ [TOPIC] test-topic 6 | ~ configs: 7 | + retention.ms: 60000 8 | 9 | 10 | Successfully applied. 11 | 12 | Applying: [CREATE] 13 | 14 | + [TOPIC] new-topic 15 | + partitions: 6 16 | + replication: 2 17 | 18 | 19 | Successfully applied. 20 | 21 | Applying: [UPDATE] 22 | 23 | ~ [TOPIC] topic-with-configs-1 24 | ~ configs: 25 | - segment.bytes (100000) 26 | + retention.ms: 100000 27 | 28 | 29 | Successfully applied. 30 | 31 | Applying: [UPDATE] 32 | 33 | ~ [TOPIC] topic-with-configs-2 34 | ~ configs: 35 | ~ retention.ms: 100000 ( 60000 ) 36 | 37 | 38 | Successfully applied. 39 | 40 | [SUCCESS] Apply complete! Resources: 1 created, 3 updated, 0 deleted. 41 | -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-modification-3-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "test-topic", 5 | "action": "UPDATE", 6 | "topicDetailsPlan": null, 7 | "topicConfigPlans": [ 8 | { 9 | "key": "retention.ms", 10 | "value": "60000", 11 | "previousValue": null, 12 | "action": "ADD" 13 | } 14 | ] 15 | }, 16 | { 17 | "name": "new-topic", 18 | "action": "ADD", 19 | "topicDetailsPlan": { 20 | "partitions": 6, 21 | "previousPartitions": null, 22 | "partitionsAction": "ADD", 23 | "replication": 2, 24 | "previousReplication": null, 25 | "replicationAction": "ADD" 26 | }, 27 | "topicConfigPlans": [] 28 | }, 29 | { 30 | "name": "topic-with-configs-1", 31 | "action": "UPDATE", 32 | "topicDetailsPlan": null, 33 | "topicConfigPlans": [ 34 | { 35 | "key": "segment.bytes", 36 | "value": null, 37 | "previousValue": "100000", 38 | "action": "REMOVE" 39 | }, 40 | { 41 | "key": "retention.ms", 42 | "value": "100000", 43 | "previousValue": null, 44 | "action": "ADD" 45 | } 46 | ] 47 | }, 48 | { 49 | "name": "topic-with-configs-2", 50 | "action": "UPDATE", 51 | "topicDetailsPlan": null, 52 | "topicConfigPlans": [ 53 | { 54 | "key": "retention.ms", 55 | "value": "100000", 56 | "previousValue": "60000", 57 | "action": "UPDATE" 58 | } 59 | ] 60 | }, 61 | { 62 | "name": "delete-topic", 63 | "action": "REMOVE", 64 | "topicDetailsPlan": null, 65 | "topicConfigPlans": [] 66 | } 67 | ], 68 | "aclPlans": [ 69 | { 70 | "name": "Unnamed ACL", 71 | "aclDetails": { 72 | "name": "test-topic", 73 | "type": "TOPIC", 74 | "pattern": "LITERAL", 75 | "principal": "User:test", 76 | "host": "*", 77 | "operation": "READ", 78 | "permission": "ALLOW" 79 | }, 80 | "action": "REMOVE" 81 | } 82 | ] 83 | } -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-modification-3.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | test-topic: 3 | partitions: 1 4 | replication: 2 5 | configs: 6 | retention.ms: 60000 7 | 8 | new-topic: 9 | partitions: 6 10 | replication: 2 11 | 12 | topic-with-configs-1: 13 | partitions: 3 14 | replication: 2 15 | configs: 16 | retention.ms: 100000 17 | cleanup.policy: compact 18 | 19 | topic-with-configs-2: 20 | partitions: 6 21 | replication: 2 22 | configs: 23 | retention.ms: 100000 24 | -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-modification-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [UPDATE] 4 | 5 | ~ [TOPIC] test-topic 6 | ~ configs: 7 | + retention.ms: 60000 8 | 9 | 10 | Successfully applied. 11 | 12 | Applying: [CREATE] 13 | 14 | + [TOPIC] new-topic 15 | + partitions: 6 16 | + replication: 2 17 | 18 | 19 | Successfully applied. 20 | 21 | Applying: [DELETE] 22 | 23 | - [TOPIC] delete-topic 24 | 25 | 26 | Successfully applied. 27 | 28 | Applying: [DELETE] 29 | 30 | - [TOPIC] topic-with-configs-2 31 | 32 | 33 | Successfully applied. 34 | 35 | Applying: [DELETE] 36 | 37 | - [TOPIC] topic-with-configs-1 38 | 39 | 40 | Successfully applied. 41 | 42 | Applying: [DELETE] 43 | 44 | - [ACL] Unnamed ACL 45 | - resource_name: test-topic 46 | - resource_type: TOPIC 47 | - resource_pattern: LITERAL 48 | - resource_principal: User:test 49 | - host: * 50 | - operation: READ 51 | - permission: ALLOW 52 | 53 | 54 | Successfully applied. 55 | 56 | [SUCCESS] Apply complete! Resources: 1 created, 1 updated, 4 deleted. 57 | -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-modification-no-delete-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [UPDATE] 4 | 5 | ~ [TOPIC] test-topic 6 | ~ configs: 7 | + retention.ms: 60000 8 | 9 | 10 | Successfully applied. 11 | 12 | Applying: [CREATE] 13 | 14 | + [TOPIC] new-topic 15 | + partitions: 6 16 | + replication: 2 17 | 18 | 19 | Successfully applied. 20 | 21 | [SUCCESS] Apply complete! Resources: 1 created, 1 updated, 0 deleted. 22 | -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-modification-no-delete-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "test-topic", 5 | "action": "UPDATE", 6 | "topicDetailsPlan": null, 7 | "topicConfigPlans": [ 8 | { 9 | "key": "retention.ms", 10 | "value": "60000", 11 | "previousValue": null, 12 | "action": "ADD" 13 | } 14 | ] 15 | }, 16 | { 17 | "name": "new-topic", 18 | "action": "ADD", 19 | "topicDetailsPlan": { 20 | "partitions": 6, 21 | "previousPartitions": null, 22 | "partitionsAction": "ADD", 23 | "replication": 2, 24 | "previousReplication": null, 25 | "replicationAction": "ADD" 26 | }, 27 | "topicConfigPlans": [] 28 | } 29 | ], 30 | "aclPlans": [] 31 | } -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-modification-no-delete.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | test-topic: 3 | partitions: 1 4 | replication: 2 5 | configs: 6 | retention.ms: 60000 7 | 8 | new-topic: 9 | partitions: 6 10 | replication: 2 -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-modification-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "test-topic", 5 | "action": "UPDATE", 6 | "topicDetailsPlan": null, 7 | "topicConfigPlans": [ 8 | { 9 | "key": "retention.ms", 10 | "value": "60000", 11 | "previousValue": null, 12 | "action": "ADD" 13 | } 14 | ] 15 | }, 16 | { 17 | "name": "new-topic", 18 | "action": "ADD", 19 | "topicDetailsPlan": { 20 | "partitions": 6, 21 | "previousPartitions": null, 22 | "partitionsAction": "ADD", 23 | "replication": 2, 24 | "previousReplication": null, 25 | "replicationAction": "ADD" 26 | }, 27 | "topicConfigPlans": [] 28 | }, 29 | { 30 | "name": "delete-topic", 31 | "action": "REMOVE", 32 | "topicDetailsPlan": null, 33 | "topicConfigPlans": [] 34 | }, 35 | { 36 | "name": "topic-with-configs-2", 37 | "action": "REMOVE", 38 | "topicDetailsPlan": null, 39 | "topicConfigPlans": [] 40 | }, 41 | { 42 | "name": "topic-with-configs-1", 43 | "action": "REMOVE", 44 | "topicDetailsPlan": null, 45 | "topicConfigPlans": [] 46 | } 47 | ], 48 | "aclPlans": [ 49 | { 50 | "name": "Unnamed ACL", 51 | "aclDetails": { 52 | "name": "test-topic", 53 | "type": "TOPIC", 54 | "pattern": "LITERAL", 55 | "principal": "User:test", 56 | "host": "*", 57 | "operation": "READ", 58 | "permission": "ALLOW" 59 | }, 60 | "action": "REMOVE" 61 | } 62 | ] 63 | } -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-modification.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | test-topic: 3 | partitions: 1 4 | replication: 2 5 | configs: 6 | retention.ms: 60000 7 | 8 | new-topic: 9 | partitions: 6 10 | replication: 2 -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-remove-replicas-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "topic-with-configs-1", 5 | "action": "UPDATE", 6 | "topicDetailsPlan": { 7 | "partitions": null, 8 | "previousPartitions": null, 9 | "partitionsAction": "NO_CHANGE", 10 | "replication": 1, 11 | "previousReplication": 2, 12 | "replicationAction": "UPDATE" 13 | }, 14 | "topicConfigPlans": [] 15 | }, 16 | { 17 | "name": "topic-with-configs-2", 18 | "action": "UPDATE", 19 | "topicDetailsPlan": { 20 | "partitions": null, 21 | "previousPartitions": null, 22 | "partitionsAction": "NO_CHANGE", 23 | "replication": 1, 24 | "previousReplication": 2, 25 | "replicationAction": "UPDATE" 26 | }, 27 | "topicConfigPlans": [] 28 | } 29 | ], 30 | "aclPlans": [ 31 | { 32 | "name": "Unnamed ACL", 33 | "aclDetails": { 34 | "name": "test-topic", 35 | "type": "TOPIC", 36 | "pattern": "LITERAL", 37 | "principal": "User:test", 38 | "host": "*", 39 | "operation": "READ", 40 | "permission": "ALLOW" 41 | }, 42 | "action": "REMOVE" 43 | } 44 | ] 45 | } -------------------------------------------------------------------------------- /src/test/resources/plans/seed-topic-remove-replicas.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | delete-topic: 3 | partitions: 1 4 | replication: 2 5 | 6 | test-topic: 7 | partitions: 1 8 | replication: 2 9 | 10 | topic-with-configs-1: 11 | partitions: 3 12 | replication: 1 13 | configs: 14 | cleanup.policy: compact 15 | segment.bytes: 100000 16 | 17 | topic-with-configs-2: 18 | partitions: 6 19 | replication: 1 20 | configs: 21 | retention.ms: 60000 22 | -------------------------------------------------------------------------------- /src/test/resources/plans/simple-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [CREATE] 4 | 5 | + [TOPIC] test-topic 6 | + partitions: 6 7 | + replication: 2 8 | 9 | 10 | Successfully applied. 11 | 12 | [SUCCESS] Apply complete! Resources: 1 created, 0 updated, 0 deleted. 13 | -------------------------------------------------------------------------------- /src/test/resources/plans/simple-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "test-topic", 5 | "action": "ADD", 6 | "topicDetailsPlan": { 7 | "partitions": 6, 8 | "previousPartitions": null, 9 | "partitionsAction": "ADD", 10 | "replication": 2, 11 | "previousReplication": null, 12 | "replicationAction": "ADD" 13 | }, 14 | "topicConfigPlans": [] 15 | } 16 | ], 17 | "aclPlans": [] 18 | } -------------------------------------------------------------------------------- /src/test/resources/plans/simple-users-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [CREATE] 4 | 5 | + [TOPIC] test-topic 6 | + partitions: 6 7 | + replication: 2 8 | 9 | 10 | Successfully applied. 11 | 12 | Applying: [CREATE] 13 | 14 | + [ACL] test-user-0 15 | + resource_name: kafka-cluster 16 | + resource_type: CLUSTER 17 | + resource_pattern: LITERAL 18 | + resource_principal: User:test 19 | + host: * 20 | + operation: DESCRIBE 21 | + permission: ALLOW 22 | 23 | 24 | Successfully applied. 25 | 26 | Applying: [CREATE] 27 | 28 | + [ACL] test-user-1 29 | + resource_name: * 30 | + resource_type: TOPIC 31 | + resource_pattern: LITERAL 32 | + resource_principal: User:test 33 | + host: * 34 | + operation: DESCRIBE 35 | + permission: ALLOW 36 | 37 | 38 | Successfully applied. 39 | 40 | Applying: [CREATE] 41 | 42 | + [ACL] test-user-2 43 | + resource_name: * 44 | + resource_type: TOPIC 45 | + resource_pattern: LITERAL 46 | + resource_principal: User:test 47 | + host: * 48 | + operation: DESCRIBE_CONFIGS 49 | + permission: ALLOW 50 | 51 | 52 | Successfully applied. 53 | 54 | Applying: [CREATE] 55 | 56 | + [ACL] test-user-3 57 | + resource_name: * 58 | + resource_type: GROUP 59 | + resource_pattern: LITERAL 60 | + resource_principal: User:test 61 | + host: * 62 | + operation: READ 63 | + permission: ALLOW 64 | 65 | 66 | Successfully applied. 67 | 68 | Applying: [CREATE] 69 | 70 | + [ACL] test-user-4 71 | + resource_name: * 72 | + resource_type: GROUP 73 | + resource_pattern: LITERAL 74 | + resource_principal: User:test 75 | + host: * 76 | + operation: DESCRIBE 77 | + permission: ALLOW 78 | 79 | 80 | Successfully applied. 81 | 82 | Applying: [CREATE] 83 | 84 | + [ACL] test-user-5 85 | + resource_name: * 86 | + resource_type: TOPIC 87 | + resource_pattern: LITERAL 88 | + resource_principal: User:test 89 | + host: * 90 | + operation: READ 91 | + permission: ALLOW 92 | 93 | 94 | Successfully applied. 95 | 96 | Applying: [CREATE] 97 | 98 | + [ACL] test-user-6 99 | + resource_name: * 100 | + resource_type: GROUP 101 | + resource_pattern: LITERAL 102 | + resource_principal: User:test 103 | + host: * 104 | + operation: READ 105 | + permission: ALLOW 106 | 107 | 108 | Successfully applied. 109 | 110 | Applying: [CREATE] 111 | 112 | + [ACL] test-user-7 113 | + resource_name: * 114 | + resource_type: TOPIC 115 | + resource_pattern: LITERAL 116 | + resource_principal: User:test 117 | + host: * 118 | + operation: WRITE 119 | + permission: ALLOW 120 | 121 | 122 | Successfully applied. 123 | 124 | [SUCCESS] Apply complete! Resources: 9 created, 0 updated, 0 deleted. 125 | -------------------------------------------------------------------------------- /src/test/resources/plans/simple-users-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "test-topic", 5 | "action": "ADD", 6 | "topicDetailsPlan": { 7 | "partitions": 6, 8 | "previousPartitions": null, 9 | "partitionsAction": "ADD", 10 | "replication": 2, 11 | "previousReplication": null, 12 | "replicationAction": "ADD" 13 | }, 14 | "topicConfigPlans": [] 15 | } 16 | ], 17 | "aclPlans": [ 18 | { 19 | "name": "test-user-0", 20 | "aclDetails": { 21 | "name": "kafka-cluster", 22 | "type": "CLUSTER", 23 | "pattern": "LITERAL", 24 | "principal": "User:test", 25 | "host": "*", 26 | "operation": "DESCRIBE", 27 | "permission": "ALLOW" 28 | }, 29 | "action": "ADD" 30 | }, 31 | { 32 | "name": "test-user-1", 33 | "aclDetails": { 34 | "name": "*", 35 | "type": "TOPIC", 36 | "pattern": "LITERAL", 37 | "principal": "User:test", 38 | "host": "*", 39 | "operation": "DESCRIBE", 40 | "permission": "ALLOW" 41 | }, 42 | "action": "ADD" 43 | }, 44 | { 45 | "name": "test-user-2", 46 | "aclDetails": { 47 | "name": "*", 48 | "type": "TOPIC", 49 | "pattern": "LITERAL", 50 | "principal": "User:test", 51 | "host": "*", 52 | "operation": "DESCRIBE_CONFIGS", 53 | "permission": "ALLOW" 54 | }, 55 | "action": "ADD" 56 | }, 57 | { 58 | "name": "test-user-3", 59 | "aclDetails": { 60 | "name": "*", 61 | "type": "GROUP", 62 | "pattern": "LITERAL", 63 | "principal": "User:test", 64 | "host": "*", 65 | "operation": "READ", 66 | "permission": "ALLOW" 67 | }, 68 | "action": "ADD" 69 | }, 70 | { 71 | "name": "test-user-4", 72 | "aclDetails": { 73 | "name": "*", 74 | "type": "GROUP", 75 | "pattern": "LITERAL", 76 | "principal": "User:test", 77 | "host": "*", 78 | "operation": "DESCRIBE", 79 | "permission": "ALLOW" 80 | }, 81 | "action": "ADD" 82 | }, 83 | { 84 | "name": "test-user-5", 85 | "aclDetails": { 86 | "name": "*", 87 | "type": "TOPIC", 88 | "pattern": "LITERAL", 89 | "principal": "User:test", 90 | "host": "*", 91 | "operation": "READ", 92 | "permission": "ALLOW" 93 | }, 94 | "action": "ADD" 95 | }, 96 | { 97 | "name": "test-user-6", 98 | "aclDetails": { 99 | "name": "*", 100 | "type": "GROUP", 101 | "pattern": "LITERAL", 102 | "principal": "User:test", 103 | "host": "*", 104 | "operation": "READ", 105 | "permission": "ALLOW" 106 | }, 107 | "action": "ADD" 108 | }, 109 | { 110 | "name": "test-user-7", 111 | "aclDetails": { 112 | "name": "*", 113 | "type": "TOPIC", 114 | "pattern": "LITERAL", 115 | "principal": "User:test", 116 | "host": "*", 117 | "operation": "WRITE", 118 | "permission": "ALLOW" 119 | }, 120 | "action": "ADD" 121 | } 122 | ] 123 | } -------------------------------------------------------------------------------- /src/test/resources/plans/simple-users.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | test-topic: 3 | partitions: 6 4 | replication: 2 5 | 6 | users: 7 | test-user: 8 | principal: User:test 9 | roles: 10 | - operator 11 | - reader 12 | - writer -------------------------------------------------------------------------------- /src/test/resources/plans/simple.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | test-topic: 3 | partitions: 6 4 | replication: 2 -------------------------------------------------------------------------------- /src/test/resources/plans/skip-acls-apply-apply-output.txt: -------------------------------------------------------------------------------- 1 | Executing apply... 2 | 3 | Applying: [CREATE] 4 | 5 | + [TOPIC] MY_TOPIC 6 | + partitions: 6 7 | + replication: 1 8 | 9 | 10 | Successfully applied. 11 | 12 | Applying: [CREATE] 13 | 14 | + [TOPIC] another.topic.0 15 | + partitions: 1 16 | + replication: 1 17 | + configs: 18 | + cleanup.policy: compact 19 | + segment.bytes: 100000 20 | 21 | 22 | Successfully applied. 23 | 24 | [SUCCESS] Apply complete! Resources: 2 created, 0 updated, 0 deleted. 25 | -------------------------------------------------------------------------------- /src/test/resources/plans/skip-acls-apply-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "MY_TOPIC", 5 | "action": "ADD", 6 | "topicDetailsPlan": { 7 | "partitions": 6, 8 | "previousPartitions": null, 9 | "partitionsAction": "ADD", 10 | "replication": 1, 11 | "previousReplication": null, 12 | "replicationAction": "ADD" 13 | }, 14 | "topicConfigPlans": [] 15 | }, 16 | { 17 | "name": "another.topic.0", 18 | "action": "ADD", 19 | "topicDetailsPlan": { 20 | "partitions": 1, 21 | "previousPartitions": null, 22 | "partitionsAction": "ADD", 23 | "replication": 1, 24 | "previousReplication": null, 25 | "replicationAction": "ADD" 26 | }, 27 | "topicConfigPlans": [ 28 | { 29 | "key": "cleanup.policy", 30 | "value": "compact", 31 | "previousValue": null, 32 | "action": "ADD" 33 | }, 34 | { 35 | "key": "segment.bytes", 36 | "value": "100000", 37 | "previousValue": null, 38 | "action": "ADD" 39 | } 40 | ] 41 | } 42 | ], 43 | "aclPlans": [ 44 | { 45 | "name": "test-service-0", 46 | "aclDetails": { 47 | "name": "another.topic.0", 48 | "type": "TOPIC", 49 | "pattern": "LITERAL", 50 | "principal": "User:test", 51 | "host": "*", 52 | "operation": "WRITE", 53 | "permission": "ALLOW" 54 | }, 55 | "action": "ADD" 56 | }, 57 | { 58 | "name": "test-service-1", 59 | "aclDetails": { 60 | "name": "MY_TOPIC", 61 | "type": "TOPIC", 62 | "pattern": "LITERAL", 63 | "principal": "User:test", 64 | "host": "*", 65 | "operation": "READ", 66 | "permission": "ALLOW" 67 | }, 68 | "action": "ADD" 69 | }, 70 | { 71 | "name": "test-service-2", 72 | "aclDetails": { 73 | "name": "test-service", 74 | "type": "GROUP", 75 | "pattern": "LITERAL", 76 | "principal": "User:test", 77 | "host": "*", 78 | "operation": "READ", 79 | "permission": "ALLOW" 80 | }, 81 | "action": "ADD" 82 | }, 83 | { 84 | "name": "my-other-service-0", 85 | "aclDetails": { 86 | "name": "another.topic.0", 87 | "type": "TOPIC", 88 | "pattern": "LITERAL", 89 | "principal": "User:test", 90 | "host": "*", 91 | "operation": "READ", 92 | "permission": "ALLOW" 93 | }, 94 | "action": "ADD" 95 | }, 96 | { 97 | "name": "my-other-service-1", 98 | "aclDetails": { 99 | "name": "my-other-service", 100 | "type": "GROUP", 101 | "pattern": "LITERAL", 102 | "principal": "User:test", 103 | "host": "*", 104 | "operation": "READ", 105 | "permission": "ALLOW" 106 | }, 107 | "action": "ADD" 108 | } 109 | ] 110 | } 111 | -------------------------------------------------------------------------------- /src/test/resources/plans/skip-acls-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "MY_TOPIC", 5 | "action": "ADD", 6 | "topicDetailsPlan": { 7 | "partitions": 6, 8 | "previousPartitions": null, 9 | "partitionsAction": "ADD", 10 | "replication": 1, 11 | "previousReplication": null, 12 | "replicationAction": "ADD" 13 | }, 14 | "topicConfigPlans": [] 15 | }, 16 | { 17 | "name": "another.topic.0", 18 | "action": "ADD", 19 | "topicDetailsPlan": { 20 | "partitions": 1, 21 | "previousPartitions": null, 22 | "partitionsAction": "ADD", 23 | "replication": 1, 24 | "previousReplication": null, 25 | "replicationAction": "ADD" 26 | }, 27 | "topicConfigPlans": [ 28 | { 29 | "key": "cleanup.policy", 30 | "value": "compact", 31 | "previousValue": null, 32 | "action": "ADD" 33 | }, 34 | { 35 | "key": "segment.bytes", 36 | "value": "100000", 37 | "previousValue": null, 38 | "action": "ADD" 39 | } 40 | ] 41 | } 42 | ], 43 | "aclPlans": [] 44 | } 45 | -------------------------------------------------------------------------------- /src/test/resources/plans/skip-acls.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | MY_TOPIC: 3 | partitions: 6 4 | replication: 1 5 | 6 | another.topic.0: 7 | partitions: 1 8 | replication: 1 9 | configs: 10 | cleanup.policy: compact 11 | segment.bytes: 100000 12 | 13 | services: 14 | test-service: 15 | type: application 16 | principal: User:test 17 | produces: 18 | - another.topic.0 19 | consumes: 20 | - MY_TOPIC 21 | 22 | my-other-service: 23 | type: application 24 | principal: User:test 25 | consumes: 26 | - another.topic.0 -------------------------------------------------------------------------------- /src/test/resources/plans/topics-and-services-plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "topicPlans": [ 3 | { 4 | "name": "MY_TOPIC", 5 | "action": "ADD", 6 | "topicDetailsPlan": { 7 | "partitions": 6, 8 | "previousPartitions": null, 9 | "partitionsAction": "ADD", 10 | "replication": 2, 11 | "previousReplication": null, 12 | "replicationAction": "ADD" 13 | }, 14 | "topicConfigPlans": [] 15 | }, 16 | { 17 | "name": "another.topic.0", 18 | "action": "ADD", 19 | "topicDetailsPlan": { 20 | "partitions": 1, 21 | "previousPartitions": null, 22 | "partitionsAction": "ADD", 23 | "replication": 2, 24 | "previousReplication": null, 25 | "replicationAction": "ADD" 26 | }, 27 | "topicConfigPlans": [ 28 | { 29 | "key": "cleanup.policy", 30 | "value": "compact", 31 | "previousValue": null, 32 | "action": "ADD" 33 | }, 34 | { 35 | "key": "segment.bytes", 36 | "value": "100000", 37 | "previousValue": null, 38 | "action": "ADD" 39 | } 40 | ] 41 | } 42 | ], 43 | "aclPlans": [ 44 | { 45 | "name": "test-service-0", 46 | "aclDetails": { 47 | "name": "another.topic.0", 48 | "type": "TOPIC", 49 | "pattern": "LITERAL", 50 | "principal": "User:test", 51 | "host": "*", 52 | "operation": "WRITE", 53 | "permission": "ALLOW" 54 | }, 55 | "action": "ADD" 56 | }, 57 | { 58 | "name": "test-service-1", 59 | "aclDetails": { 60 | "name": "MY_TOPIC", 61 | "type": "TOPIC", 62 | "pattern": "LITERAL", 63 | "principal": "User:test", 64 | "host": "*", 65 | "operation": "READ", 66 | "permission": "ALLOW" 67 | }, 68 | "action": "ADD" 69 | }, 70 | { 71 | "name": "test-service-2", 72 | "aclDetails": { 73 | "name": "test-service", 74 | "type": "GROUP", 75 | "pattern": "LITERAL", 76 | "principal": "User:test", 77 | "host": "*", 78 | "operation": "READ", 79 | "permission": "ALLOW" 80 | }, 81 | "action": "ADD" 82 | }, 83 | { 84 | "name": "my-other-service-0", 85 | "aclDetails": { 86 | "name": "another.topic.0", 87 | "type": "TOPIC", 88 | "pattern": "LITERAL", 89 | "principal": "User:test", 90 | "host": "*", 91 | "operation": "READ", 92 | "permission": "ALLOW" 93 | }, 94 | "action": "ADD" 95 | }, 96 | { 97 | "name": "my-other-service-1", 98 | "aclDetails": { 99 | "name": "my-other-service", 100 | "type": "GROUP", 101 | "pattern": "LITERAL", 102 | "principal": "User:test", 103 | "host": "*", 104 | "operation": "READ", 105 | "permission": "ALLOW" 106 | }, 107 | "action": "ADD" 108 | } 109 | ] 110 | } -------------------------------------------------------------------------------- /src/test/resources/plans/topics-and-services.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | MY_TOPIC: 3 | partitions: 6 4 | replication: 2 5 | 6 | another.topic.0: 7 | partitions: 1 8 | replication: 2 9 | configs: 10 | cleanup.policy: compact 11 | segment.bytes: 100000 12 | 13 | services: 14 | test-service: 15 | type: application 16 | principal: User:test 17 | produces: 18 | - another.topic.0 19 | consumes: 20 | - MY_TOPIC 21 | 22 | my-other-service: 23 | type: application 24 | principal: User:test 25 | consumes: 26 | - another.topic.0 -------------------------------------------------------------------------------- /src/test/resources/plans/unrecognized-property-output.txt: -------------------------------------------------------------------------------- 1 | Generating execution plan... 2 | 3 | [INVALID] Unrecognized field: [test] in state file definition: topics -> test-topic 4 | -------------------------------------------------------------------------------- /src/test/resources/plans/unrecognized-property.yaml: -------------------------------------------------------------------------------- 1 | topics: 2 | test-topic: 3 | test: invalid 4 | partitions: 1 5 | replication: 2 6 | -------------------------------------------------------------------------------- /stub.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | MYSELF=`which "$0" 2>/dev/null` 3 | [ $? -gt 0 -a -f "$0" ] && MYSELF="./$0" 4 | java=java 5 | if test -n "$JAVA_HOME"; then 6 | java="$JAVA_HOME/bin/java" 7 | fi 8 | exec "$java" -jar $MYSELF "$@" 9 | exit 1 --------------------------------------------------------------------------------