├── .github └── CODEOWNERS ├── .gitignore ├── Readme.md ├── acls-gateway-security ├── Readme.md ├── aclCluster-admin.properties ├── aclCluster-consumer.properties ├── aclCluster-producer.properties ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-06-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-07-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-08-ADD_INTERCEPTOR.txt │ ├── step-09-CREATE_TOPICS.txt │ ├── step-10-CREATE_TOPICS.txt │ ├── step-11-LIST_TOPICS.txt │ ├── step-12-SH.txt │ ├── step-13-CONSUME.txt │ ├── step-14-SH.txt │ ├── step-15-LIST_TOPICS.txt │ ├── step-16-SH.txt │ ├── step-17-LIST_TOPICS.txt │ ├── step-18-PRODUCE.txt │ ├── step-19-CONSUME.txt │ ├── step-20-PRODUCE.txt │ └── step-21-DOCKER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_VIRTUAL_CLUSTER.sh ├── step-07-CREATE_VIRTUAL_CLUSTER.sh ├── step-08-ADD_INTERCEPTOR.sh ├── step-08-acl.json ├── step-09-CREATE_TOPICS.sh ├── step-10-CREATE_TOPICS.sh ├── step-11-LIST_TOPICS.sh ├── step-12-SH.sh ├── step-13-CONSUME.sh ├── step-14-SH.sh ├── step-15-LIST_TOPICS.sh ├── step-16-SH.sh ├── step-17-LIST_TOPICS.sh ├── step-18-PRODUCE.sh ├── step-19-CONSUME.sh ├── step-20-PRODUCE.sh ├── step-21-DOCKER.sh └── utils.sh ├── acls-vcluster ├── Readme.md ├── aclCluster-admin.properties ├── aclCluster-consumer.properties ├── aclCluster-producer.properties ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-06-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-07-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-08-ADD_INTERCEPTOR.txt │ ├── step-09-CREATE_TOPICS.txt │ ├── step-10-CREATE_TOPICS.txt │ ├── step-11-LIST_TOPICS.txt │ ├── step-12-SH.txt │ ├── step-13-CONSUME.txt │ ├── step-14-SH.txt │ ├── step-15-LIST_TOPICS.txt │ ├── step-16-SH.txt │ ├── step-17-LIST_TOPICS.txt │ ├── step-18-PRODUCE.txt │ ├── step-19-CONSUME.txt │ ├── step-20-PRODUCE.txt │ └── step-21-DOCKER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_VIRTUAL_CLUSTER.sh ├── step-07-CREATE_VIRTUAL_CLUSTER.sh ├── step-08-ADD_INTERCEPTOR.sh ├── step-08-acl.json ├── step-09-CREATE_TOPICS.sh ├── step-10-CREATE_TOPICS.sh ├── step-11-LIST_TOPICS.sh ├── step-12-SH.sh ├── step-13-CONSUME.sh ├── step-14-SH.sh ├── step-15-LIST_TOPICS.sh ├── step-16-SH.sh ├── step-17-LIST_TOPICS.sh ├── step-18-PRODUCE.sh ├── step-19-CONSUME.sh ├── step-20-PRODUCE.sh ├── step-21-DOCKER.sh └── utils.sh ├── audit ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ └── step-05-CREATE_VIRTUAL_CLUSTER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-ADD_INTERCEPTOR.sh ├── step-06-guard-on-produce.json ├── step-07-LIST_INTERCEPTORS.sh ├── step-08-CREATE_TOPICS.sh ├── step-09-PRODUCE.sh ├── step-10-AUDITLOG.sh ├── step-11-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── chaos-duplicate-messages ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ └── step-05-CREATE_VIRTUAL_CLUSTER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-ADD_INTERCEPTOR.sh ├── step-07-duplicate-messages.json ├── step-08-LIST_INTERCEPTORS.sh ├── step-09-PRODUCE.sh ├── step-10-CONSUME.sh ├── step-11-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── chaos-simulate-broken-broker ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ └── step-05-CREATE_VIRTUAL_CLUSTER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-ADD_INTERCEPTOR.sh ├── step-07-simulate-broken-brokers.json ├── step-08-LIST_INTERCEPTORS.sh ├── step-09-SH.sh ├── step-10-REMOVE_INTERCEPTORS.sh ├── step-11-LIST_INTERCEPTORS.sh ├── step-12-SH.sh ├── step-13-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── chaos-simulate-invalid-schema-id ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ └── step-10-SH.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-ADD_INTERCEPTOR.sh ├── step-07-simulate-invalid-schema-id.json ├── step-08-LIST_INTERCEPTORS.sh ├── step-09-SH.sh ├── step-10-SH.sh ├── step-11-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── chaos-simulate-leader-election-errors ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ └── step-05-CREATE_VIRTUAL_CLUSTER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-ADD_INTERCEPTOR.sh ├── step-07-simulate-leader-elections-errors.json ├── step-08-LIST_INTERCEPTORS.sh ├── step-09-SH.sh ├── step-10-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── chaos-simulate-message-corruption ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ └── step-05-CREATE_VIRTUAL_CLUSTER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-ADD_INTERCEPTOR.sh ├── step-07-simulate-massage-corruption.json ├── step-08-LIST_INTERCEPTORS.sh ├── step-09-PRODUCE.sh ├── step-10-CONSUME.sh ├── step-11-DOCKER.sh ├── teamA-sa.properties ├── temp.txt └── utils.sh ├── chaos-simulate-slow-broker ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ └── step-05-CREATE_VIRTUAL_CLUSTER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-ADD_INTERCEPTOR.sh ├── step-07-simulate-slow-broker.json ├── step-08-LIST_INTERCEPTORS.sh ├── step-09-SH.sh ├── step-10-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── chaos-simulate-slow-producers-consumers ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ └── step-05-CREATE_VIRTUAL_CLUSTER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-ADD_INTERCEPTOR.sh ├── step-07-simulate-slow-producer-consumers.json ├── step-08-LIST_INTERCEPTORS.sh ├── step-09-SH.sh ├── step-10-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── cluster-switching ├── Readme.md ├── clusters.yaml ├── docker-compose.yaml ├── mm2.properties ├── output │ ├── step-07-DOCKER.txt │ ├── step-08-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-13-CONSUME.txt │ ├── step-16-FAILOVER.txt │ ├── step-19-CONSUME.txt │ └── step-20-CONSUME.txt ├── run.sh ├── scenario.yaml ├── step-07-DOCKER.sh ├── step-08-CREATE_VIRTUAL_CLUSTER.sh ├── step-09-CREATE_TOPICS.sh ├── step-10-PRODUCE.sh ├── step-11-LIST_TOPICS.sh ├── step-12-CONSUME.sh ├── step-13-CONSUME.sh ├── step-14-LIST_TOPICS.sh ├── step-15-FAILOVER.sh ├── step-16-FAILOVER.sh ├── step-17-PRODUCE.sh ├── step-18-CONSUME.sh ├── step-19-CONSUME.sh ├── step-20-CONSUME.sh ├── step-21-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── data-masking ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ └── step-05-CREATE_VIRTUAL_CLUSTER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-ADD_INTERCEPTOR.sh ├── step-07-data-masking.json ├── step-08-LIST_INTERCEPTORS.sh ├── step-09-PRODUCE.sh ├── step-10-CONSUME.sh ├── step-11-REMOVE_INTERCEPTORS.sh ├── step-12-CONSUME.sh ├── step-13-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── encryption-crypto-shredding ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-06-CREATE_TOPICS.txt │ ├── step-07-LIST_TOPICS.txt │ ├── step-08-ADD_INTERCEPTOR.txt │ ├── step-09-LIST_INTERCEPTORS.txt │ ├── step-10-PRODUCE.txt │ ├── step-11-CONSUME.txt │ ├── step-12-ADD_INTERCEPTOR.txt │ ├── step-13-LIST_INTERCEPTORS.txt │ ├── step-14-CONSUME.txt │ ├── step-15-SH.txt │ ├── step-16-SH.txt │ ├── step-17-CONSUME.txt │ └── step-18-DOCKER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-LIST_TOPICS.sh ├── step-08-ADD_INTERCEPTOR.sh ├── step-08-crypto-shredding-encrypt.json ├── step-09-LIST_INTERCEPTORS.sh ├── step-10-PRODUCE.sh ├── step-11-CONSUME.sh ├── step-12-ADD_INTERCEPTOR.sh ├── step-12-crypto-shredding-decrypt.json ├── step-13-LIST_INTERCEPTORS.sh ├── step-14-CONSUME.sh ├── step-15-SH.sh ├── step-16-SH.sh ├── step-17-CONSUME.sh ├── step-18-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── encryption-decrypt-only-specific-fields ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ └── step-05-CREATE_VIRTUAL_CLUSTER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-ADD_INTERCEPTOR.sh ├── step-07-encrypt.json ├── step-08-LIST_INTERCEPTORS.sh ├── step-09-PRODUCE.sh ├── step-10-CONSUME.sh ├── step-11-ADD_INTERCEPTOR.sh ├── step-11-decrypt.json ├── step-12-LIST_INTERCEPTORS.sh ├── step-13-CONSUME.sh ├── step-14-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── encryption-external-storage ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ └── step-15-CONSUME.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-ADD_INTERCEPTOR.sh ├── step-07-encrypt.json ├── step-08-LIST_INTERCEPTORS.sh ├── step-09-CONSUME.sh ├── step-10-PRODUCE.sh ├── step-11-CONSUME.sh ├── step-12-ADD_INTERCEPTOR.sh ├── step-12-decrypt.json ├── step-13-LIST_INTERCEPTORS.sh ├── step-14-CONSUME.sh ├── step-15-CONSUME.sh ├── step-16-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── encryption-payload-external-storage ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ └── step-15-CONSUME.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-ADD_INTERCEPTOR.sh ├── step-07-encrypt.json ├── step-08-LIST_INTERCEPTORS.sh ├── step-09-CONSUME.sh ├── step-10-PRODUCE.sh ├── step-11-CONSUME.sh ├── step-12-ADD_INTERCEPTOR.sh ├── step-12-decrypt.json ├── step-13-LIST_INTERCEPTORS.sh ├── step-14-CONSUME.sh ├── step-15-CONSUME.sh ├── step-16-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── encryption-payload ├── Readme.md ├── docker-compose.yaml ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-ADD_INTERCEPTOR.sh ├── step-07-encrypt-full-payload.json ├── step-08-LIST_INTERCEPTORS.sh ├── step-09-PRODUCE.sh ├── step-10-CONSUME.sh ├── step-11-ADD_INTERCEPTOR.sh ├── step-11-decrypt-full-payload.json ├── step-12-LIST_INTERCEPTORS.sh ├── step-13-CONSUME.sh ├── step-14-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── encryption-performance ├── Readme.md ├── docker-compose.yaml ├── examples.json ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ └── step-08-ADD_INTERCEPTOR.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-ADD_INTERCEPTOR.sh ├── step-07-encrypt.json ├── step-08-ADD_INTERCEPTOR.sh ├── step-08-decrypt.json ├── step-09-LIST_INTERCEPTORS.sh ├── step-10-SH.sh ├── step-11-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── encryption-third-party ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ └── step-12-CONSUME.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_VIRTUAL_CLUSTER.sh ├── step-07-CREATE_TOPICS.sh ├── step-08-ADD_INTERCEPTOR.sh ├── step-08-encrypt-on-consume.json ├── step-09-LIST_INTERCEPTORS.sh ├── step-10-PRODUCE.sh ├── step-11-CONSUME.sh ├── step-12-CONSUME.sh ├── step-13-DOCKER.sh ├── teamA-sa.properties ├── teamA-third-party.properties └── utils.sh ├── encryption-vault-with-secret-management ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-06-CREATE_TOPICS.txt │ ├── step-07-LIST_TOPICS.txt │ ├── step-08-ADD_INTERCEPTOR.txt │ ├── step-09-LIST_INTERCEPTORS.txt │ ├── step-10-PRODUCE.txt │ ├── step-11-CONSUME.txt │ ├── step-12-ADD_INTERCEPTOR.txt │ ├── step-13-LIST_INTERCEPTORS.txt │ ├── step-14-CONSUME.txt │ └── step-15-DOCKER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-LIST_TOPICS.sh ├── step-08-ADD_INTERCEPTOR.sh ├── step-08-crypto-shredding-encrypt.json ├── step-09-LIST_INTERCEPTORS.sh ├── step-10-PRODUCE.sh ├── step-11-CONSUME.sh ├── step-12-ADD_INTERCEPTOR.sh ├── step-12-crypto-shredding-decrypt.json ├── step-13-LIST_INTERCEPTORS.sh ├── step-14-CONSUME.sh ├── step-15-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── header-injection ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ └── step-05-CREATE_VIRTUAL_CLUSTER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-ADD_INTERCEPTOR.sh ├── step-07-inject-headers.json ├── step-08-PRODUCE.sh ├── step-09-CONSUME.sh ├── step-10-ADD_INTERCEPTOR.sh ├── step-10-remove-headers.json ├── step-11-CONSUME.sh ├── step-12-REMOVE_INTERCEPTORS.sh ├── step-13-CONSUME.sh ├── step-14-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── ksqlDB ├── Readme.md ├── docker-compose.yaml ├── ksql.sql ├── output │ ├── step-04-DOCKER.txt │ └── step-05-CREATE_VIRTUAL_CLUSTER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-CREATE_TOPICS.sh ├── step-08-CREATE_CONCENTRATION_RULE.sh ├── step-08-concentration-rule.json ├── step-09-SH.sh ├── step-10-LIST_TOPICS.sh ├── step-11-LIST_TOPICS.sh ├── step-13-SH.sh ├── step-14-LIST_TOPICS.sh ├── step-15-LIST_TOPICS.sh ├── step-16-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── large-messages ├── Readme.md ├── credentials ├── docker-compose.yaml ├── from-kafka.bin ├── large-message.bin ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-07-SH.txt │ ├── step-08-CREATE_TOPICS.txt │ ├── step-09-ADD_INTERCEPTOR.txt │ ├── step-10-SH.txt │ ├── step-11-SH.txt │ ├── step-12-SH.txt │ ├── step-13-SH.txt │ ├── step-14-SH.txt │ ├── step-15-CONSUME.txt │ └── step-16-DOCKER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-07-SH.sh ├── step-08-CREATE_TOPICS.sh ├── step-09-ADD_INTERCEPTOR.sh ├── step-09-large-messages.json ├── step-10-SH.sh ├── step-11-SH.sh ├── step-12-SH.sh ├── step-13-SH.sh ├── step-14-SH.sh ├── step-15-CONSUME.sh ├── step-16-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── latency ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_TOPICS.txt │ ├── step-06-SH.txt │ └── step-07-CREATE_VIRTUAL_CLUSTER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_TOPICS.sh ├── step-06-SH.sh ├── step-07-CREATE_VIRTUAL_CLUSTER.sh ├── step-08-CREATE_TOPICS.sh ├── step-09-SH.sh ├── step-10-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── merge-cluster ├── Readme.md ├── clusters.yaml ├── docker-compose.yaml ├── output │ ├── step-05-DOCKER.txt │ ├── step-06-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-08-CREATE_TOPICS.txt │ ├── step-10-SH.txt │ ├── step-12-PRODUCE.txt │ ├── step-14-CONSUME.txt │ ├── step-15-CONSUME.txt │ └── step-16-CONSUME.txt ├── run.sh ├── scenario.yaml ├── step-05-DOCKER.sh ├── step-06-CREATE_VIRTUAL_CLUSTER.sh ├── step-07-CREATE_TOPICS.sh ├── step-08-CREATE_TOPICS.sh ├── step-09-SH.sh ├── step-10-SH.sh ├── step-11-PRODUCE.sh ├── step-12-PRODUCE.sh ├── step-13-CONSUME.sh ├── step-14-CONSUME.sh ├── step-15-CONSUME.sh ├── step-16-CONSUME.sh ├── step-17-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── move-to-docs.sh ├── multi-tenancy ├── Readme.md ├── docker-compose.yaml ├── london-sa.properties ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-LIST_TOPICS.txt │ ├── step-06-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-07-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-08-CREATE_TOPICS.txt │ ├── step-09-CREATE_TOPICS.txt │ ├── step-10-LIST_TOPICS.txt │ ├── step-11-LIST_TOPICS.txt │ ├── step-12-PRODUCE.txt │ ├── step-13-CONSUME.txt │ ├── step-14-PRODUCE.txt │ ├── step-15-CONSUME.txt │ ├── step-16-CREATE_TOPICS.txt │ ├── step-17-PRODUCE.txt │ ├── step-18-SH.txt │ ├── step-19-LIST_TOPICS.txt │ ├── step-20-CREATE_TOPICS.txt │ ├── step-21-PRODUCE.txt │ ├── step-22-SH.txt │ ├── step-23-LIST_TOPICS.txt │ ├── step-24-CONSUME.txt │ ├── step-25-CONSUME.txt │ ├── step-26-SH.txt │ ├── step-27-LIST_TOPICS.txt │ ├── step-28-CONSUME.txt │ └── step-29-DOCKER.txt ├── paris-sa.properties ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-LIST_TOPICS.sh ├── step-06-CREATE_VIRTUAL_CLUSTER.sh ├── step-07-CREATE_VIRTUAL_CLUSTER.sh ├── step-08-CREATE_TOPICS.sh ├── step-09-CREATE_TOPICS.sh ├── step-10-LIST_TOPICS.sh ├── step-11-LIST_TOPICS.sh ├── step-12-PRODUCE.sh ├── step-13-CONSUME.sh ├── step-14-PRODUCE.sh ├── step-15-CONSUME.sh ├── step-16-CREATE_TOPICS.sh ├── step-17-PRODUCE.sh ├── step-18-SH.sh ├── step-19-LIST_TOPICS.sh ├── step-20-CREATE_TOPICS.sh ├── step-21-PRODUCE.sh ├── step-22-SH.sh ├── step-23-LIST_TOPICS.sh ├── step-24-CONSUME.sh ├── step-25-CONSUME.sh ├── step-26-SH.sh ├── step-27-LIST_TOPICS.sh ├── step-28-CONSUME.sh ├── step-29-DOCKER.sh └── utils.sh ├── oauth ├── Readme.md ├── conduktor-realm.json ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ ├── step-06-CREATE_TOPICS.txt │ ├── step-07-LIST_TOPICS.txt │ └── step-08-DOCKER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-LIST_TOPICS.sh ├── step-08-DOCKER.sh ├── user-1.properties └── utils.sh ├── safeguard-client-id ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-06-CREATE_TOPICS.txt │ ├── step-07-LIST_TOPICS.txt │ ├── step-08-ADD_INTERCEPTOR.txt │ ├── step-09-LIST_INTERCEPTORS.txt │ ├── step-10-CREATE_TOPICS.txt │ ├── step-11-SH.txt │ ├── step-12-CREATE_TOPICS.txt │ ├── step-13-AUDITLOG.txt │ └── step-14-DOCKER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-LIST_TOPICS.sh ├── step-08-ADD_INTERCEPTOR.sh ├── step-08-client-id.json ├── step-09-LIST_INTERCEPTORS.sh ├── step-10-CREATE_TOPICS.sh ├── step-11-SH.sh ├── step-12-CREATE_TOPICS.sh ├── step-13-AUDITLOG.sh ├── step-14-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── safeguard-schema-id ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-06-CREATE_TOPICS.txt │ ├── step-07-LIST_TOPICS.txt │ ├── step-08-ADD_INTERCEPTOR.txt │ ├── step-09-LIST_INTERCEPTORS.txt │ ├── step-10-PRODUCE.txt │ ├── step-11-CONSUME.txt │ ├── step-12-SH.txt │ ├── step-13-SH.txt │ ├── step-14-CONSUME.txt │ └── step-15-DOCKER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-LIST_TOPICS.sh ├── step-08-ADD_INTERCEPTOR.sh ├── step-08-schema-id.json ├── step-09-LIST_INTERCEPTORS.sh ├── step-10-PRODUCE.sh ├── step-11-CONSUME.sh ├── step-12-SH.sh ├── step-13-SH.sh ├── step-14-CONSUME.sh ├── step-15-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── safeguard-validate-schema-payload-avro ├── Readme.md ├── docker-compose.yaml ├── invalid-payload.json ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-11-SH.txt │ ├── step-16-SH.txt │ ├── step-17-SH.txt │ └── step-20-SH.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-08-SH.sh ├── step-10-SH.sh ├── step-11-SH.sh ├── step-12-ADD_INTERCEPTOR.sh ├── step-12-guard-schema-payload-validate.json ├── step-13-LIST_INTERCEPTORS.sh ├── step-15-SH.sh ├── step-16-SH.sh ├── step-17-SH.sh ├── step-18-AUDITLOG.sh ├── step-19-SH.sh ├── step-20-SH.sh ├── step-21-DOCKER.sh ├── teamA-sa.properties ├── user-schema-with-validation-rules.avsc ├── user-schema.avsc ├── utils.sh └── valid-payload.json ├── safeguard-validate-schema-payload-json ├── Readme.md ├── docker-compose.yaml ├── invalid-payload.json ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-11-SH.txt │ ├── step-12-SH.txt │ └── step-18-SH.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-08-SH.sh ├── step-10-SH.sh ├── step-11-SH.sh ├── step-12-SH.sh ├── step-13-ADD_INTERCEPTOR.sh ├── step-13-guard-schema-payload-validate.json ├── step-14-LIST_INTERCEPTORS.sh ├── step-15-SH.sh ├── step-16-AUDITLOG.sh ├── step-17-SH.sh ├── step-18-SH.sh ├── step-19-DOCKER.sh ├── teamA-sa.properties ├── user-schema-with-validation-rules.json ├── utils.sh └── valid-payload.json ├── safeguard-validate-schema-payload-proto ├── Readme.md ├── docker-compose.yaml ├── invalid-payload.json ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-11-SH.txt │ ├── step-16-SH.txt │ ├── step-17-SH.txt │ └── step-20-SH.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-08-SH.sh ├── step-10-SH.sh ├── step-11-SH.sh ├── step-12-ADD_INTERCEPTOR.sh ├── step-12-guard-schema-payload-validate.json ├── step-13-LIST_INTERCEPTORS.sh ├── step-15-SH.sh ├── step-16-SH.sh ├── step-17-SH.sh ├── step-18-AUDITLOG.sh ├── step-19-SH.sh ├── step-20-SH.sh ├── step-21-DOCKER.sh ├── teamA-sa.properties ├── user-schema-with-validation-rules.proto ├── user-schema.proto ├── utils.sh └── valid-payload.json ├── safeguard-validate-schema-payload ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-13-SH.txt │ ├── step-14-SH.txt │ └── step-15-SH.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-ADD_INTERCEPTOR.sh ├── step-07-guard-schema-payload-validate.json ├── step-08-LIST_INTERCEPTORS.sh ├── step-12-SH.sh ├── step-13-SH.sh ├── step-14-SH.sh ├── step-15-SH.sh ├── step-16-AUDITLOG.sh ├── step-17-SH.sh ├── step-18-AUDITLOG.sh ├── step-19-DOCKER.sh ├── teamA-sa.properties ├── user-schema.avsc ├── user-schema.json ├── user-schema.proto └── utils.sh ├── safeguard ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-13-CREATE_TOPICS.txt │ ├── step-16-ALTER_TOPICS.txt │ └── step-19-PRODUCE.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-PRODUCE.sh ├── step-08-CONSUME.sh ├── step-09-DESCRIBE_TOPICS.sh ├── step-10-ADD_INTERCEPTOR.sh ├── step-10-guard-on-create-topic.json ├── step-11-LIST_INTERCEPTORS.sh ├── step-12-CREATE_TOPICS.sh ├── step-13-CREATE_TOPICS.sh ├── step-14-ADD_INTERCEPTOR.sh ├── step-14-guard-on-alter-topic.json ├── step-15-ALTER_TOPICS.sh ├── step-16-ALTER_TOPICS.sh ├── step-17-ADD_INTERCEPTOR.sh ├── step-17-guard-on-produce.json ├── step-18-PRODUCE.sh ├── step-19-PRODUCE.sh ├── step-20-ADD_INTERCEPTOR.sh ├── step-20-produce-rate.json ├── step-21-PRODUCE.sh ├── step-22-AUDITLOG.sh ├── step-23-REMOVE_INTERCEPTORS.sh ├── step-24-ADD_INTERCEPTOR.sh ├── step-24-consumer-group-name-policy.json ├── step-25-CONSUME.sh ├── step-26-AUDITLOG.sh ├── step-27-CONSUME.sh ├── step-28-REMOVE_INTERCEPTORS.sh ├── step-29-ADD_INTERCEPTOR.sh ├── step-29-guard-limit-connection.json ├── step-30-CONSUME.sh ├── step-31-AUDITLOG.sh ├── step-32-REMOVE_INTERCEPTORS.sh ├── step-33-ADD_INTERCEPTOR.sh ├── step-33-guard-agressive-auto-commit.json ├── step-34-CONSUME.sh ├── step-35-AUDITLOG.sh ├── step-36-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── sni-routing ├── Readme.md ├── client.config ├── docker-compose.yaml ├── keystore.jks ├── openssl.config ├── run.sh ├── san.crt ├── san.key ├── san.p12 ├── scenario.yaml ├── step-04-SH.sh ├── step-05-DOCKER.sh ├── step-06-SH.sh ├── step-07-SH.sh ├── step-08-SH.sh ├── step-09-SH.sh ├── step-10-DOCKER.sh ├── truststore.jks └── utils.sh ├── sql-data-quality-producer ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-09-PRODUCE.txt │ └── step-10-PRODUCE.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-ADD_INTERCEPTOR.sh ├── step-07-cars-quality.json ├── step-08-PRODUCE.sh ├── step-09-PRODUCE.sh ├── step-10-PRODUCE.sh ├── step-11-CONSUME.sh ├── step-12-CONSUME.sh ├── step-13-AUDITLOG.sh ├── step-14-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── sql-topic-schema-registry ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_VIRTUAL_CLUSTER.txt │ ├── step-06-CREATE_TOPICS.txt │ ├── step-07-LIST_TOPICS.txt │ ├── step-08-SH.txt │ ├── step-09-SH.txt │ ├── step-10-CREATE_TOPICS.txt │ ├── step-11-ADD_INTERCEPTOR.txt │ ├── step-12-LIST_INTERCEPTORS.txt │ ├── step-13-SH.txt │ └── step-14-DOCKER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-LIST_TOPICS.sh ├── step-08-SH.sh ├── step-09-SH.sh ├── step-10-CREATE_TOPICS.sh ├── step-11-ADD_INTERCEPTOR.sh ├── step-11-red-cars.json ├── step-12-LIST_INTERCEPTORS.sh ├── step-13-SH.sh ├── step-14-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── sql-topics-in-gateway-security-mode ├── Readme.md ├── docker-compose.yaml ├── passthrough-sa.properties ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-PRODUCE.sh ├── step-08-CONSUME.sh ├── step-09-ADD_INTERCEPTOR.sh ├── step-09-red-cars.json ├── step-10-LIST_INTERCEPTORS.sh ├── step-11-CONSUME.sh ├── step-13-DOCKER.sh └── utils.sh ├── sql-topics-in-vcluster-mode ├── Readme.md ├── docker-compose.yaml ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-PRODUCE.sh ├── step-08-CONSUME.sh ├── step-09-ADD_INTERCEPTOR.sh ├── step-09-red-cars.json ├── step-10-LIST_INTERCEPTORS.sh ├── step-11-CONSUME.sh ├── step-13-DOCKER.sh ├── teamA-sa.properties └── utils.sh ├── sql-topics ├── Readme.md ├── docker-compose.yaml ├── output │ └── step-04-DOCKER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_TOPICS.sh ├── step-06-PRODUCE.sh ├── step-07-CONSUME.sh ├── step-08-ADD_INTERCEPTOR.sh ├── step-08-red-cars.json ├── step-09-LIST_INTERCEPTORS.sh ├── step-10-CONSUME.sh ├── step-12-DOCKER.sh └── utils.sh ├── ssl-and-user-mapping ├── Readme.md ├── client.config ├── docker-compose.yaml ├── keystore.jks ├── openssl.config ├── output │ ├── step-04-SH.txt │ ├── step-05-DOCKER.txt │ ├── step-06-ADD_USER_MAPPING.txt │ ├── step-07-CREATE_TOPICS.txt │ ├── step-08-LIST_TOPICS.txt │ └── step-09-LIST_TOPICS.txt ├── run.sh ├── san.crt ├── san.key ├── san.p12 ├── scenario.yaml ├── step-04-SH.sh ├── step-05-DOCKER.sh ├── step-06-ADD_USER_MAPPING.sh ├── step-06-user-mapping.json ├── step-07-CREATE_TOPICS.sh ├── step-08-LIST_TOPICS.sh ├── step-09-LIST_TOPICS.sh ├── step-10-DOCKER.sh ├── truststore.jks └── utils.sh ├── ssl ├── Readme.md ├── client.config ├── docker-compose.yaml ├── keystore.jks ├── openssl.config ├── output │ ├── step-04-SH.txt │ ├── step-05-DOCKER.txt │ ├── step-06-CREATE_TOPICS.txt │ ├── step-07-LIST_TOPICS.txt │ └── step-08-LIST_TOPICS.txt ├── run.sh ├── san.crt ├── san.key ├── san.p12 ├── scenario.yaml ├── step-04-SH.sh ├── step-05-DOCKER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-LIST_TOPICS.sh ├── step-08-LIST_TOPICS.sh ├── step-09-DOCKER.sh ├── truststore.jks └── utils.sh ├── throughput ├── Readme.md ├── docker-compose.yaml ├── output │ ├── step-04-DOCKER.txt │ ├── step-05-CREATE_TOPICS.txt │ ├── step-06-SH.txt │ └── step-07-CREATE_VIRTUAL_CLUSTER.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_TOPICS.sh ├── step-06-SH.sh ├── step-07-CREATE_VIRTUAL_CLUSTER.sh ├── step-08-CREATE_TOPICS.sh ├── step-09-SH.sh ├── step-10-DOCKER.sh ├── teamA-sa.properties └── utils.sh └── topic-concentration ├── Readme.md ├── docker-compose.yaml ├── output ├── step-04-DOCKER.txt ├── step-05-CREATE_VIRTUAL_CLUSTER.txt ├── step-10-LIST_TOPICS.txt └── step-17-CONSUME.txt ├── run.sh ├── scenario.yaml ├── step-04-DOCKER.sh ├── step-05-CREATE_VIRTUAL_CLUSTER.sh ├── step-06-CREATE_TOPICS.sh ├── step-07-CREATE_CONCENTRATION_RULE.sh ├── step-07-concentration-rule.json ├── step-08-CREATE_TOPICS.sh ├── step-09-LIST_TOPICS.sh ├── step-10-LIST_TOPICS.sh ├── step-11-CREATE_TOPICS.sh ├── step-12-LIST_TOPICS.sh ├── step-13-PRODUCE.sh ├── step-14-CONSUME.sh ├── step-15-PRODUCE.sh ├── step-16-CONSUME.sh ├── step-17-CONSUME.sh ├── step-18-DOCKER.sh ├── teamA-sa.properties └── utils.sh /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @conduktor/conduktor-proxy 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Misc 2 | .DS_Store 3 | .env.local 4 | .env.development.local 5 | .env.test.local 6 | .env.production.local 7 | 8 | .idea 9 | .env 10 | record-*sh 11 | upload-asciinema.sh 12 | Readme-doc.md 13 | Readme-doc.md-asciinema 14 | images/ 15 | asciinema/ 16 | output/ 17 | -------------------------------------------------------------------------------- /acls-gateway-security/output/step-08-ADD_INTERCEPTOR.txt: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass": "io.conduktor.gateway.interceptor.AclsInterceptorPlugin", 3 | "priority": 100, 4 | "config": {} 5 | } 6 | { 7 | "message": "acl is created" 8 | } 9 | -------------------------------------------------------------------------------- /acls-gateway-security/output/step-09-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Error while executing topic command : Cluster not authorized 2 | [2024-04-09 23:39:25,143] ERROR org.apache.kafka.common.errors.ClusterAuthorizationException: Cluster not authorized 3 | (org.apache.kafka.tools.TopicCommand) 4 | -------------------------------------------------------------------------------- /acls-gateway-security/output/step-10-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic restricted-topic. 2 | -------------------------------------------------------------------------------- /acls-gateway-security/output/step-11-LIST_TOPICS.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /acls-gateway-security/output/step-15-LIST_TOPICS.txt: -------------------------------------------------------------------------------- 1 | restricted-topic 2 | -------------------------------------------------------------------------------- /acls-gateway-security/output/step-17-LIST_TOPICS.txt: -------------------------------------------------------------------------------- 1 | restricted-topic 2 | -------------------------------------------------------------------------------- /acls-gateway-security/output/step-18-PRODUCE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/acls-gateway-security/output/step-18-PRODUCE.txt -------------------------------------------------------------------------------- /acls-gateway-security/output/step-19-CONSUME.txt: -------------------------------------------------------------------------------- 1 | [2024-04-09 23:40:01,295] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) 2 | org.apache.kafka.common.errors.TimeoutException 3 | Processed a total of 1 messages 4 | { 5 | "msg": "test message" 6 | } 7 | -------------------------------------------------------------------------------- /acls-gateway-security/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /acls-gateway-security/step-08-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-08-acl.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/aclCluster/interceptor/acl" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-08-acl.json | jq 10 | -------------------------------------------------------------------------------- /acls-gateway-security/step-08-acl.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.AclsInterceptorPlugin", 3 | "priority" : 100, 4 | "config" : { } 5 | } -------------------------------------------------------------------------------- /acls-gateway-security/step-09-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config aclCluster-consumer.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic restricted-topic 9 | -------------------------------------------------------------------------------- /acls-gateway-security/step-10-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config aclCluster-admin.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic restricted-topic 9 | -------------------------------------------------------------------------------- /acls-gateway-security/step-11-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config aclCluster-consumer.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /acls-gateway-security/step-12-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-acls \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config aclCluster-admin.properties \ 5 | --add \ 6 | --allow-principal User:consumer \ 7 | --operation read \ 8 | --topic restricted-topic -------------------------------------------------------------------------------- /acls-gateway-security/step-13-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --topic _conduktor_gateway_acls \ 5 | --from-beginning \ 6 | --timeout-ms 10000 \ 7 | --property print.key=true | jq 8 | -------------------------------------------------------------------------------- /acls-gateway-security/step-14-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-acls \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config aclCluster-admin.properties \ 5 | --add \ 6 | --allow-principal User:consumer \ 7 | --operation read \ 8 | --group console-consumer \ 9 | --resource-pattern-type prefixed -------------------------------------------------------------------------------- /acls-gateway-security/step-15-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config aclCluster-consumer.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /acls-gateway-security/step-16-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-acls \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config aclCluster-admin.properties \ 5 | --add \ 6 | --allow-principal User:producer \ 7 | --operation write \ 8 | --topic restricted-topic -------------------------------------------------------------------------------- /acls-gateway-security/step-17-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config aclCluster-producer.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /acls-gateway-security/step-18-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"msg":"test message"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config aclCluster-producer.properties \ 6 | --topic restricted-topic -------------------------------------------------------------------------------- /acls-gateway-security/step-19-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config aclCluster-consumer.properties \ 5 | --topic restricted-topic \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /acls-gateway-security/step-20-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"msg":"I would be surprised if it would work!"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config aclCluster-consumer.properties \ 6 | --topic restricted-topic -------------------------------------------------------------------------------- /acls-gateway-security/step-21-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /acls-vcluster/aclCluster-admin.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='admin' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6ImFkbWluIiwidmNsdXN0ZXIiOiJhY2xDbHVzdGVyIiwiZXhwIjoxNzIwNDY4MzA0fQ.TWZLYq9Eu1Bi669AHfR-b7n-LPFjvkNOZXL3wimZVaY'; 6 | 7 | -------------------------------------------------------------------------------- /acls-vcluster/output/step-08-ADD_INTERCEPTOR.txt: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass": "io.conduktor.gateway.interceptor.AclsInterceptorPlugin", 3 | "priority": 100, 4 | "config": {} 5 | } 6 | { 7 | "message": "acl is created" 8 | } 9 | -------------------------------------------------------------------------------- /acls-vcluster/output/step-09-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Error while executing topic command : Cluster not authorized 2 | [2024-04-09 23:48:06,885] ERROR org.apache.kafka.common.errors.ClusterAuthorizationException: Cluster not authorized 3 | (org.apache.kafka.tools.TopicCommand) 4 | -------------------------------------------------------------------------------- /acls-vcluster/output/step-10-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic restricted-topic. 2 | -------------------------------------------------------------------------------- /acls-vcluster/output/step-11-LIST_TOPICS.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /acls-vcluster/output/step-15-LIST_TOPICS.txt: -------------------------------------------------------------------------------- 1 | restricted-topic 2 | -------------------------------------------------------------------------------- /acls-vcluster/output/step-17-LIST_TOPICS.txt: -------------------------------------------------------------------------------- 1 | restricted-topic 2 | -------------------------------------------------------------------------------- /acls-vcluster/output/step-18-PRODUCE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/acls-vcluster/output/step-18-PRODUCE.txt -------------------------------------------------------------------------------- /acls-vcluster/output/step-19-CONSUME.txt: -------------------------------------------------------------------------------- 1 | [2024-04-09 23:48:46,020] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) 2 | org.apache.kafka.common.errors.TimeoutException 3 | Processed a total of 1 messages 4 | { 5 | "msg": "test message" 6 | } 7 | -------------------------------------------------------------------------------- /acls-vcluster/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /acls-vcluster/step-08-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-08-acl.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/aclCluster/interceptor/acl" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-08-acl.json | jq 10 | -------------------------------------------------------------------------------- /acls-vcluster/step-08-acl.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.AclsInterceptorPlugin", 3 | "priority" : 100, 4 | "config" : { } 5 | } -------------------------------------------------------------------------------- /acls-vcluster/step-09-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config aclCluster-consumer.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic restricted-topic 9 | -------------------------------------------------------------------------------- /acls-vcluster/step-10-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config aclCluster-admin.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic restricted-topic 9 | -------------------------------------------------------------------------------- /acls-vcluster/step-11-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config aclCluster-consumer.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /acls-vcluster/step-12-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-acls \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config aclCluster-admin.properties \ 5 | --add \ 6 | --allow-principal User:consumer \ 7 | --operation read \ 8 | --topic restricted-topic -------------------------------------------------------------------------------- /acls-vcluster/step-13-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --topic _conduktor_gateway_acls \ 5 | --from-beginning \ 6 | --timeout-ms 10000 \ 7 | --property print.key=true | jq 8 | -------------------------------------------------------------------------------- /acls-vcluster/step-14-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-acls \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config aclCluster-admin.properties \ 5 | --add \ 6 | --allow-principal User:consumer \ 7 | --operation read \ 8 | --group console-consumer \ 9 | --resource-pattern-type prefixed -------------------------------------------------------------------------------- /acls-vcluster/step-15-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config aclCluster-consumer.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /acls-vcluster/step-16-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-acls \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config aclCluster-admin.properties \ 5 | --add \ 6 | --allow-principal User:producer \ 7 | --operation write \ 8 | --topic restricted-topic -------------------------------------------------------------------------------- /acls-vcluster/step-17-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config aclCluster-producer.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /acls-vcluster/step-18-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"msg":"test message"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config aclCluster-producer.properties \ 6 | --topic restricted-topic -------------------------------------------------------------------------------- /acls-vcluster/step-19-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config aclCluster-consumer.properties \ 5 | --topic restricted-topic \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /acls-vcluster/step-20-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"msg":"I would be surprised if it would work!"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config aclCluster-consumer.properties \ 6 | --topic restricted-topic -------------------------------------------------------------------------------- /acls-vcluster/step-21-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /audit/output/step-05-CREATE_VIRTUAL_CLUSTER.txt: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ2ODU2N30.tn7s54D-g59zrqWmMdR_Xfmbma8EfoNfBmy4gN0ug38'; 6 | 7 | -------------------------------------------------------------------------------- /audit/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /audit/step-06-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-06-guard-on-produce.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/guard-on-produce" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-06-guard-on-produce.json | jq 10 | -------------------------------------------------------------------------------- /audit/step-06-guard-on-produce.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.safeguard.ProducePolicyPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "acks" : { 6 | "value" : [ -1 ], 7 | "action" : "BLOCK" 8 | }, 9 | "compressions" : { 10 | "value" : [ "NONE", "GZIP" ], 11 | "action" : "BLOCK" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /audit/step-07-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /audit/step-08-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic cars 9 | -------------------------------------------------------------------------------- /audit/step-09-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"type":"Fiat","color":"red","price":-1}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --request-required-acks 1 \ 7 | --compression-codec snappy \ 8 | --topic cars -------------------------------------------------------------------------------- /audit/step-10-AUDITLOG.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --topic _conduktor_gateway_auditlogs \ 5 | --from-beginning \ 6 | --timeout-ms 3000 \| jq 'select(.type=="SAFEGUARD" and .eventData.plugin=="io.conduktor.gateway.interceptor.safeguard.ProducePolicyPlugin")' 7 | -------------------------------------------------------------------------------- /audit/step-11-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /audit/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ2ODcwNX0.CosDt29-MX4QRJy4avIKkA21OlJrAqajJSLNXObNt40'; 6 | 7 | -------------------------------------------------------------------------------- /chaos-duplicate-messages/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /chaos-duplicate-messages/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic topic-duplicate 9 | -------------------------------------------------------------------------------- /chaos-duplicate-messages/step-07-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-07-duplicate-messages.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/duplicate-messages" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-07-duplicate-messages.json | jq 10 | -------------------------------------------------------------------------------- /chaos-duplicate-messages/step-07-duplicate-messages.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.chaos.DuplicateMessagesPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "rateInPercent" : 100, 6 | "topic" : "topic-duplicate", 7 | "target" : "PRODUCE" 8 | } 9 | } -------------------------------------------------------------------------------- /chaos-duplicate-messages/step-08-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /chaos-duplicate-messages/step-09-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"message": "hello world"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --topic topic-duplicate -------------------------------------------------------------------------------- /chaos-duplicate-messages/step-10-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic topic-duplicate \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /chaos-duplicate-messages/step-11-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /chaos-duplicate-messages/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ2OTAxMH0.Ekh8y7T1LKzo4FSWSuun8GEhMDMzsydSbwa-4vyKJlU'; 6 | 7 | -------------------------------------------------------------------------------- /chaos-simulate-broken-broker/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /chaos-simulate-broken-broker/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic my-topic 9 | -------------------------------------------------------------------------------- /chaos-simulate-broken-broker/step-07-simulate-broken-brokers.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.chaos.SimulateBrokenBrokersPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "rateInPercent" : 100, 6 | "errorMap" : { 7 | "FETCH" : "UNKNOWN_SERVER_ERROR", 8 | "PRODUCE" : "CORRUPT_MESSAGE" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /chaos-simulate-broken-broker/step-08-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /chaos-simulate-broken-broker/step-09-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-producer-perf-test \ 3 | --producer.config teamA-sa.properties \ 4 | --record-size 10 \ 5 | --throughput 1 \ 6 | --num-records 10 \ 7 | --producer-prop retries=5 \ 8 | --topic my-topic -------------------------------------------------------------------------------- /chaos-simulate-broken-broker/step-10-REMOVE_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request DELETE "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/simulate-broken-brokers" \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /chaos-simulate-broken-broker/step-11-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /chaos-simulate-broken-broker/step-12-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-producer-perf-test \ 3 | --producer.config teamA-sa.properties \ 4 | --record-size 10 \ 5 | --throughput 1 \ 6 | --num-records 10 \ 7 | --topic my-topic -------------------------------------------------------------------------------- /chaos-simulate-broken-broker/step-13-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /chaos-simulate-broken-broker/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ2OTM2Mn0.37AJbBtEQL961VC4aYi17C9l3o915zz_HyQXgl9lvTc'; 6 | 7 | -------------------------------------------------------------------------------- /chaos-simulate-invalid-schema-id/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /chaos-simulate-invalid-schema-id/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic with-schema 9 | -------------------------------------------------------------------------------- /chaos-simulate-invalid-schema-id/step-07-simulate-invalid-schema-id.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.chaos.SimulateInvalidSchemaIdPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "topic" : "with-schema", 6 | "invalidSchemaId" : 999, 7 | "target" : "CONSUME" 8 | } 9 | } -------------------------------------------------------------------------------- /chaos-simulate-invalid-schema-id/step-08-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /chaos-simulate-invalid-schema-id/step-10-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-json-schema-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --topic with-schema \ 5 | --consumer.config teamA-sa.properties \ 6 | --from-beginning -------------------------------------------------------------------------------- /chaos-simulate-invalid-schema-id/step-11-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /chaos-simulate-invalid-schema-id/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ2OTY4MH0.QbOXMB1Z7tq0jFTa9Slo2gPN4iG7LNQA-8h5a6N94Wo'; 6 | 7 | -------------------------------------------------------------------------------- /chaos-simulate-leader-election-errors/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /chaos-simulate-leader-election-errors/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic my-topic 9 | -------------------------------------------------------------------------------- /chaos-simulate-leader-election-errors/step-07-simulate-leader-elections-errors.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.chaos.SimulateLeaderElectionsErrorsPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "rateInPercent" : 50 6 | } 7 | } -------------------------------------------------------------------------------- /chaos-simulate-leader-election-errors/step-08-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /chaos-simulate-leader-election-errors/step-09-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-producer-perf-test \ 3 | --producer-props bootstrap.servers=localhost:6969 \ 4 | --producer.config teamA-sa.properties \ 5 | --record-size 10 \ 6 | --throughput 1 \ 7 | --producer-prop retries=5 \ 8 | --num-records 10 \ 9 | --topic my-topic -------------------------------------------------------------------------------- /chaos-simulate-leader-election-errors/step-10-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /chaos-simulate-leader-election-errors/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ2OTk3Nn0.sElwYYbIx1QpKj6x6aiL7vnCpiZRCtOPylq4H8C1T4o'; 6 | 7 | -------------------------------------------------------------------------------- /chaos-simulate-message-corruption/Readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/chaos-simulate-message-corruption/Readme.md -------------------------------------------------------------------------------- /chaos-simulate-message-corruption/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /chaos-simulate-message-corruption/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic with-random-bytes 9 | -------------------------------------------------------------------------------- /chaos-simulate-message-corruption/step-07-simulate-massage-corruption.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.chaos.ProduceSimulateMessageCorruptionPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "topic" : "with-random-bytes", 6 | "sizeInBytes" : 10, 7 | "rateInPercent" : 100 8 | } 9 | } -------------------------------------------------------------------------------- /chaos-simulate-message-corruption/step-08-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /chaos-simulate-message-corruption/step-09-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"message": "hello world"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --topic with-random-bytes -------------------------------------------------------------------------------- /chaos-simulate-message-corruption/step-10-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic with-random-bytes \ 6 | --from-beginning \ 7 | --timeout-ms 10000 8 | -------------------------------------------------------------------------------- /chaos-simulate-message-corruption/step-11-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /chaos-simulate-message-corruption/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3MDI3MX0.zXlvOU1BvJC44GIeJuQvIxL5h57ndBZyIxRxkBVN78U'; 6 | 7 | -------------------------------------------------------------------------------- /chaos-simulate-slow-broker/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /chaos-simulate-slow-broker/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic slow-topic 9 | -------------------------------------------------------------------------------- /chaos-simulate-slow-broker/step-07-simulate-slow-broker.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.chaos.SimulateSlowBrokerPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "rateInPercent" : 100, 6 | "minLatencyMs" : 2000, 7 | "maxLatencyMs" : 2001 8 | } 9 | } -------------------------------------------------------------------------------- /chaos-simulate-slow-broker/step-08-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /chaos-simulate-slow-broker/step-09-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-producer-perf-test \ 3 | --producer.config teamA-sa.properties \ 4 | --record-size 10 \ 5 | --throughput 1 \ 6 | --num-records 10 \ 7 | --topic slow-topic -------------------------------------------------------------------------------- /chaos-simulate-slow-broker/step-10-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /chaos-simulate-slow-broker/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3MDU3Mn0.8m-BdCIasz75xvlsum0HPhd2ZhPpUuxGah4AFraco8g'; 6 | 7 | -------------------------------------------------------------------------------- /chaos-simulate-slow-producers-consumers/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /chaos-simulate-slow-producers-consumers/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic slow-topic 9 | -------------------------------------------------------------------------------- /chaos-simulate-slow-producers-consumers/step-07-simulate-slow-producer-consumers.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.chaos.SimulateSlowProducersConsumersPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "topic" : "slow-topic", 6 | "rateInPercent" : 100, 7 | "minLatencyMs" : 3000, 8 | "maxLatencyMs" : 3001 9 | } 10 | } -------------------------------------------------------------------------------- /chaos-simulate-slow-producers-consumers/step-08-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /chaos-simulate-slow-producers-consumers/step-09-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-producer-perf-test \ 3 | --producer.config teamA-sa.properties \ 4 | --record-size 10 \ 5 | --throughput 1 \ 6 | --num-records 10 \ 7 | --topic slow-topic -------------------------------------------------------------------------------- /chaos-simulate-slow-producers-consumers/step-10-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /chaos-simulate-slow-producers-consumers/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3MDg2Mn0.t82GWolrkRbJU53c6lB-KyfcCOsBCzW5VWWen8Hi6vY'; 6 | 7 | -------------------------------------------------------------------------------- /cluster-switching/clusters.yaml: -------------------------------------------------------------------------------- 1 | config: 2 | main: 3 | bootstrap.servers: kafka1:9092,kafka2:9093,kafka3:9094 4 | 5 | failover: 6 | bootstrap.servers: failover-kafka1:9092,failover-kafka2:9093,failover-kafka3:9094 7 | gateway.roles: failover 8 | -------------------------------------------------------------------------------- /cluster-switching/output/step-08-CREATE_VIRTUAL_CLUSTER.txt: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3MTA2M30.RVlGt-y6NbeqGXs9I5rBXWZsSQuf4FfXBpVlEUG9YGI'; 6 | 7 | -------------------------------------------------------------------------------- /cluster-switching/output/step-13-CONSUME.txt: -------------------------------------------------------------------------------- 1 | Processed a total of 1 messages 2 | { 3 | "name": "tom", 4 | "username": "tom@conduktor.io", 5 | "password": "motorhead", 6 | "visa": "#abc123", 7 | "address": "Chancery lane, London" 8 | } 9 | -------------------------------------------------------------------------------- /cluster-switching/output/step-16-FAILOVER.txt: -------------------------------------------------------------------------------- 1 | { 2 | "message": "Cluster switched" 3 | } 4 | -------------------------------------------------------------------------------- /cluster-switching/step-07-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /cluster-switching/step-09-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic users 9 | -------------------------------------------------------------------------------- /cluster-switching/step-11-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --list 5 | -------------------------------------------------------------------------------- /cluster-switching/step-12-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:29092,localhost:29093,localhost:29094 \ 4 | --topic _topicMappings \ 5 | --from-beginning \ 6 | --max-messages 1 \ 7 | --timeout-ms 15000 | jq 8 | -------------------------------------------------------------------------------- /cluster-switching/step-13-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:29092,localhost:29093,localhost:29094 \ 4 | --topic teamAusers \ 5 | --from-beginning \ 6 | --max-messages 1 \ 7 | --timeout-ms 15000 | jq 8 | -------------------------------------------------------------------------------- /cluster-switching/step-14-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:29092,localhost:29093,localhost:29094 \ 4 | --list 5 | -------------------------------------------------------------------------------- /cluster-switching/step-15-FAILOVER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request POST 'http://localhost:8888/admin/pclusters/v1/pcluster/main/switch?to=failover' \ 4 | --user 'admin:conduktor' \ 5 | --silent | jq 6 | -------------------------------------------------------------------------------- /cluster-switching/step-16-FAILOVER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request POST 'http://localhost:8889/admin/pclusters/v1/pcluster/main/switch?to=failover' \ 4 | --user 'admin:conduktor' \ 5 | --silent | jq 6 | -------------------------------------------------------------------------------- /cluster-switching/step-17-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"name":"alice","username":"alice@conduktor.io","password":"youpi","visa":"#812SSS","address":"Les ifs"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --topic users -------------------------------------------------------------------------------- /cluster-switching/step-18-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic users \ 6 | --from-beginning \ 7 | --max-messages 3 \ 8 | --timeout-ms 10000 | jq 9 | -------------------------------------------------------------------------------- /cluster-switching/step-19-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --topic teamAusers \ 5 | --from-beginning \ 6 | --timeout-ms 10000 | jq 7 | -------------------------------------------------------------------------------- /cluster-switching/step-20-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:29092,localhost:29093,localhost:29094 \ 4 | --topic teamAusers \ 5 | --from-beginning \ 6 | --max-messages 3 \ 7 | --timeout-ms 15000 | jq 8 | -------------------------------------------------------------------------------- /cluster-switching/step-21-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /cluster-switching/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3MTMyMX0.c8qOJywtxWDc3m4Z2qJuuQeYDrV2OPV5jw2tJeofxVk'; 6 | 7 | -------------------------------------------------------------------------------- /data-masking/output/step-05-CREATE_VIRTUAL_CLUSTER.txt: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3MTU5OX0.yYJUqhlGN3JPYoLxZs8SracYxcrY6XpJsDfi8slfmnI'; 6 | 7 | -------------------------------------------------------------------------------- /data-masking/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /data-masking/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic customers 9 | -------------------------------------------------------------------------------- /data-masking/step-07-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-07-data-masking.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/data-masking" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-07-data-masking.json | jq 10 | -------------------------------------------------------------------------------- /data-masking/step-08-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /data-masking/step-10-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic customers \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /data-masking/step-11-REMOVE_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request DELETE "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/data-masking" \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /data-masking/step-12-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic customers \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /data-masking/step-13-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /data-masking/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3MTc3OX0.CuSmqD1vA-bovWyV6LjyAJU1A7XwYEgMYuYXF34HX1k'; 6 | 7 | -------------------------------------------------------------------------------- /encryption-crypto-shredding/output/step-06-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic customers-shredding. 2 | -------------------------------------------------------------------------------- /encryption-crypto-shredding/output/step-07-LIST_TOPICS.txt: -------------------------------------------------------------------------------- 1 | customers-shredding 2 | -------------------------------------------------------------------------------- /encryption-crypto-shredding/output/step-10-PRODUCE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/encryption-crypto-shredding/output/step-10-PRODUCE.txt -------------------------------------------------------------------------------- /encryption-crypto-shredding/output/step-15-SH.txt: -------------------------------------------------------------------------------- 1 | [ 2 | "secret-for-laura", 3 | "secret-for-tom" 4 | ] 5 | -------------------------------------------------------------------------------- /encryption-crypto-shredding/output/step-16-SH.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/encryption-crypto-shredding/output/step-16-SH.txt -------------------------------------------------------------------------------- /encryption-crypto-shredding/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /encryption-crypto-shredding/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic customers-shredding 9 | -------------------------------------------------------------------------------- /encryption-crypto-shredding/step-07-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /encryption-crypto-shredding/step-09-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /encryption-crypto-shredding/step-11-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic customers-shredding \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /encryption-crypto-shredding/step-13-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /encryption-crypto-shredding/step-14-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic customers-shredding \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /encryption-crypto-shredding/step-15-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8200/v1/transit/keys/?list=true' \ 4 | --silent \ 5 | --header "X-Vault-Token: vault-plaintext-root-token" | jq -r ".data.keys" -------------------------------------------------------------------------------- /encryption-crypto-shredding/step-17-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic customers-shredding \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /encryption-crypto-shredding/step-18-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /encryption-crypto-shredding/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3MjIwNn0.R8DxrwJXrbWxBxV2fRRjWZePpjsh8wZlbVRJlIn8tHE'; 6 | 7 | -------------------------------------------------------------------------------- /encryption-decrypt-only-specific-fields/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /encryption-decrypt-only-specific-fields/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic customers 9 | -------------------------------------------------------------------------------- /encryption-decrypt-only-specific-fields/step-07-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-07-encrypt.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/encrypt" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-07-encrypt.json | jq 10 | -------------------------------------------------------------------------------- /encryption-decrypt-only-specific-fields/step-08-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /encryption-decrypt-only-specific-fields/step-10-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic customers \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /encryption-decrypt-only-specific-fields/step-11-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-11-decrypt.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/decrypt" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-11-decrypt.json | jq 10 | -------------------------------------------------------------------------------- /encryption-decrypt-only-specific-fields/step-12-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /encryption-decrypt-only-specific-fields/step-13-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic customers \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /encryption-decrypt-only-specific-fields/step-14-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /encryption-decrypt-only-specific-fields/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3MjY0NX0.0JDixT0msgt2KB3CiZca0tqdQPnbpKK68k8lXc9a2ZY'; 6 | 7 | -------------------------------------------------------------------------------- /encryption-external-storage/output/step-15-CONSUME.txt: -------------------------------------------------------------------------------- 1 | jq: parse error: Invalid numeric literal at line 1, column 18 2 | [2024-04-10 01:08:30,312] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) 3 | org.apache.kafka.common.errors.TimeoutException 4 | Processed a total of 2 messages 5 | -------------------------------------------------------------------------------- /encryption-external-storage/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /encryption-external-storage/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic customers 9 | -------------------------------------------------------------------------------- /encryption-external-storage/step-07-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-07-encrypt.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/encrypt" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-07-encrypt.json | jq 10 | -------------------------------------------------------------------------------- /encryption-external-storage/step-08-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /encryption-external-storage/step-09-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --topic _conduktor_gateway_encryption_configs \ 5 | --from-beginning \ 6 | --timeout-ms 10000 | jq 7 | -------------------------------------------------------------------------------- /encryption-external-storage/step-11-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic customers \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /encryption-external-storage/step-12-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-12-decrypt.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/decrypt" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-12-decrypt.json | jq 10 | -------------------------------------------------------------------------------- /encryption-external-storage/step-12-decrypt.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.DecryptPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "topic" : "customers", 6 | "kmsConfig" : { 7 | "vault" : { 8 | "uri" : "http://vault:8200", 9 | "token" : "vault-plaintext-root-token", 10 | "version" : 1 11 | } 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /encryption-external-storage/step-13-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /encryption-external-storage/step-14-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic customers \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /encryption-external-storage/step-15-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --topic teamAcustomers \ 5 | --from-beginning \ 6 | --timeout-ms 10000 \ 7 | --property print.headers=true | jq 8 | -------------------------------------------------------------------------------- /encryption-external-storage/step-16-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /encryption-external-storage/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3MzA3OH0.1s8tIIOJ4j7R9h-V0rwjyOC4cYtufsYbJ897DNwTwzY'; 6 | 7 | -------------------------------------------------------------------------------- /encryption-payload-external-storage/output/step-15-CONSUME.txt: -------------------------------------------------------------------------------- 1 | jq: parse error: Invalid numeric literal at line 1, column 18 2 | [2024-04-10 01:22:34,127] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) 3 | org.apache.kafka.common.errors.TimeoutException 4 | Processed a total of 2 messages 5 | -------------------------------------------------------------------------------- /encryption-payload-external-storage/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /encryption-payload-external-storage/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic customers 9 | -------------------------------------------------------------------------------- /encryption-payload-external-storage/step-07-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-07-encrypt.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/encrypt" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-07-encrypt.json | jq 10 | -------------------------------------------------------------------------------- /encryption-payload-external-storage/step-07-encrypt.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.EncryptPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "externalStorage" : true, 6 | "payload" : { 7 | "keySecretId" : "full-payload-secret", 8 | "algorithm" : "AES_GCM" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /encryption-payload-external-storage/step-08-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /encryption-payload-external-storage/step-09-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --topic _conduktor_gateway_encryption_configs \ 5 | --from-beginning \ 6 | --timeout-ms 10000 | jq 7 | -------------------------------------------------------------------------------- /encryption-payload-external-storage/step-11-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic customers \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /encryption-payload-external-storage/step-12-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-12-decrypt.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/decrypt" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-12-decrypt.json | jq 10 | -------------------------------------------------------------------------------- /encryption-payload-external-storage/step-12-decrypt.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.DecryptPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "topic" : "customers", 6 | "kmsConfig" : { 7 | "vault" : { 8 | "uri" : "http://vault:8200", 9 | "token" : "vault-plaintext-root-token", 10 | "version" : 1 11 | } 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /encryption-payload-external-storage/step-13-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /encryption-payload-external-storage/step-14-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic customers \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /encryption-payload-external-storage/step-15-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --topic teamAcustomers \ 5 | --from-beginning \ 6 | --timeout-ms 10000 \ 7 | --property print.headers=true | jq 8 | -------------------------------------------------------------------------------- /encryption-payload-external-storage/step-16-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /encryption-payload-external-storage/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3MzkyMX0.ChW9-TWHJfmFHdmH2bB7acLDP78xGMAxxPu2ERaYKwo'; 6 | 7 | -------------------------------------------------------------------------------- /encryption-payload/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /encryption-payload/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic customers 9 | -------------------------------------------------------------------------------- /encryption-payload/step-07-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-07-encrypt-full-payload.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/encrypt-full-payload" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-07-encrypt-full-payload.json | jq 10 | -------------------------------------------------------------------------------- /encryption-payload/step-07-encrypt-full-payload.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.EncryptPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "payload" : { 6 | "keySecretId" : "full-payload-secret", 7 | "algorithm" : "AES_GCM" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /encryption-payload/step-08-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /encryption-payload/step-10-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic customers \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /encryption-payload/step-11-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-11-decrypt-full-payload.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/decrypt-full-payload" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-11-decrypt-full-payload.json | jq 10 | -------------------------------------------------------------------------------- /encryption-payload/step-12-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /encryption-payload/step-13-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic customers \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /encryption-payload/step-14-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /encryption-payload/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3MzQ4Mn0.mlV94w7GDBzONUuwAlqDtwEMVetCxw2WcZcvhDbvuIM'; 6 | 7 | -------------------------------------------------------------------------------- /encryption-performance/examples.json: -------------------------------------------------------------------------------- 1 | {"name": "tom","username": "tom@conduktor.io","password": "motorhead", "visa":"#abc123", "address":"Chancery lane, London"} 2 | {"name": "laura","username": "laura@conduktor.io","password": "kitesurf","visa": "#888999XZ;","address": "Dubai, UAE"}' -------------------------------------------------------------------------------- /encryption-performance/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /encryption-performance/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic customers 9 | -------------------------------------------------------------------------------- /encryption-performance/step-07-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-07-encrypt.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/encrypt" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-07-encrypt.json | jq 10 | -------------------------------------------------------------------------------- /encryption-performance/step-08-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-08-decrypt.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/decrypt" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-08-decrypt.json | jq 10 | -------------------------------------------------------------------------------- /encryption-performance/step-08-decrypt.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.DecryptPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "topic" : "customers", 6 | "kmsConfig" : { 7 | "vault" : { 8 | "uri" : "http://vault:8200", 9 | "token" : "vault-plaintext-root-token", 10 | "version" : 1 11 | } 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /encryption-performance/step-09-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /encryption-performance/step-10-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-producer-perf-test \ 3 | --topic customers \ 4 | --throughput -1 \ 5 | --num-records 1000000 \ 6 | --producer-props \ 7 | bootstrap.servers=localhost:6969 \ 8 | linger.ms=100 \ 9 | compression.type=lz4 \ 10 | --producer.config teamA-sa.properties \ 11 | --payload-file examples.json -------------------------------------------------------------------------------- /encryption-performance/step-11-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /encryption-performance/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3NDQxNH0.fznoq6eeylr5OTkXQrIF06gHajBqac6NuRyzxdxxvkM'; 6 | 7 | -------------------------------------------------------------------------------- /encryption-third-party/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /encryption-third-party/step-07-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic customers 9 | -------------------------------------------------------------------------------- /encryption-third-party/step-09-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA/username/third-party' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /encryption-third-party/step-11-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic customers \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /encryption-third-party/step-12-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-third-party.properties \ 5 | --topic customers \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /encryption-third-party/step-13-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /encryption-third-party/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3NTkwNH0.LUpYAuk3rPN9B_VPwTT7dtWgSXNyBIIlEmxDG6wY9KA'; 6 | 7 | -------------------------------------------------------------------------------- /encryption-vault-with-secret-management/output/step-06-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic customers. 2 | -------------------------------------------------------------------------------- /encryption-vault-with-secret-management/output/step-07-LIST_TOPICS.txt: -------------------------------------------------------------------------------- 1 | customers 2 | -------------------------------------------------------------------------------- /encryption-vault-with-secret-management/output/step-10-PRODUCE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/encryption-vault-with-secret-management/output/step-10-PRODUCE.txt -------------------------------------------------------------------------------- /encryption-vault-with-secret-management/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /encryption-vault-with-secret-management/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic customers 9 | -------------------------------------------------------------------------------- /encryption-vault-with-secret-management/step-07-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /encryption-vault-with-secret-management/step-09-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /encryption-vault-with-secret-management/step-11-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic customers \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /encryption-vault-with-secret-management/step-13-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /encryption-vault-with-secret-management/step-14-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic customers \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /encryption-vault-with-secret-management/step-15-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /header-injection/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /header-injection/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic users 9 | -------------------------------------------------------------------------------- /header-injection/step-07-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-07-inject-headers.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/inject-headers" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-07-inject-headers.json | jq 10 | -------------------------------------------------------------------------------- /header-injection/step-07-inject-headers.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.DynamicHeaderInjectionPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "headers" : { 6 | "X-MY-KEY" : "my own value", 7 | "X-USER" : "{{user}}", 8 | "X-INTERPOLATED" : "User {{user}} via ip {{userIp}}" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /header-injection/step-09-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic users \ 6 | --from-beginning \ 7 | --max-messages 2 \ 8 | --timeout-ms 10000 \ 9 | --property print.headers=true | jq 10 | -------------------------------------------------------------------------------- /header-injection/step-10-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-10-remove-headers.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/remove-headers" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-10-remove-headers.json | jq 10 | -------------------------------------------------------------------------------- /header-injection/step-10-remove-headers.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.safeguard.MessageHeaderRemovalPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "headerKeyRegex" : "X-MY-.*" 6 | } 7 | } -------------------------------------------------------------------------------- /header-injection/step-11-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic users \ 6 | --from-beginning \ 7 | --max-messages 2 \ 8 | --timeout-ms 10000 \ 9 | --property print.headers=true | jq 10 | -------------------------------------------------------------------------------- /header-injection/step-12-REMOVE_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request DELETE "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/remove-headers" \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /header-injection/step-13-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic users \ 6 | --from-beginning \ 7 | --max-messages 2 \ 8 | --timeout-ms 10000 \ 9 | --property print.headers=true | jq 10 | -------------------------------------------------------------------------------- /header-injection/step-14-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /header-injection/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3NjYwMH0.KTeweJ6oCTbIjXS-Z1DJxq_3QGb9L4bfBaodXHFVkWU'; 6 | 7 | -------------------------------------------------------------------------------- /ksqlDB/output/step-05-CREATE_VIRTUAL_CLUSTER.txt: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3Njc4OX0.wIf0tn_sQ2pX9SCrzhJheT0XweMLS_mufzNM8UmO_Kw'; 6 | 7 | -------------------------------------------------------------------------------- /ksqlDB/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /ksqlDB/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --replication-factor 1 \ 5 | --partitions 100 \ 6 | --create --if-not-exists \ 7 | --topic concentrated 8 | -------------------------------------------------------------------------------- /ksqlDB/step-07-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --replication-factor 1 \ 5 | --partitions 100 \ 6 | --create --if-not-exists \ 7 | --topic concentrated_compacted 8 | -------------------------------------------------------------------------------- /ksqlDB/step-08-CREATE_CONCENTRATION_RULE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-08-concentration-rule.json | jq 3 | 4 | curl \ 5 | --request POST 'http://localhost:8888/admin/vclusters/v1/vcluster/teamA/concentration-rules' \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data "@step-08-concentration-rule.json" | jq 10 | -------------------------------------------------------------------------------- /ksqlDB/step-08-concentration-rule.json: -------------------------------------------------------------------------------- 1 | { 2 | "clusterId" : "main", 3 | "physicalTopicName" : "concentrated", 4 | "pattern" : "concentrated-.*" 5 | } -------------------------------------------------------------------------------- /ksqlDB/step-10-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /ksqlDB/step-11-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --list 5 | -------------------------------------------------------------------------------- /ksqlDB/step-13-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker exec ksqldb-server ksql 'http://localhost:8088' -f /sql/ksql.sql -------------------------------------------------------------------------------- /ksqlDB/step-14-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /ksqlDB/step-15-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --list 5 | -------------------------------------------------------------------------------- /ksqlDB/step-16-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /ksqlDB/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3Njk2NH0._8KbRt5kWrbYpbq73V2RfBvPyoRqhU2CRNx-LkQ_2ts'; 6 | 7 | -------------------------------------------------------------------------------- /large-messages/credentials: -------------------------------------------------------------------------------- 1 | [minio] 2 | aws_access_key_id = minio 3 | aws_secret_access_key = minio123 4 | -------------------------------------------------------------------------------- /large-messages/output/step-07-SH.txt: -------------------------------------------------------------------------------- 1 | { 2 | "Location": "/bucket" 3 | } 4 | -------------------------------------------------------------------------------- /large-messages/output/step-08-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic large-messages. 2 | -------------------------------------------------------------------------------- /large-messages/output/step-10-SH.txt: -------------------------------------------------------------------------------- 1 | -rw-r--r--@ 1 framiere staff 40M 10 avr 02:19 large-message.bin 2 | -------------------------------------------------------------------------------- /large-messages/output/step-11-SH.txt: -------------------------------------------------------------------------------- 1 | Reading payloads from: /Users/framiere/conduktor/conduktor-proxy/functional-testing/target/2024.04.10-01:54:14/large-messages/large-message.bin 2 | Number of messages read: 1 3 | 1 records sent, 0,674764 records/sec (26,99 MB/sec), 1474,00 ms avg latency, 1474,00 ms max latency, 1474 ms 50th, 1474 ms 95th, 1474 ms 99th, 1474 ms 99.9th. 4 | -------------------------------------------------------------------------------- /large-messages/output/step-12-SH.txt: -------------------------------------------------------------------------------- 1 | Processed a total of 1 messages 2 | -------------------------------------------------------------------------------- /large-messages/output/step-13-SH.txt: -------------------------------------------------------------------------------- 1 | -rw-r--r--@ 1 framiere staff 41943041 10 avr 02:19 from-kafka.bin 2 | -rw-r--r--@ 1 framiere staff 41943041 10 avr 02:19 large-message.bin 3 | -------------------------------------------------------------------------------- /large-messages/output/step-14-SH.txt: -------------------------------------------------------------------------------- 1 | 2024-04-09 22:19:49 40.0 MiB large-messages/84a95586-c2ea-407b-99e5-8cd921f58ae7 2 | -------------------------------------------------------------------------------- /large-messages/output/step-15-CONSUME.txt: -------------------------------------------------------------------------------- 1 | jq: parse error: Invalid numeric literal at line 1, column 17 2 | [2024-04-10 02:20:04,392] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) 3 | org.apache.kafka.common.errors.TimeoutException 4 | Processed a total of 1 messages 5 | -------------------------------------------------------------------------------- /large-messages/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /large-messages/step-07-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose exec cli-aws \ 3 | aws \ 4 | --profile minio \ 5 | --endpoint-url=http://minio:9000 \ 6 | --region eu-south-1 \ 7 | s3api create-bucket \ 8 | --bucket bucket -------------------------------------------------------------------------------- /large-messages/step-08-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic large-messages 9 | -------------------------------------------------------------------------------- /large-messages/step-09-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-09-large-messages.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/large-messages" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-09-large-messages.json | jq 10 | -------------------------------------------------------------------------------- /large-messages/step-10-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | openssl rand -hex $((20*1024*1024)) > large-message.bin 3 | ls -lh large-message.bin -------------------------------------------------------------------------------- /large-messages/step-12-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic large-messages \ 6 | --from-beginning \ 7 | --max-messages 1 > from-kafka.bin -------------------------------------------------------------------------------- /large-messages/step-13-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ls -lH *bin -------------------------------------------------------------------------------- /large-messages/step-14-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose exec cli-aws \ 3 | aws \ 4 | --profile minio \ 5 | --endpoint-url=http://minio:9000 \ 6 | --region eu-south-1 \ 7 | s3 \ 8 | ls s3://bucket --recursive --human-readable -------------------------------------------------------------------------------- /large-messages/step-15-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --topic teamAlarge-messages \ 5 | --from-beginning \ 6 | --timeout-ms 10000 \ 7 | --property print.headers=true | jq 8 | -------------------------------------------------------------------------------- /large-messages/step-16-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /large-messages/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3NzM0NX0.awpJ1y7mKo0Je4mwwHWDWN0xc7vRVsW7GGZfL2p_14o'; 6 | 7 | -------------------------------------------------------------------------------- /latency/output/step-05-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic physical-kafka. 2 | -------------------------------------------------------------------------------- /latency/output/step-07-CREATE_VIRTUAL_CLUSTER.txt: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3NzYzN30.hgWWOZ0NO2YCqrFBhWC0Sonpn2PSg6R4FmVdYeZSIMs'; 6 | 7 | -------------------------------------------------------------------------------- /latency/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /latency/step-05-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --replication-factor 1 \ 5 | --partitions 10 \ 6 | --create --if-not-exists \ 7 | --topic physical-kafka 8 | -------------------------------------------------------------------------------- /latency/step-06-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-run-class kafka.tools.EndToEndLatency \ 3 | localhost:19092,localhost:19093,localhost:19094 \ 4 | physical-kafka 10000 all 255 -------------------------------------------------------------------------------- /latency/step-08-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 10 \ 7 | --create --if-not-exists \ 8 | --topic via-gateway 9 | -------------------------------------------------------------------------------- /latency/step-09-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-run-class kafka.tools.EndToEndLatency \ 3 | localhost:6969 \ 4 | via-gateway 10000 all 255 \ 5 | teamA-sa.properties -------------------------------------------------------------------------------- /latency/step-10-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /latency/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3Nzg0N30.3fui2ajoQCVpUbJowNkHhwmPg5t5R-ea7slbRi34Ixs'; 6 | 7 | -------------------------------------------------------------------------------- /merge-cluster/clusters.yaml: -------------------------------------------------------------------------------- 1 | config: 2 | main: 3 | bootstrap.servers: kafka1:9092,kafka2:9093,kafka3:9094 4 | 5 | cluster1: 6 | bootstrap.servers: s1_kafka1:9092,s1_kafka2:9093,s1_kafka3:9094 7 | gateway.roles: upstream -------------------------------------------------------------------------------- /merge-cluster/output/step-06-CREATE_VIRTUAL_CLUSTER.txt: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3ODAxOH0.9PI2ih0677PHNdxUo38pvMXjDnZFzOu2AfUhDDGWL6E'; 6 | 7 | -------------------------------------------------------------------------------- /merge-cluster/output/step-08-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic cars. 2 | -------------------------------------------------------------------------------- /merge-cluster/output/step-10-SH.txt: -------------------------------------------------------------------------------- 1 | { 2 | "message": "us_cars is created" 3 | } 4 | -------------------------------------------------------------------------------- /merge-cluster/output/step-12-PRODUCE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/merge-cluster/output/step-12-PRODUCE.txt -------------------------------------------------------------------------------- /merge-cluster/output/step-14-CONSUME.txt: -------------------------------------------------------------------------------- 1 | [2024-04-10 02:34:07,406] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) 2 | org.apache.kafka.common.errors.TimeoutException 3 | Processed a total of 1 messages 4 | { 5 | "name": "us_cars_record" 6 | } 7 | -------------------------------------------------------------------------------- /merge-cluster/output/step-15-CONSUME.txt: -------------------------------------------------------------------------------- 1 | [2024-04-10 02:34:19,181] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) 2 | org.apache.kafka.common.errors.TimeoutException 3 | Processed a total of 1 messages 4 | { 5 | "name": "eu_cars_record" 6 | } 7 | -------------------------------------------------------------------------------- /merge-cluster/output/step-16-CONSUME.txt: -------------------------------------------------------------------------------- 1 | [2024-04-10 02:34:32,741] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) 2 | org.apache.kafka.common.errors.TimeoutException 3 | Processed a total of 1 messages 4 | { 5 | "name": "us_cars_record" 6 | } 7 | -------------------------------------------------------------------------------- /merge-cluster/step-05-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /merge-cluster/step-07-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --replication-factor 1 \ 5 | --partitions 1 \ 6 | --create --if-not-exists \ 7 | --topic cars 8 | -------------------------------------------------------------------------------- /merge-cluster/step-08-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:29092,localhost:29093,localhost:29094 \ 4 | --replication-factor 1 \ 5 | --partitions 1 \ 6 | --create --if-not-exists \ 7 | --topic cars 8 | -------------------------------------------------------------------------------- /merge-cluster/step-09-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --silent \ 4 | --request POST localhost:8888/internal/alias-topic/teamA/eu_cars \ 5 | --user 'admin:conduktor' \ 6 | --header 'Content-Type: application/json' \ 7 | --data-raw '{ 8 | "clusterId": "main", 9 | "physicalTopicName": "cars" 10 | }' | jq -------------------------------------------------------------------------------- /merge-cluster/step-10-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --silent \ 4 | --request POST localhost:8888/internal/alias-topic/teamA/us_cars \ 5 | --user 'admin:conduktor' \ 6 | --header 'Content-Type: application/json' \ 7 | --data-raw '{ 8 | "clusterId": "cluster1", 9 | "physicalTopicName": "cars" 10 | }' | jq -------------------------------------------------------------------------------- /merge-cluster/step-11-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"name":"eu_cars_record"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --topic eu_cars -------------------------------------------------------------------------------- /merge-cluster/step-12-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"name":"us_cars_record"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --topic us_cars -------------------------------------------------------------------------------- /merge-cluster/step-13-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic eu_cars \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /merge-cluster/step-14-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic us_cars \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /merge-cluster/step-15-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --topic cars \ 5 | --from-beginning \ 6 | --timeout-ms 10000 | jq 7 | -------------------------------------------------------------------------------- /merge-cluster/step-16-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:29092,localhost:29093,localhost:29094 \ 4 | --topic cars \ 5 | --from-beginning \ 6 | --timeout-ms 10000 | jq 7 | -------------------------------------------------------------------------------- /merge-cluster/step-17-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /merge-cluster/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3ODI1Mn0.iVZhmVfsiYP4JUokPxu8xkv11aLhb_y7M0hQhiXjn1I'; 6 | 7 | -------------------------------------------------------------------------------- /move-to-docs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | demos=~/conduktor/conduktor-docs/docs/gateway/demos 4 | for i in `find . -type d -depth 1` ; do 5 | parent=`grep -m 1 'tag:' $i/Readme-doc.md | awk '{print $2}'` 6 | parentUpper=$(echo $parent | awk '{print toupper(substr($0,1,1)) substr($0,2)}') 7 | mkdir -p $demos/$parent 8 | cp $i/Readme-doc.md-asciinema $demos/$parent/$i.md 9 | done 10 | -------------------------------------------------------------------------------- /multi-tenancy/london-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJsb25kb24iLCJleHAiOjE3MjA0Nzg4MTh9.ezFPLqZWesj_8gX60vh8jn2hqS3oWcj9aYqM2EuZZtY'; 6 | 7 | -------------------------------------------------------------------------------- /multi-tenancy/output/step-07-CREATE_VIRTUAL_CLUSTER.txt: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJwYXJpcyIsImV4cCI6MTcyMDQ3ODUxOX0.Jb9TRqossTJCuCkXBh-CmW7AUI-TgTJ8Ap-GGDIP1kE'; 6 | 7 | -------------------------------------------------------------------------------- /multi-tenancy/output/step-08-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic londonTopic. 2 | -------------------------------------------------------------------------------- /multi-tenancy/output/step-09-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic parisTopic. 2 | -------------------------------------------------------------------------------- /multi-tenancy/output/step-10-LIST_TOPICS.txt: -------------------------------------------------------------------------------- 1 | londonTopic 2 | -------------------------------------------------------------------------------- /multi-tenancy/output/step-11-LIST_TOPICS.txt: -------------------------------------------------------------------------------- 1 | parisTopic 2 | -------------------------------------------------------------------------------- /multi-tenancy/output/step-12-PRODUCE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/multi-tenancy/output/step-12-PRODUCE.txt -------------------------------------------------------------------------------- /multi-tenancy/output/step-13-CONSUME.txt: -------------------------------------------------------------------------------- 1 | jq: parse error: Invalid numeric literal at line 1, column 18 2 | [2024-04-10 02:42:17,493] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) 3 | org.apache.kafka.common.errors.TimeoutException 4 | Processed a total of 1 messages 5 | -------------------------------------------------------------------------------- /multi-tenancy/output/step-14-PRODUCE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/multi-tenancy/output/step-14-PRODUCE.txt -------------------------------------------------------------------------------- /multi-tenancy/output/step-15-CONSUME.txt: -------------------------------------------------------------------------------- 1 | jq: parse error: Invalid numeric literal at line 1, column 20 2 | [2024-04-10 02:42:30,664] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) 3 | org.apache.kafka.common.errors.TimeoutException 4 | Processed a total of 1 messages 5 | -------------------------------------------------------------------------------- /multi-tenancy/output/step-16-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic existingLondonTopic. 2 | -------------------------------------------------------------------------------- /multi-tenancy/output/step-17-PRODUCE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/multi-tenancy/output/step-17-PRODUCE.txt -------------------------------------------------------------------------------- /multi-tenancy/output/step-18-SH.txt: -------------------------------------------------------------------------------- 1 | { 2 | "logicalTopicName": "existingLondonTopic", 3 | "clusterId": "main", 4 | "physicalTopicName": "existingLondonTopic", 5 | "readOnly": false, 6 | "type": "ALIAS" 7 | } 8 | -------------------------------------------------------------------------------- /multi-tenancy/output/step-19-LIST_TOPICS.txt: -------------------------------------------------------------------------------- 1 | existingLondonTopic 2 | londonTopic 3 | -------------------------------------------------------------------------------- /multi-tenancy/output/step-20-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic existingSharedTopic. 2 | -------------------------------------------------------------------------------- /multi-tenancy/output/step-21-PRODUCE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/multi-tenancy/output/step-21-PRODUCE.txt -------------------------------------------------------------------------------- /multi-tenancy/output/step-22-SH.txt: -------------------------------------------------------------------------------- 1 | { 2 | "logicalTopicName": "existingSharedTopic", 3 | "clusterId": "main", 4 | "physicalTopicName": "existingSharedTopic", 5 | "readOnly": false, 6 | "type": "ALIAS" 7 | } 8 | -------------------------------------------------------------------------------- /multi-tenancy/output/step-23-LIST_TOPICS.txt: -------------------------------------------------------------------------------- 1 | existingLondonTopic 2 | existingSharedTopic 3 | londonTopic 4 | -------------------------------------------------------------------------------- /multi-tenancy/output/step-24-CONSUME.txt: -------------------------------------------------------------------------------- 1 | jq: parse error: Invalid numeric literal at line 1, column 18 2 | [2024-04-10 02:42:50,507] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) 3 | org.apache.kafka.common.errors.TimeoutException 4 | Processed a total of 1 messages 5 | -------------------------------------------------------------------------------- /multi-tenancy/output/step-25-CONSUME.txt: -------------------------------------------------------------------------------- 1 | [2024-04-10 02:43:02,246] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) 2 | org.apache.kafka.common.errors.TimeoutException 3 | Processed a total of 1 messages 4 | { 5 | "message": "Existing shared message" 6 | } 7 | -------------------------------------------------------------------------------- /multi-tenancy/output/step-26-SH.txt: -------------------------------------------------------------------------------- 1 | { 2 | "logicalTopicName": "existingSharedTopic", 3 | "clusterId": "main", 4 | "physicalTopicName": "existingSharedTopic", 5 | "readOnly": false, 6 | "type": "ALIAS" 7 | } 8 | -------------------------------------------------------------------------------- /multi-tenancy/output/step-27-LIST_TOPICS.txt: -------------------------------------------------------------------------------- 1 | existingSharedTopic 2 | parisTopic 3 | -------------------------------------------------------------------------------- /multi-tenancy/output/step-28-CONSUME.txt: -------------------------------------------------------------------------------- 1 | [2024-04-10 02:43:15,440] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) 2 | org.apache.kafka.common.errors.TimeoutException 3 | Processed a total of 1 messages 4 | { 5 | "message": "Existing shared message" 6 | } 7 | -------------------------------------------------------------------------------- /multi-tenancy/paris-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJwYXJpcyIsImV4cCI6MTcyMDQ3ODgyNH0.O3iaO8-GzfYtmPdi0iRS8bEMS2KrWmai99ot-BzE5xw'; 6 | 7 | -------------------------------------------------------------------------------- /multi-tenancy/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /multi-tenancy/step-05-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --list 5 | -------------------------------------------------------------------------------- /multi-tenancy/step-08-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config london-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic londonTopic 9 | -------------------------------------------------------------------------------- /multi-tenancy/step-09-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config paris-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic parisTopic 9 | -------------------------------------------------------------------------------- /multi-tenancy/step-10-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config london-sa.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /multi-tenancy/step-11-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config paris-sa.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /multi-tenancy/step-12-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"message: "Hello from London"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config london-sa.properties \ 6 | --topic londonTopic -------------------------------------------------------------------------------- /multi-tenancy/step-13-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config london-sa.properties \ 5 | --topic londonTopic \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /multi-tenancy/step-14-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"message: "Bonjour depuis Paris"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config paris-sa.properties \ 6 | --topic parisTopic -------------------------------------------------------------------------------- /multi-tenancy/step-15-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config paris-sa.properties \ 5 | --topic parisTopic \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /multi-tenancy/step-16-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --replication-factor 1 \ 5 | --partitions 1 \ 6 | --create --if-not-exists \ 7 | --topic existingLondonTopic 8 | -------------------------------------------------------------------------------- /multi-tenancy/step-17-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"message: "Hello from London"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 5 | --topic existingLondonTopic -------------------------------------------------------------------------------- /multi-tenancy/step-18-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --silent \ 4 | --request POST localhost:8888/admin/vclusters/v1/vcluster/london/topics/existingLondonTopic \ 5 | --user 'admin:conduktor' \ 6 | --header 'Content-Type: application/json' \ 7 | --data-raw '{ 8 | "physicalTopicName": "existingLondonTopic", 9 | "readOnly": false, 10 | "type": "alias" 11 | }' | jq -------------------------------------------------------------------------------- /multi-tenancy/step-19-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config london-sa.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /multi-tenancy/step-20-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --replication-factor 1 \ 5 | --partitions 1 \ 6 | --create --if-not-exists \ 7 | --topic existingSharedTopic 8 | -------------------------------------------------------------------------------- /multi-tenancy/step-21-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"message": "Existing shared message"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 5 | --topic existingSharedTopic -------------------------------------------------------------------------------- /multi-tenancy/step-22-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --silent \ 4 | --request POST localhost:8888/admin/vclusters/v1/vcluster/london/topics/existingSharedTopic \ 5 | --user 'admin:conduktor' \ 6 | --header 'Content-Type: application/json' \ 7 | --data-raw '{ 8 | "physicalTopicName": "existingSharedTopic", 9 | "readOnly": false, 10 | "type": "alias" 11 | }' | jq -------------------------------------------------------------------------------- /multi-tenancy/step-23-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config london-sa.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /multi-tenancy/step-24-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config london-sa.properties \ 5 | --topic existingLondonTopic \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /multi-tenancy/step-25-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config london-sa.properties \ 5 | --topic existingSharedTopic \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /multi-tenancy/step-26-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --silent \ 4 | --request POST localhost:8888/admin/vclusters/v1/vcluster/paris/topics/existingSharedTopic \ 5 | --user 'admin:conduktor' \ 6 | --header 'Content-Type: application/json' \ 7 | --data-raw '{ 8 | "physicalTopicName": "existingSharedTopic", 9 | "readOnly": false, 10 | "type": "alias" 11 | }' | jq -------------------------------------------------------------------------------- /multi-tenancy/step-27-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config paris-sa.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /multi-tenancy/step-28-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config paris-sa.properties \ 5 | --topic existingSharedTopic \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /multi-tenancy/step-29-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /oauth/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . utils.sh 4 | 5 | header 'Oauth' 6 | execute "step-04-DOCKER.sh" "Starting the docker environment" 7 | execute "step-06-CREATE_TOPICS.sh" "Creating topic cars on gateway1" 8 | execute "step-07-LIST_TOPICS.sh" "Listing topics in gateway1" 9 | execute "step-08-DOCKER.sh" "Tearing down the docker environment" 10 | -------------------------------------------------------------------------------- /oauth/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /oauth/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config user-1.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic cars 9 | -------------------------------------------------------------------------------- /oauth/step-07-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config user-1.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /oauth/step-08-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /safeguard-client-id/output/step-06-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic users. 2 | -------------------------------------------------------------------------------- /safeguard-client-id/output/step-07-LIST_TOPICS.txt: -------------------------------------------------------------------------------- 1 | users 2 | -------------------------------------------------------------------------------- /safeguard-client-id/output/step-08-ADD_INTERCEPTOR.txt: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass": "io.conduktor.gateway.interceptor.safeguard.ClientIdRequiredPolicyPlugin", 3 | "priority": 100, 4 | "config": { 5 | "namingConvention": "naming-convention-.*" 6 | } 7 | } 8 | { 9 | "message": "client-id is created" 10 | } 11 | -------------------------------------------------------------------------------- /safeguard-client-id/output/step-11-SH.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/safeguard-client-id/output/step-11-SH.txt -------------------------------------------------------------------------------- /safeguard-client-id/output/step-12-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic customers. 2 | -------------------------------------------------------------------------------- /safeguard-client-id/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /safeguard-client-id/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic users 9 | -------------------------------------------------------------------------------- /safeguard-client-id/step-07-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /safeguard-client-id/step-08-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-08-client-id.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/client-id" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-08-client-id.json | jq 10 | -------------------------------------------------------------------------------- /safeguard-client-id/step-08-client-id.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.safeguard.ClientIdRequiredPolicyPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "namingConvention" : "naming-convention-.*" 6 | } 7 | } -------------------------------------------------------------------------------- /safeguard-client-id/step-09-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /safeguard-client-id/step-10-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic customers 9 | -------------------------------------------------------------------------------- /safeguard-client-id/step-11-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo >> teamA-sa.properties 3 | echo "client.id=naming-convention-for-this-application" >> teamA-sa.properties -------------------------------------------------------------------------------- /safeguard-client-id/step-12-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic customers 9 | -------------------------------------------------------------------------------- /safeguard-client-id/step-13-AUDITLOG.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --topic _conduktor_gateway_auditlogs \ 5 | --from-beginning \ 6 | --timeout-ms 3000 \| jq 'select(.type=="SAFEGUARD" and .eventData.plugin=="io.conduktor.gateway.interceptor.safeguard.ClientIdRequiredPolicyPlugin")' 7 | -------------------------------------------------------------------------------- /safeguard-client-id/step-14-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /safeguard-schema-id/output/step-06-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic users. 2 | -------------------------------------------------------------------------------- /safeguard-schema-id/output/step-07-LIST_TOPICS.txt: -------------------------------------------------------------------------------- 1 | users 2 | -------------------------------------------------------------------------------- /safeguard-schema-id/output/step-08-ADD_INTERCEPTOR.txt: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass": "io.conduktor.gateway.interceptor.safeguard.TopicRequiredSchemaIdPolicyPlugin", 3 | "priority": 100, 4 | "config": { 5 | "topic": "users", 6 | "schemaIdRequired": true 7 | } 8 | } 9 | { 10 | "message": "schema-id is created" 11 | } 12 | -------------------------------------------------------------------------------- /safeguard-schema-id/output/step-10-PRODUCE.txt: -------------------------------------------------------------------------------- 1 | [2024-04-10 03:16:59,779] ERROR Error when sending message to topic users with key: null, value: 21 bytes with error: (org.apache.kafka.clients.producer.internals.ErrorLoggingCallback) 2 | org.apache.kafka.common.errors.PolicyViolationException: Request parameters do not satisfy the configured policy. Topic 'users' with schemaId is required. 3 | -------------------------------------------------------------------------------- /safeguard-schema-id/output/step-11-CONSUME.txt: -------------------------------------------------------------------------------- 1 | [2024-04-10 03:17:11,021] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) 2 | org.apache.kafka.common.errors.TimeoutException 3 | Processed a total of 0 messages 4 | -------------------------------------------------------------------------------- /safeguard-schema-id/output/step-13-SH.txt: -------------------------------------------------------------------------------- 1 | [ 2 | "users-value" 3 | ] 4 | -------------------------------------------------------------------------------- /safeguard-schema-id/output/step-14-CONSUME.txt: -------------------------------------------------------------------------------- 1 | jq: parse error: Invalid numeric literal at line 1, column 6 2 | [2024-04-10 03:17:24,912] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) 3 | org.apache.kafka.common.errors.TimeoutException 4 | Processed a total of 1 messages 5 | -------------------------------------------------------------------------------- /safeguard-schema-id/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /safeguard-schema-id/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic users 9 | -------------------------------------------------------------------------------- /safeguard-schema-id/step-07-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /safeguard-schema-id/step-08-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-08-schema-id.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/schema-id" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-08-schema-id.json | jq 10 | -------------------------------------------------------------------------------- /safeguard-schema-id/step-08-schema-id.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.safeguard.TopicRequiredSchemaIdPolicyPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "topic" : "users", 6 | "schemaIdRequired" : true 7 | } 8 | } -------------------------------------------------------------------------------- /safeguard-schema-id/step-09-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /safeguard-schema-id/step-10-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"msg":"hello world"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --topic users -------------------------------------------------------------------------------- /safeguard-schema-id/step-11-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic users \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /safeguard-schema-id/step-13-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl --silent http://localhost:8081/subjects/ | jq -------------------------------------------------------------------------------- /safeguard-schema-id/step-14-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic users \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /safeguard-schema-id/step-15-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /safeguard-schema-id/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ4MDc5Mn0.4EvcLSL91ym532MMdXPGE1OFDAYdzJ-UMoxLcNkXwRM'; 6 | 7 | -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-avro/output/step-16-SH.txt: -------------------------------------------------------------------------------- 1 | [1,2] -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-avro/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-avro/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic topic-avro 9 | -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-avro/step-08-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl -s \ 3 | http://localhost:8081/subjects/topic-avro/versions \ 4 | -X POST \ 5 | -H "Content-Type: application/vnd.schemaregistry.v1+json" \ 6 | --data "{\"schemaType\": \"AVRO\", \"schema\": $(cat user-schema.avsc | jq tostring)}" -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-avro/step-10-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat invalid-payload.json | jq -c | \ 3 | kafka-avro-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --topic topic-avro \ 7 | --property schema.registry.url=http://localhost:8081 \ 8 | --property value.schema.id=1 -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-avro/step-11-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-avro-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic topic-avro \ 6 | --from-beginning \ 7 | --timeout-ms 3000 -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-avro/step-13-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-avro/step-15-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl -s \ 3 | http://localhost:8081/subjects/topic-avro/versions \ 4 | -X POST \ 5 | -H "Content-Type: application/vnd.schemaregistry.v1+json" \ 6 | --data "{\"schemaType\": \"AVRO\", \"schema\": $(cat user-schema-with-validation-rules.avsc | jq tostring)}" -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-avro/step-16-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl -s http://localhost:8081/subjects/topic-avro/versions -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-avro/step-17-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat invalid-payload.json | jq -c | \ 3 | kafka-avro-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --topic topic-avro \ 7 | --property schema.registry.url=http://localhost:8081 \ 8 | --property value.schema.id=2 -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-avro/step-19-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat valid-payload.json | jq -c | \ 3 | kafka-avro-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --topic topic-avro \ 7 | --property schema.registry.url=http://localhost:8081 \ 8 | --property value.schema.id=2 -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-avro/step-20-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-avro-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic topic-avro \ 6 | --from-beginning \ 7 | --timeout-ms 3000 -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-avro/step-21-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-json/output/step-11-SH.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/safeguard-validate-schema-payload-json/output/step-11-SH.txt -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-json/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-json/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic topic-json-schema 9 | -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-json/step-08-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl -s \ 3 | http://localhost:8081/subjects/topic-json-schema/versions \ 4 | -X POST \ 5 | -H "Content-Type: application/vnd.schemaregistry.v1+json" \ 6 | --data "{\"schemaType\": \"JSON\", \"schema\": $(cat user-schema-with-validation-rules.json | jq tostring)}" -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-json/step-10-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat invalid-payload.json | jq -c | \ 3 | kafka-json-schema-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --topic topic-json-schema \ 7 | --property schema.registry.url=http://localhost:8081 \ 8 | --property value.schema.id=1 -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-json/step-12-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-json-schema-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic topic-json-schema \ 6 | --from-beginning \ 7 | --skip-message-on-error \ 8 | --timeout-ms 3000 -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-json/step-14-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-json/step-17-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat valid-payload.json | jq -c | \ 3 | kafka-json-schema-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --topic topic-json-schema \ 7 | --property schema.registry.url=http://localhost:8081 \ 8 | --property value.schema.id=1 -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-json/step-18-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-json-schema-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic topic-json-schema \ 6 | --from-beginning \ 7 | --skip-message-on-error \ 8 | --timeout-ms 3000 -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-json/step-19-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-proto/output/step-16-SH.txt: -------------------------------------------------------------------------------- 1 | [1,2] -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-proto/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-proto/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic topic-protobuf 9 | -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-proto/step-08-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl -s \ 3 | http://localhost:8081/subjects/topic-protobuf/versions \ 4 | -X POST \ 5 | -H "Content-Type: application/vnd.schemaregistry.v1+json" \ 6 | --data "{\"schemaType\": \"PROTOBUF\", \"schema\": $(cat user-schema.proto | jq -Rs)}" -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-proto/step-10-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat invalid-payload.json | jq -c | \ 3 | kafka-protobuf-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --topic topic-protobuf \ 7 | --property schema.registry.url=http://localhost:8081 \ 8 | --property value.schema.id=1 -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-proto/step-11-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-protobuf-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic topic-protobuf \ 6 | --from-beginning \ 7 | --timeout-ms 3000 -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-proto/step-13-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-proto/step-15-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl -s \ 3 | http://localhost:8081/subjects/topic-protobuf/versions \ 4 | -X POST \ 5 | -H "Content-Type: application/vnd.schemaregistry.v1+json" \ 6 | --data "{\"schemaType\": \"PROTOBUF\", \"schema\": $(cat user-schema-with-validation-rules.proto | jq -Rs)}" -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-proto/step-16-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl -s http://localhost:8081/subjects/topic-protobuf/versions -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-proto/step-17-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat invalid-payload.json | jq -c | \ 3 | kafka-protobuf-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --topic topic-protobuf \ 7 | --property schema.registry.url=http://localhost:8081 \ 8 | --property value.schema.id=2 -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-proto/step-19-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat valid-payload.json | jq -c | \ 3 | kafka-protobuf-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --topic topic-protobuf \ 7 | --property schema.registry.url=http://localhost:8081 \ 8 | --property value.schema.id=2 -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-proto/step-20-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-protobuf-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic topic-protobuf \ 6 | --from-beginning \ 7 | --timeout-ms 3000 -------------------------------------------------------------------------------- /safeguard-validate-schema-payload-proto/step-21-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /safeguard-validate-schema-payload/output/step-13-SH.txt: -------------------------------------------------------------------------------- 1 | nb schemas = 3 2 | -------------------------------------------------------------------------------- /safeguard-validate-schema-payload/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /safeguard-validate-schema-payload/step-08-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /safeguard-validate-schema-payload/step-13-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo nb schemas = $(curl --silent http://localhost:8081/subjects/ | jq 'length') -------------------------------------------------------------------------------- /safeguard-validate-schema-payload/step-19-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /safeguard-validate-schema-payload/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ4MTEzMX0.oTIVpYCzkLuNazy_gMJmO9631LUaFqSGeKNmJJzGVtc'; 6 | 7 | -------------------------------------------------------------------------------- /safeguard/output/step-05-CREATE_VIRTUAL_CLUSTER.txt: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3OTQyOH0.NplDti0SEj4iAdXhugvkH0CBWYlqA_icLAFHnDRTo3M'; 6 | 7 | -------------------------------------------------------------------------------- /safeguard/output/step-13-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic roads. 2 | -------------------------------------------------------------------------------- /safeguard/output/step-16-ALTER_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Completed updating config for topic roads. 2 | -------------------------------------------------------------------------------- /safeguard/output/step-19-PRODUCE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/safeguard/output/step-19-PRODUCE.txt -------------------------------------------------------------------------------- /safeguard/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /safeguard/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic cars 9 | -------------------------------------------------------------------------------- /safeguard/step-08-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic cars \ 6 | --from-beginning \ 7 | --max-messages 3 \ 8 | --timeout-ms 10000 | jq 9 | -------------------------------------------------------------------------------- /safeguard/step-09-DESCRIBE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --describe \ 6 | --topic cars 7 | -------------------------------------------------------------------------------- /safeguard/step-10-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-10-guard-on-create-topic.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/guard-on-create-topic" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-10-guard-on-create-topic.json | jq 10 | -------------------------------------------------------------------------------- /safeguard/step-10-guard-on-create-topic.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.safeguard.CreateTopicPolicyPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "replicationFactor" : { 6 | "min" : 2, 7 | "max" : 2 8 | }, 9 | "numPartition" : { 10 | "min" : 1, 11 | "max" : 3 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /safeguard/step-11-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /safeguard/step-12-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 100 \ 7 | --create --if-not-exists \ 8 | --topic roads 9 | -------------------------------------------------------------------------------- /safeguard/step-13-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 2 \ 6 | --partitions 3 \ 7 | --create --if-not-exists \ 8 | --topic roads 9 | -------------------------------------------------------------------------------- /safeguard/step-14-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-14-guard-on-alter-topic.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/guard-on-alter-topic" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-14-guard-on-alter-topic.json | jq 10 | -------------------------------------------------------------------------------- /safeguard/step-14-guard-on-alter-topic.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.safeguard.AlterTopicConfigPolicyPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "retentionMs" : { 6 | "min" : 86400000, 7 | "max" : 432000000 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /safeguard/step-15-ALTER_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-configs \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --alter \ 6 | --entity-type topics \ 7 | --entity-name roads \ 8 | --add-config retention.ms=5184000000 9 | -------------------------------------------------------------------------------- /safeguard/step-16-ALTER_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-configs \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --alter \ 6 | --entity-type topics \ 7 | --entity-name roads \ 8 | --add-config retention.ms=259200000 9 | -------------------------------------------------------------------------------- /safeguard/step-17-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-17-guard-on-produce.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/guard-on-produce" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-17-guard-on-produce.json | jq 10 | -------------------------------------------------------------------------------- /safeguard/step-17-guard-on-produce.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.safeguard.ProducePolicyPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "acks" : { 6 | "value" : [ -1 ], 7 | "action" : "BLOCK" 8 | }, 9 | "compressions" : { 10 | "value" : [ "NONE", "GZIP" ], 11 | "action" : "BLOCK" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /safeguard/step-18-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"type":"Fiat","color":"red","price":-1}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --request-required-acks 1 \ 7 | --compression-codec snappy \ 8 | --topic cars -------------------------------------------------------------------------------- /safeguard/step-19-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"type":"Fiat","color":"red","price":-1}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --request-required-acks -1 \ 7 | --compression-codec gzip \ 8 | --topic cars -------------------------------------------------------------------------------- /safeguard/step-20-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-20-produce-rate.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/produce-rate" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-20-produce-rate.json | jq 10 | -------------------------------------------------------------------------------- /safeguard/step-20-produce-rate.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.safeguard.ProducerRateLimitingPolicyPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "maximumBytesPerSecond" : 1 6 | } 7 | } -------------------------------------------------------------------------------- /safeguard/step-21-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"type":"Fiat","color":"red","price":-1}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --request-required-acks -1 \ 7 | --compression-codec none \ 8 | --topic cars -------------------------------------------------------------------------------- /safeguard/step-22-AUDITLOG.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --topic _conduktor_gateway_auditlogs \ 5 | --from-beginning \ 6 | --timeout-ms 3000 \| jq 'select(.type=="SAFEGUARD" and .eventData.plugin=="io.conduktor.gateway.interceptor.safeguard.ProducerRateLimitingPolicyPlugin")' 7 | -------------------------------------------------------------------------------- /safeguard/step-23-REMOVE_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request DELETE "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/produce-rate" \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /safeguard/step-24-consumer-group-name-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.safeguard.ConsumerGroupPolicyPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "groupId" : { 6 | "value" : "my-group.*", 7 | "action" : "BLOCK" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /safeguard/step-25-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic cars \ 6 | --from-beginning \ 7 | --timeout-ms 10000 \ 8 | --group group-not-within-policy | jq 9 | -------------------------------------------------------------------------------- /safeguard/step-26-AUDITLOG.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --topic _conduktor_gateway_auditlogs \ 5 | --from-beginning \ 6 | --timeout-ms 3000 \| jq 'select(.type=="SAFEGUARD" and .eventData.plugin=="io.conduktor.gateway.interceptor.safeguard.ConsumerGroupPolicyPlugin")' 7 | -------------------------------------------------------------------------------- /safeguard/step-27-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic cars \ 6 | --from-beginning \ 7 | --timeout-ms 10000 \ 8 | --group my-group-within-policy | jq 9 | -------------------------------------------------------------------------------- /safeguard/step-28-REMOVE_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request DELETE "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/consumer-group-name-policy" \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /safeguard/step-29-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-29-guard-limit-connection.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/guard-limit-connection" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-29-guard-limit-connection.json | jq 10 | -------------------------------------------------------------------------------- /safeguard/step-29-guard-limit-connection.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.safeguard.LimitConnectionPolicyPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "maximumConnectionsPerSecond" : 1, 6 | "action" : "BLOCK" 7 | } 8 | } -------------------------------------------------------------------------------- /safeguard/step-30-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic cars \ 6 | --from-beginning \ 7 | --timeout-ms 10000 \ 8 | --group my-group-id-convention-cars | jq 9 | -------------------------------------------------------------------------------- /safeguard/step-31-AUDITLOG.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --topic _conduktor_gateway_auditlogs \ 5 | --from-beginning \ 6 | --timeout-ms 3000 \| jq 'select(.type=="SAFEGUARD" and .eventData.plugin=="io.conduktor.gateway.interceptor.safeguard.LimitConnectionPolicyPlugin")' 7 | -------------------------------------------------------------------------------- /safeguard/step-32-REMOVE_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request DELETE "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/guard-limit-connection" \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /safeguard/step-33-guard-agressive-auto-commit.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.safeguard.LimitCommitOffsetPolicyPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "maximumCommitsPerMinute" : 1, 6 | "action" : "BLOCK" 7 | } 8 | } -------------------------------------------------------------------------------- /safeguard/step-34-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic cars \ 6 | --from-beginning \ 7 | --timeout-ms 10000 \ 8 | --group group-with-aggressive-autocommit | jq 9 | -------------------------------------------------------------------------------- /safeguard/step-35-AUDITLOG.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --topic _conduktor_gateway_auditlogs \ 5 | --from-beginning \ 6 | --timeout-ms 3000 \| jq 'select(.type=="SAFEGUARD" and .eventData.plugin=="io.conduktor.gateway.interceptor.safeguard.LimitCommitOffsetPolicyPlugin")' 7 | -------------------------------------------------------------------------------- /safeguard/step-36-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /safeguard/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ3OTc0Nn0.A-EHR8Qg-AoxtAea3xUfJTVUv85vLfNr45TfONCaIZY'; 6 | 7 | -------------------------------------------------------------------------------- /sni-routing/client.config: -------------------------------------------------------------------------------- 1 | 2 | security.protocol=SSL 3 | ssl.truststore.location=/clientConfig/truststore.jks 4 | ssl.truststore.password=123456 5 | 6 | -------------------------------------------------------------------------------- /sni-routing/keystore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/sni-routing/keystore.jks -------------------------------------------------------------------------------- /sni-routing/openssl.config: -------------------------------------------------------------------------------- 1 | [req] 2 | distinguished_name=req 3 | [san] 4 | subjectAltName=DNS.1:localhost,DNS.2:gateway-sni.conduktor.local,DNS.3:*.gateway-sni.conduktor.local -------------------------------------------------------------------------------- /sni-routing/san.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/sni-routing/san.p12 -------------------------------------------------------------------------------- /sni-routing/step-05-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /sni-routing/step-06-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose exec kafka-client \ 3 | kafka-topics \ 4 | --bootstrap-server broker-sni-gateway1main1.gateway-sni.conduktor.local:6969 \ 5 | --create \ 6 | --replication-factor 3 \ 7 | --partitions 1 \ 8 | --topic clientTopic \ 9 | --command-config /clientConfig/client.config -------------------------------------------------------------------------------- /sni-routing/step-07-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "Hello world 1" | docker compose exec -i kafka-client \ 3 | kafka-console-producer \ 4 | --bootstrap-server broker-sni-gateway1main1.gateway-sni.conduktor.local:6969 \ 5 | --topic clientTopic \ 6 | --producer.config /clientConfig/client.config -------------------------------------------------------------------------------- /sni-routing/step-08-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "Hello world 2" | docker compose exec -i kafka-client \ 3 | kafka-console-producer \ 4 | --bootstrap-server broker-sni-gateway2main2.gateway-sni.conduktor.local:6969 \ 5 | --topic clientTopic \ 6 | --producer.config /clientConfig/client.config -------------------------------------------------------------------------------- /sni-routing/step-09-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose exec kafka-client \ 3 | kafka-console-consumer \ 4 | --bootstrap-server broker-sni-gateway1main3.gateway-sni.conduktor.local:6969 \ 5 | --topic clientTopic \ 6 | --from-beginning \ 7 | --max-messages 2 \ 8 | --consumer.config /clientConfig/client.config -------------------------------------------------------------------------------- /sni-routing/step-10-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /sni-routing/truststore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/sni-routing/truststore.jks -------------------------------------------------------------------------------- /sql-data-quality-producer/output/step-09-PRODUCE.txt: -------------------------------------------------------------------------------- 1 | [2024-04-10 04:08:36,877] ERROR Error when sending message to topic cars with key: 26 bytes, value: 44 bytes with error: (org.apache.kafka.clients.producer.internals.ErrorLoggingCallback) 2 | org.apache.kafka.common.errors.PolicyViolationException: Request parameters do not satisfy the configured policy: Data quality policy is violated. 3 | -------------------------------------------------------------------------------- /sql-data-quality-producer/output/step-10-PRODUCE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/sql-data-quality-producer/output/step-10-PRODUCE.txt -------------------------------------------------------------------------------- /sql-data-quality-producer/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /sql-data-quality-producer/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic cars 9 | -------------------------------------------------------------------------------- /sql-data-quality-producer/step-07-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-07-cars-quality.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/cars-quality" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-07-cars-quality.json | jq 10 | -------------------------------------------------------------------------------- /sql-data-quality-producer/step-07-cars-quality.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.safeguard.DataQualityProducerPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "statement" : "SELECT * FROM cars WHERE color = 'red' and record.key.year > 2020", 6 | "action" : "BLOCK_WHOLE_BATCH", 7 | "deadLetterTopic" : "dead-letter-topic" 8 | } 9 | } -------------------------------------------------------------------------------- /sql-data-quality-producer/step-08-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"type":"SUV","price":2000,"color":"blue"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --topic cars -------------------------------------------------------------------------------- /sql-data-quality-producer/step-09-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"year":2010,"make":"BMW"}\t{"type":"Sports","price":1000,"color":"red"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --property "parse.key=true" \ 7 | --topic cars -------------------------------------------------------------------------------- /sql-data-quality-producer/step-11-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic cars \ 6 | --from-beginning \ 7 | --max-messages 1 \ 8 | --timeout-ms 10000 \ 9 | --property print.key=true \ 10 | --property print.headers=true | jq 11 | -------------------------------------------------------------------------------- /sql-data-quality-producer/step-12-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:19092,localhost:29093,localhost:29094 \ 4 | --topic dead-letter-topic \ 5 | --from-beginning \ 6 | --max-messages 2 \ 7 | --timeout-ms 10000 \ 8 | --property print.key=true \ 9 | --property print.headers=true | jq 10 | -------------------------------------------------------------------------------- /sql-data-quality-producer/step-13-AUDITLOG.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:19092,localhost:29093,localhost:29094 \ 4 | --topic _conduktor_gateway_auditlogs \ 5 | --from-beginning \ 6 | --timeout-ms 3000 \| jq 'select(.type=="SAFEGUARD" and .eventData.plugin=="io.conduktor.gateway.interceptor.DataQualityProducerInterceptor")' 7 | -------------------------------------------------------------------------------- /sql-data-quality-producer/step-14-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /sql-data-quality-producer/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ4Mzg4N30.ZK4cVj-fRNRFifYUvACMYrCbuL3gDODXZKFWwOCrHIc'; 6 | 7 | -------------------------------------------------------------------------------- /sql-topic-schema-registry/output/step-06-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic cars. 2 | -------------------------------------------------------------------------------- /sql-topic-schema-registry/output/step-07-LIST_TOPICS.txt: -------------------------------------------------------------------------------- 1 | cars 2 | -------------------------------------------------------------------------------- /sql-topic-schema-registry/output/step-09-SH.txt: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Sports", 3 | "price": 75, 4 | "color": "blue" 5 | } 6 | { 7 | "type": "SUV", 8 | "price": 55, 9 | "color": "red" 10 | } 11 | -------------------------------------------------------------------------------- /sql-topic-schema-registry/output/step-10-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic red-cars. 2 | -------------------------------------------------------------------------------- /sql-topic-schema-registry/output/step-13-SH.txt: -------------------------------------------------------------------------------- 1 | { 2 | "type": "SUV", 3 | "price": 55, 4 | "color": "red" 5 | } 6 | -------------------------------------------------------------------------------- /sql-topic-schema-registry/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /sql-topic-schema-registry/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic cars 9 | -------------------------------------------------------------------------------- /sql-topic-schema-registry/step-07-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /sql-topic-schema-registry/step-09-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-avro-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic cars \ 6 | --property schema.registry.url=http://localhost:8081 \ 7 | --from-beginning \ 8 | --max-messages 2 2>&1 | grep "{" | jq -------------------------------------------------------------------------------- /sql-topic-schema-registry/step-10-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic red-cars 9 | -------------------------------------------------------------------------------- /sql-topic-schema-registry/step-11-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-11-red-cars.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/red-cars" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-11-red-cars.json | jq 10 | -------------------------------------------------------------------------------- /sql-topic-schema-registry/step-11-red-cars.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.VirtualSqlTopicPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "virtualTopic" : "red-cars", 6 | "statement" : "SELECT * FROM cars WHERE color = 'red'", 7 | "schemaRegistryConfig" : { 8 | "host" : "http://schema-registry:8081" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /sql-topic-schema-registry/step-12-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /sql-topic-schema-registry/step-13-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-avro-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic red-cars \ 6 | --property schema.registry.url=http://localhost:8081 \ 7 | --from-beginning \ 8 | --max-messages 1 2>&1 | grep "{" | jq -------------------------------------------------------------------------------- /sql-topic-schema-registry/step-14-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /sql-topic-schema-registry/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ4NDE4NH0.WQPz3yvnXnRIVvFbN8sbxw1C4NS3lXjk6f5NSK2RDGs'; 6 | 7 | -------------------------------------------------------------------------------- /sql-topics-in-gateway-security-mode/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /sql-topics-in-gateway-security-mode/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config passthrough-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic cars 9 | -------------------------------------------------------------------------------- /sql-topics-in-gateway-security-mode/step-08-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config passthrough-sa.properties \ 5 | --topic cars \ 6 | --from-beginning \ 7 | --max-messages 2 \ 8 | --timeout-ms 10000 | jq 9 | -------------------------------------------------------------------------------- /sql-topics-in-gateway-security-mode/step-09-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-09-red-cars.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/passthrough/interceptor/red-cars" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-09-red-cars.json | jq 10 | -------------------------------------------------------------------------------- /sql-topics-in-gateway-security-mode/step-09-red-cars.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.VirtualSqlTopicPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "virtualTopic" : "red-cars", 6 | "statement" : "SELECT type, price as money FROM cars WHERE color = 'red'" 7 | } 8 | } -------------------------------------------------------------------------------- /sql-topics-in-gateway-security-mode/step-10-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/passthrough' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /sql-topics-in-gateway-security-mode/step-11-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config passthrough-sa.properties \ 5 | --topic red-cars \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /sql-topics-in-gateway-security-mode/step-13-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /sql-topics-in-vcluster-mode/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /sql-topics-in-vcluster-mode/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 1 \ 7 | --create --if-not-exists \ 8 | --topic cars 9 | -------------------------------------------------------------------------------- /sql-topics-in-vcluster-mode/step-08-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic cars \ 6 | --from-beginning \ 7 | --max-messages 2 \ 8 | --timeout-ms 10000 | jq 9 | -------------------------------------------------------------------------------- /sql-topics-in-vcluster-mode/step-09-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-09-red-cars.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/teamA/interceptor/red-cars" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-09-red-cars.json | jq 10 | -------------------------------------------------------------------------------- /sql-topics-in-vcluster-mode/step-09-red-cars.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.VirtualSqlTopicPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "virtualTopic" : "red-cars", 6 | "statement" : "SELECT type, price as money FROM cars WHERE color = 'red'" 7 | } 8 | } -------------------------------------------------------------------------------- /sql-topics-in-vcluster-mode/step-10-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/teamA' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /sql-topics-in-vcluster-mode/step-11-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic red-cars \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /sql-topics-in-vcluster-mode/step-13-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /sql-topics-in-vcluster-mode/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ4NTQ5N30.lM2UIfHWpjc-ViS1abtIcvejgStpNHO-Q7C48zK8w7w'; 6 | 7 | -------------------------------------------------------------------------------- /sql-topics/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /sql-topics/step-05-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --replication-factor 1 \ 5 | --partitions 1 \ 6 | --create --if-not-exists \ 7 | --topic cars 8 | -------------------------------------------------------------------------------- /sql-topics/step-06-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"type":"Sports","price":75,"color":"blue"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --topic cars 6 | 7 | echo '{"type":"SUV","price":55,"color":"red"}' | \ 8 | kafka-console-producer \ 9 | --bootstrap-server localhost:6969 \ 10 | --topic cars -------------------------------------------------------------------------------- /sql-topics/step-07-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --topic cars \ 5 | --from-beginning \ 6 | --max-messages 2 \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /sql-topics/step-08-ADD_INTERCEPTOR.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-08-red-cars.json | jq 3 | 4 | curl \ 5 | --request POST "http://localhost:8888/admin/interceptors/v1/vcluster/passthrough/interceptor/red-cars" \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data @step-08-red-cars.json | jq 10 | -------------------------------------------------------------------------------- /sql-topics/step-08-red-cars.json: -------------------------------------------------------------------------------- 1 | { 2 | "pluginClass" : "io.conduktor.gateway.interceptor.VirtualSqlTopicPlugin", 3 | "priority" : 100, 4 | "config" : { 5 | "virtualTopic" : "red-cars", 6 | "statement" : "SELECT type, price as money FROM cars WHERE color = 'red'" 7 | } 8 | } -------------------------------------------------------------------------------- /sql-topics/step-09-LIST_INTERCEPTORS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | curl \ 3 | --request GET 'http://localhost:8888/admin/interceptors/v1/vcluster/passthrough' \ 4 | --header 'Content-Type: application/json' \ 5 | --user 'admin:conduktor' \ 6 | --silent | jq 7 | -------------------------------------------------------------------------------- /sql-topics/step-10-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --topic red-cars \ 5 | --from-beginning \ 6 | --timeout-ms 10000 | jq 7 | -------------------------------------------------------------------------------- /sql-topics/step-12-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /ssl-and-user-mapping/keystore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/ssl-and-user-mapping/keystore.jks -------------------------------------------------------------------------------- /ssl-and-user-mapping/openssl.config: -------------------------------------------------------------------------------- 1 | [req] 2 | distinguished_name=req 3 | [san] 4 | subjectAltName=DNS.1:localhost,DNS.2:host.docker.internal -------------------------------------------------------------------------------- /ssl-and-user-mapping/output/step-06-ADD_USER_MAPPING.txt: -------------------------------------------------------------------------------- 1 | { 2 | "username": "CN=username" 3 | } 4 | { 5 | "message": "User mapping is created" 6 | } 7 | -------------------------------------------------------------------------------- /ssl-and-user-mapping/output/step-07-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic foo. 2 | -------------------------------------------------------------------------------- /ssl-and-user-mapping/san.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/ssl-and-user-mapping/san.p12 -------------------------------------------------------------------------------- /ssl-and-user-mapping/step-05-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /ssl-and-user-mapping/step-06-ADD_USER_MAPPING.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-06-user-mapping.json | jq 3 | 4 | curl \ 5 | --request POST 'http://localhost:8888/admin/userMappings/v1' \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data "@step-06-user-mapping.json" | jq 10 | -------------------------------------------------------------------------------- /ssl-and-user-mapping/step-06-user-mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "username" : "CN=username" 3 | } -------------------------------------------------------------------------------- /ssl-and-user-mapping/step-07-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config client.config \ 5 | --replication-factor 1 \ 6 | --partitions 10 \ 7 | --create --if-not-exists \ 8 | --topic foo 9 | -------------------------------------------------------------------------------- /ssl-and-user-mapping/step-08-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config client.config \ 5 | --list 6 | -------------------------------------------------------------------------------- /ssl-and-user-mapping/step-09-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --list 5 | -------------------------------------------------------------------------------- /ssl-and-user-mapping/step-10-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /ssl-and-user-mapping/truststore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/ssl-and-user-mapping/truststore.jks -------------------------------------------------------------------------------- /ssl/keystore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/ssl/keystore.jks -------------------------------------------------------------------------------- /ssl/openssl.config: -------------------------------------------------------------------------------- 1 | [req] 2 | distinguished_name=req 3 | [san] 4 | subjectAltName=DNS.1:localhost,DNS.2:host.docker.internal -------------------------------------------------------------------------------- /ssl/output/step-06-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic foo. 2 | -------------------------------------------------------------------------------- /ssl/san.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/ssl/san.p12 -------------------------------------------------------------------------------- /ssl/step-05-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /ssl/step-06-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config client.config \ 5 | --replication-factor 1 \ 6 | --partitions 10 \ 7 | --create --if-not-exists \ 8 | --topic foo 9 | -------------------------------------------------------------------------------- /ssl/step-07-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config client.config \ 5 | --list 6 | -------------------------------------------------------------------------------- /ssl/step-08-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --list 5 | -------------------------------------------------------------------------------- /ssl/step-09-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /ssl/truststore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conduktor/conduktor-gateway-demos/1b186792072ac084721385adfc814682ec08b317/ssl/truststore.jks -------------------------------------------------------------------------------- /throughput/output/step-05-CREATE_TOPICS.txt: -------------------------------------------------------------------------------- 1 | Created topic physical-kafka. 2 | -------------------------------------------------------------------------------- /throughput/output/step-07-CREATE_VIRTUAL_CLUSTER.txt: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ4NjE5N30.sWSMYhOOKLf7wLou5Gce9IeAlna8LUuqMmpi-3CxIPc'; 6 | 7 | -------------------------------------------------------------------------------- /throughput/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /throughput/step-05-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --replication-factor 1 \ 5 | --partitions 10 \ 6 | --create --if-not-exists \ 7 | --topic physical-kafka 8 | -------------------------------------------------------------------------------- /throughput/step-06-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-producer-perf-test \ 3 | --topic physical-kafka \ 4 | --throughput -1 \ 5 | --num-records 2500000 \ 6 | --record-size 255 \ 7 | --producer-props bootstrap.servers=localhost:19092,localhost:19093,localhost:19094 -------------------------------------------------------------------------------- /throughput/step-08-CREATE_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --replication-factor 1 \ 6 | --partitions 10 \ 7 | --create --if-not-exists \ 8 | --topic via-gateway 9 | -------------------------------------------------------------------------------- /throughput/step-09-SH.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-producer-perf-test \ 3 | --topic via-gateway \ 4 | --throughput -1 \ 5 | --num-records 2500000 \ 6 | --record-size 255 \ 7 | --producer-props bootstrap.servers=localhost:6969 \ 8 | --producer.config teamA-sa.properties -------------------------------------------------------------------------------- /throughput/step-10-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /throughput/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ4NjUyN30.TZo1UHNdtxH6KG8foK2Rf7S7xEAJ1Rd-5vOGGhfFi9M'; 6 | 7 | -------------------------------------------------------------------------------- /topic-concentration/output/step-17-CONSUME.txt: -------------------------------------------------------------------------------- 1 | [2024-04-10 04:59:32,817] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) 2 | org.apache.kafka.common.errors.TimeoutException 3 | Processed a total of 1 messages 4 | { 5 | "msg": "hello world" 6 | } 7 | -------------------------------------------------------------------------------- /topic-concentration/step-04-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose up --detach --wait -------------------------------------------------------------------------------- /topic-concentration/step-07-CREATE_CONCENTRATION_RULE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat step-07-concentration-rule.json | jq 3 | 4 | curl \ 5 | --request POST 'http://localhost:8888/admin/vclusters/v1/vcluster/teamA/concentration-rules' \ 6 | --header 'Content-Type: application/json' \ 7 | --user 'admin:conduktor' \ 8 | --silent \ 9 | --data "@step-07-concentration-rule.json" | jq 10 | -------------------------------------------------------------------------------- /topic-concentration/step-07-concentration-rule.json: -------------------------------------------------------------------------------- 1 | { 2 | "clusterId" : "main", 3 | "physicalTopicName" : "hold_many_concentrated_topics", 4 | "physicalTopicCompactedName" : "hold_many_concentrated_topics_compacted", 5 | "physicalTopicCompactedDeletedName" : "hold_many_concentrated_topics_compacted_deleted", 6 | "pattern" : "concentrated-.*" 7 | } -------------------------------------------------------------------------------- /topic-concentration/step-09-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /topic-concentration/step-10-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:19092,localhost:19093,localhost:19094 \ 4 | --list 5 | -------------------------------------------------------------------------------- /topic-concentration/step-12-LIST_TOPICS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-topics \ 3 | --bootstrap-server localhost:6969 \ 4 | --command-config teamA-sa.properties \ 5 | --list 6 | -------------------------------------------------------------------------------- /topic-concentration/step-13-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"type": "Sports", "price": 75, "color": "blue"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --topic concentrated-topic-with-10-partitions -------------------------------------------------------------------------------- /topic-concentration/step-14-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic concentrated-topic-with-10-partitions \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /topic-concentration/step-15-PRODUCE.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo '{"msg":"hello world"}' | \ 3 | kafka-console-producer \ 4 | --bootstrap-server localhost:6969 \ 5 | --producer.config teamA-sa.properties \ 6 | --topic concentrated-topic-with-100-partitions -------------------------------------------------------------------------------- /topic-concentration/step-16-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic concentrated-topic-with-100-partitions \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /topic-concentration/step-17-CONSUME.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | kafka-console-consumer \ 3 | --bootstrap-server localhost:6969 \ 4 | --consumer.config teamA-sa.properties \ 5 | --topic concentrated-topic-with-100-partitions \ 6 | --from-beginning \ 7 | --timeout-ms 10000 | jq 8 | -------------------------------------------------------------------------------- /topic-concentration/step-18-DOCKER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker compose down --volumes -------------------------------------------------------------------------------- /topic-concentration/teamA-sa.properties: -------------------------------------------------------------------------------- 1 | 2 | bootstrap.servers=localhost:6969 3 | security.protocol=SASL_PLAINTEXT 4 | sasl.mechanism=PLAIN 5 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='sa' password='eyJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InNhIiwidmNsdXN0ZXIiOiJ0ZWFtQSIsImV4cCI6MTcyMDQ4Nzg2MX0.Xv1FS9czhr9K5lvZynXr2C-nw2oq4zTKSN4w-5_LDyo'; 6 | 7 | --------------------------------------------------------------------------------