├── .gitattributes ├── .github ├── dependabot.yml └── workflows │ ├── cross-maven.yml │ ├── maven.yml │ └── sanity-check.yml ├── .gitignore ├── .mvn └── wrapper │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── README.md ├── debezium-server-bom └── pom.xml ├── debezium-server-core ├── pom.xml └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── debezium │ │ │ └── server │ │ │ ├── Any.java │ │ │ ├── BaseChangeConsumer.java │ │ │ ├── ConnectorLifecycle.java │ │ │ ├── CustomConsumerBuilder.java │ │ │ ├── DebeziumMetrics.java │ │ │ ├── DebeziumServer.java │ │ │ ├── DebeziumServerConfig.java │ │ │ ├── Main.java │ │ │ ├── StreamNameMapper.java │ │ │ ├── events │ │ │ ├── ConnectorCompletedEvent.java │ │ │ ├── ConnectorStartedEvent.java │ │ │ ├── ConnectorStoppedEvent.java │ │ │ ├── PollingStartedEvent.java │ │ │ ├── PollingStoppedEvent.java │ │ │ ├── TaskStartedEvent.java │ │ │ └── TaskStoppedEvent.java │ │ │ └── rest │ │ │ ├── DebeziumServerAPI.java │ │ │ └── signal │ │ │ ├── DSSignal.java │ │ │ └── SignalResource.java │ └── resources │ │ ├── META-INF │ │ └── beans.xml │ │ ├── application.properties │ │ └── default_banner.txt │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ ├── DebeziumServerApicurioProfile.java │ │ ├── DebeziumServerConfigProvidersIT.java │ │ ├── DebeziumServerConnectFormatIT.java │ │ ├── DebeziumServerConnectFormatProfile.java │ │ ├── DebeziumServerFileConfigProviderProfile.java │ │ ├── DebeziumServerIT.java │ │ ├── DebeziumServerSchemaRegistryProfile.java │ │ ├── DebeziumServerTest.java │ │ ├── DebeziumServerWithApicurioIT.java │ │ ├── DebeziumServerWithSchemaRegistryIT.java │ │ ├── Images.java │ │ ├── TestConfigSource.java │ │ └── TestConsumer.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.eclipse.microprofile.config.spi.ConfigSource │ ├── application.properties │ ├── logback-test.xml │ ├── secrets_test.txt │ └── testcontainers.properties ├── debezium-server-dist ├── pom.xml └── src │ ├── main │ └── resources │ │ ├── assemblies │ │ ├── server-distribution-prod.xml │ │ └── server-distribution.xml │ │ └── distro │ │ ├── config │ │ ├── application.properties.cassandra.redis.example │ │ ├── application.properties.example │ │ ├── application.yaml.example │ │ ├── lib │ │ │ └── cassandra │ │ │ │ └── driver.conf │ │ └── metrics.yml │ │ ├── connectors │ │ ├── debezium-connector-cassandra-3 │ │ │ ├── extra_class_path.sh │ │ │ └── jdk_java_options.sh │ │ ├── debezium-connector-cassandra-4 │ │ │ ├── extra_class_path.sh │ │ │ └── jdk_java_options.sh │ │ ├── debezium-connector-cassandra-5 │ │ │ ├── extra_class_path.sh │ │ │ └── jdk_java_options.sh │ │ └── debezium-connector-dse │ │ │ ├── extra_class_path.sh │ │ │ └── jdk_java_options.sh │ │ ├── jmx │ │ ├── enable_jmx.sh │ │ ├── jmxremote.access │ │ └── jmxremote.password │ │ ├── lib_metrics │ │ └── enable_exporter.sh │ │ ├── run.bat │ │ └── run.sh │ └── test │ └── resources │ └── redis │ ├── application.properties │ └── docker-compose.yml ├── debezium-server-eventhubs ├── pom.xml └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── debezium │ │ │ └── server │ │ │ └── eventhubs │ │ │ ├── BatchManager.java │ │ │ ├── EventDataBatchProxy.java │ │ │ └── EventHubsChangeConsumer.java │ └── resources │ │ └── META-INF │ │ └── beans.xml │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ └── eventhubs │ │ ├── EventHubsTestConfigSource.java │ │ ├── EventHubsWithPartitionKeyIT.java │ │ ├── EventHubsWithPartitionKeyProfile.java │ │ ├── EventHubsWithPartitionRouterIT.java │ │ ├── EventHubsWithPartitionRouterProfile.java │ │ ├── EventHubsWithStaticPartitionIdIT.java │ │ └── EventHubsWithStaticPartitionIdProfile.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.eclipse.microprofile.config.spi.ConfigSource │ └── logback-test.xml ├── debezium-server-http ├── pom.xml └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── debezium │ │ │ └── server │ │ │ └── http │ │ │ ├── Authenticator.java │ │ │ ├── HttpChangeConsumer.java │ │ │ ├── jwt │ │ │ ├── JWTAuthenticator.java │ │ │ ├── JWTAuthenticatorBuilder.java │ │ │ ├── JWTAuthorizationInitialRequest.java │ │ │ ├── JWTAuthorizationRefreshRequest.java │ │ │ └── JWTAuthorizationResponse.java │ │ │ └── webhooks │ │ │ ├── StandardWebhooksAuthenticator.java │ │ │ └── StandardWebhooksAuthenticatorBuilder.java │ └── resources │ │ └── META-INF │ │ └── beans.xml │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ └── http │ │ ├── HttpChangeConsumerTest.java │ │ ├── HttpIT.java │ │ ├── HttpTestConfigSource.java │ │ ├── HttpTestResourceLifecycleManager.java │ │ ├── jwt │ │ ├── JWTAuthenticatorBuilderTest.java │ │ └── JWTAuthenticatorTest.java │ │ └── webhooks │ │ ├── StandardWebhooksAuthenticatorBuilderTest.java │ │ └── StandardWebhooksAuthenticatorTest.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.eclipse.microprofile.config.spi.ConfigSource │ └── logback-test.xml ├── debezium-server-infinispan ├── pom.xml └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── debezium │ │ │ └── server │ │ │ └── infinispan │ │ │ └── InfinispanSinkConsumer.java │ └── resources │ │ └── META-INF │ │ └── beans.xml │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ └── infinispan │ │ ├── InfinispanSinkConsumerIT.java │ │ ├── InfinispanTestConfigSource.java │ │ └── InfinispanTestResourceLifecycleManager.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.eclipse.microprofile.config.spi.ConfigSource │ └── infinispan-local.xml ├── debezium-server-instructlab ├── README.md ├── pom.xml └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── debezium │ │ │ └── server │ │ │ └── instructlab │ │ │ ├── InstructLabSinkConsumer.java │ │ │ └── QnaFile.java │ └── resources │ │ └── META-INF │ │ └── beans.xml │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ └── instructlab │ │ ├── InstructLabIT.java │ │ ├── InstructLabTestConfigSource.java │ │ ├── MappingValueTest.java │ │ └── QnaFileTest.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.eclipse.microprofile.config.spi.ConfigSource │ └── logback-test.xml ├── debezium-server-kafka ├── pom.xml └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── debezium │ │ │ └── server │ │ │ └── kafka │ │ │ └── KafkaChangeConsumer.java │ └── resources │ │ └── META-INF │ │ └── beans.xml │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ └── kafka │ │ ├── KafkaIT.java │ │ ├── KafkaTestConfigSource.java │ │ └── KafkaTestResourceLifecycleManager.java │ └── resources │ └── META-INF │ └── services │ └── org.eclipse.microprofile.config.spi.ConfigSource ├── debezium-server-kinesis ├── pom.xml └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── debezium │ │ │ └── server │ │ │ └── kinesis │ │ │ └── KinesisChangeConsumer.java │ └── resources │ │ └── META-INF │ │ └── beans.xml │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ └── kinesis │ │ ├── KinesisIT.java │ │ ├── KinesisTestConfigSource.java │ │ └── KinesisUnitTest.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.eclipse.microprofile.config.spi.ConfigSource │ └── logback-test.xml ├── debezium-server-milvus ├── pom.xml └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── debezium │ │ │ └── server │ │ │ └── milvus │ │ │ ├── MilvusChangeConsumer.java │ │ │ └── MilvusSchema.java │ └── resources │ │ └── META-INF │ │ └── beans.xml │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ └── milvus │ │ ├── MilvusIT.java │ │ ├── MilvusSchemaTest.java │ │ ├── MilvusTestConfigSource.java │ │ ├── MilvusTestResourceLifecycleManager.java │ │ └── VectorPostgresTestResourceLifecycleManager.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.eclipse.microprofile.config.spi.ConfigSource │ └── logback-test.xml ├── debezium-server-nats-jetstream ├── pom.xml └── src │ ├── main │ └── java │ │ └── io │ │ └── debezium │ │ └── server │ │ └── nats │ │ └── jetstream │ │ └── NatsJetStreamChangeConsumer.java │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ └── nats │ │ └── jetstream │ │ ├── NatsJetStreamIT.java │ │ ├── NatsJetStreamTestConfigSource.java │ │ └── NatsJetStreamTestResourceLifecycleManager.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.eclipse.microprofile.config.spi.ConfigSource │ └── testcontainers.properties ├── debezium-server-nats-streaming ├── pom.xml └── src │ ├── main │ └── java │ │ └── io │ │ └── debezium │ │ └── server │ │ └── nats │ │ └── streaming │ │ └── NatsStreamingChangeConsumer.java │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ └── nats │ │ └── streaming │ │ ├── NatsStreamingIT.java │ │ ├── NatsStreamingTestConfigSource.java │ │ └── NatsStreamingTestResourceLifecycleManager.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.eclipse.microprofile.config.spi.ConfigSource │ └── testcontainers.properties ├── debezium-server-pravega ├── pom.xml └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── debezium │ │ │ └── server │ │ │ └── pravega │ │ │ ├── PravegaChangeConsumer.java │ │ │ └── PravegaSink.java │ └── resources │ │ └── META-INF │ │ └── beans.xml │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ └── pravega │ │ ├── PravegaIT.java │ │ └── PravegaTestResource.java │ └── resources │ └── META-INF │ └── services │ └── org.eclipse.microprofile.config.spi.ConfigSource ├── debezium-server-pubsub ├── pom.xml └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── debezium │ │ │ └── server │ │ │ └── pubsub │ │ │ ├── PubSubChangeConsumer.java │ │ │ └── PubSubLiteChangeConsumer.java │ └── resources │ │ └── META-INF │ │ └── beans.xml │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ └── pubsub │ │ ├── PubSubIT.java │ │ ├── PubSubLiteIT.java │ │ ├── PubSubLiteTestConfigSource.java │ │ ├── PubSubTestConfigSource.java │ │ └── PubSubTestResourceLifecycleManager.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.eclipse.microprofile.config.spi.ConfigSource │ └── logback-test.xml ├── debezium-server-pulsar ├── pom.xml └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── debezium │ │ │ └── server │ │ │ └── pulsar │ │ │ └── PulsarChangeConsumer.java │ └── resources │ │ └── META-INF │ │ └── beans.xml │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ └── pulsar │ │ ├── PulsarIT.java │ │ ├── PulsarTestConfigSource.java │ │ └── PulsarTestResourceLifecycleManager.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.eclipse.microprofile.config.spi.ConfigSource │ ├── docker │ └── conf │ │ ├── bkenv.sh │ │ ├── bookkeeper.conf │ │ ├── broker.conf │ │ ├── client.conf │ │ ├── discovery.conf │ │ ├── filesystem_offload_core_site.xml │ │ ├── functions-logging │ │ ├── console_logging_config.ini │ │ └── logging_config.ini │ │ ├── functions_worker.yml │ │ ├── global_zookeeper.conf │ │ ├── log4j2-scripts │ │ └── filter.js │ │ ├── log4j2.yaml │ │ ├── presto │ │ ├── catalog │ │ │ └── pulsar.properties │ │ ├── config.properties │ │ ├── jvm.config │ │ └── log.properties │ │ ├── proxy.conf │ │ ├── pulsar_env.sh │ │ ├── pulsar_tools_env.sh │ │ ├── schema_example.conf │ │ ├── standalone.conf │ │ ├── websocket.conf │ │ └── zookeeper.conf │ └── logback-test.xml ├── debezium-server-qdrant ├── pom.xml └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── debezium │ │ │ └── server │ │ │ └── qdrant │ │ │ ├── QdrantChangeConsumer.java │ │ │ └── QdrantMessageFactory.java │ └── resources │ │ └── META-INF │ │ └── beans.xml │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ └── qdrant │ │ ├── QdrantIT.java │ │ ├── QdrantMessageFactoryTest.java │ │ ├── QdrantTestConfigSource.java │ │ ├── QdrantTestResourceLifecycleManager.java │ │ └── VectorPostgresTestResourceLifecycleManager.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.eclipse.microprofile.config.spi.ConfigSource │ └── logback-test.xml ├── debezium-server-rabbitmq ├── pom.xml └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── debezium │ │ │ └── server │ │ │ └── rabbitmq │ │ │ ├── RabbitMqStreamChangeConsumer.java │ │ │ └── RabbitMqStreamNativeChangeConsumer.java │ └── resources │ │ └── META-INF │ │ └── beans.xml │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ └── rabbitmq │ │ ├── RabbitMqContainer.java │ │ ├── RabbitMqIT.java │ │ ├── RabbitMqStreamChangeConsumerTest.java │ │ ├── RabbitMqStreamIT.java │ │ ├── RabbitMqStreamTestResourceLifecycleManager.java │ │ ├── RabbitMqTestConfigSource.java │ │ └── RabbitMqTestResourceLifecycleManager.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.eclipse.microprofile.config.spi.ConfigSource │ └── logback-test.xml ├── debezium-server-redis ├── pom.xml └── src │ ├── main │ └── java │ │ └── io │ │ └── debezium │ │ └── server │ │ └── redis │ │ ├── RedisMemoryThreshold.java │ │ ├── RedisOffsetBackingStore.java │ │ ├── RedisSchemaHistory.java │ │ ├── RedisStreamChangeConsumer.java │ │ └── RedisStreamChangeConsumerConfig.java │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ └── redis │ │ ├── RedisMemoryThresholdTest.java │ │ ├── RedisOffsetIT.java │ │ ├── RedisOffsetTestProfile.java │ │ ├── RedisSSLOptionsStreamIT.java │ │ ├── RedisSSLOptionsStreamTestProfile.java │ │ ├── RedisSSLStreamIT.java │ │ ├── RedisSSLStreamTestProfile.java │ │ ├── RedisSSLTestResourceLifecycleManager.java │ │ ├── RedisSchemaHistoryIT.java │ │ ├── RedisSchemaHistoryTestProfile.java │ │ ├── RedisStreamHeartbeatDisabledIT.java │ │ ├── RedisStreamHeartbeatDisabledTestProfile.java │ │ ├── RedisStreamHeartbeatSkipIT.java │ │ ├── RedisStreamHeartbeatTestProfile.java │ │ ├── RedisStreamIT.java │ │ ├── RedisStreamMemoryThresholdIT.java │ │ ├── RedisStreamMemoryThresholdTestProfile.java │ │ ├── RedisStreamMessageIT.java │ │ ├── RedisStreamMessageTestProfile.java │ │ ├── RedisStreamTestProfile.java │ │ ├── RedisTestResourceLifecycleManager.java │ │ └── TestUtils.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.eclipse.microprofile.config.spi.ConfigSource │ ├── ssl │ ├── ca.crt │ ├── client-keystore.p12 │ ├── client-truststore.p12 │ ├── keys.p12 │ ├── redis.crt │ └── redis.key │ └── testcontainers.properties ├── debezium-server-rocketmq ├── pom.xml └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── debezium │ │ │ └── server │ │ │ └── rocketmq │ │ │ └── RocketMqChangeConsumer.java │ └── resources │ │ └── META-INF │ │ └── beans.xml │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ └── rocketmq │ │ ├── RocketMqContainer.java │ │ ├── RocketMqIT.java │ │ ├── RocketMqTestConfigSource.java │ │ └── RocketMqTestResourceLifecycleManager.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.eclipse.microprofile.config.spi.ConfigSource │ └── logback-test.xml ├── debezium-server-sqs ├── pom.xml └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── debezium │ │ │ └── server │ │ │ └── sqs │ │ │ └── SqsChangeConsumer.java │ └── resources │ │ └── META-INF │ │ └── beans.xml │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ └── sqs │ │ ├── SqsIT.java │ │ ├── SqsTestConfigSource.java │ │ └── SqsTestResourceLifecycleManager.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.eclipse.microprofile.config.spi.ConfigSource │ ├── logback-test.xml │ └── testcontainers.properties ├── debezium-system-tests ├── pom.xml └── src │ └── test │ ├── java │ └── io │ │ └── debezium │ │ └── server │ │ ├── DebeziumServerPostgresIT.java │ │ ├── TestConfigSource.java │ │ └── TestConsumer.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.eclipse.microprofile.config.spi.ConfigSource │ ├── application.properties │ ├── logback-test.xml │ └── testcontainers.properties ├── mvnw ├── mvnw.cmd └── pom.xml /.gitattributes: -------------------------------------------------------------------------------- 1 | # Repository specific GIT options 2 | 3 | # Set default handling of line terminators for all non explicitly listed file types: 4 | * text=auto 5 | 6 | # Force LF as internal repository format for all following files; 7 | # this overrides user settings to enforce the *right format* : 8 | *.java text 9 | *.xml text 10 | *.txt text 11 | *.md text 12 | *.html text 13 | *.properties text 14 | *.rb text 15 | *.pot text 16 | *.po text 17 | *.xsd text 18 | *.header text 19 | *.groovy text 20 | *.css text 21 | 22 | # Specify CRLF for batch files 23 | *.bat text eol=crlf 24 | 25 | # Specify we want Java-friendly readable chunk headers for diff: 26 | 27 | *.java diff=java -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Updates for GitHub Actions 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: "weekly" 8 | commit-message: 9 | prefix: "[ci] " 10 | 11 | -------------------------------------------------------------------------------- /.github/workflows/maven.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright Debezium Authors 3 | # 4 | # Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | # 6 | name: Maven CI 7 | 8 | on: 9 | push: 10 | branches: 11 | - main 12 | - 1.* 13 | - 2.* 14 | - 3.* 15 | - 4.* 16 | paths-ignore: 17 | - '*.md' 18 | 19 | jobs: 20 | build: 21 | runs-on: ubuntu-latest 22 | 23 | steps: 24 | - name: Checkout core repository 25 | uses: actions/checkout@v4 26 | with: 27 | repository: debezium/debezium 28 | ref: ${{ github.ref }} 29 | path: core 30 | - name: Checkout Debezium Server 31 | uses: actions/checkout@v4 32 | with: 33 | path: server 34 | - name: Set up JDK 35 | uses: actions/setup-java@v4 36 | with: 37 | distribution: 'temurin' 38 | java-version: 21 39 | - name: Cache Maven packages 40 | uses: actions/cache@v4 41 | with: 42 | path: ~/.m2 43 | key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} 44 | restore-keys: ${{ runner.os }}-m2 45 | - name: Maven build core 46 | run: ./server/mvnw clean install -f core/pom.xml -DskipTests -DskipITs -Dformat.formatter.goal=validate -Dformat.imports.goal=check -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 47 | - name: Maven build Debezium Server 48 | run: ./server/mvnw clean install -fae -f server/pom.xml -Passembly -Dformat.formatter.goal=validate -Dformat.imports.goal=check -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -DskipNonCore 49 | -------------------------------------------------------------------------------- /.github/workflows/sanity-check.yml: -------------------------------------------------------------------------------- 1 | name: Commit message format check 2 | on: [ pull_request ] 3 | 4 | jobs: 5 | build: 6 | name: Commit message 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Checkout repository 10 | uses: actions/checkout@v4 11 | - name: Commit messages in format DBZ-xxx 12 | env: 13 | BASE_REF: ${{ github.base_ref }} 14 | run: | 15 | ! git log --format='format:%s' refs/remotes/origin/$BASE_REF.. | tail -n +2 | grep -Ev '^(DBZ-[[:digit:]]+)|(\[release\])|(\[jenkins-jobs\]) ' 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | activemq-data/ 2 | .idea/ 3 | *.iml 4 | *.ipr 5 | *.iws 6 | .metadata/ 7 | .recommenders/ 8 | .classpath 9 | .project 10 | .cache 11 | .settings/ 12 | .factorypath 13 | .checkstyle 14 | .gradle/ 15 | .vscode/ 16 | build/ 17 | debezium-ddl-parser/gen 18 | deploy/ 19 | target/ 20 | mods/ 21 | *.swp 22 | .tags 23 | epom 24 | log 25 | npm-debug.log 26 | .DS_Store 27 | phantomjsdriver.log 28 | 29 | generated-sources/ 30 | 31 | /state/ 32 | bin/ 33 | gen/ 34 | *.tokens 35 | 36 | jenkins-jobs/docker/rhel_kafka/plugins 37 | jenkins-jobs/docker/artifact-server/plugins 38 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | wrapperVersion=3.3.2 18 | distributionType=bin 19 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.8/apache-maven-3.9.8-bin.zip 20 | wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.3.2/maven-wrapper-3.3.2.jar 21 | -------------------------------------------------------------------------------- /debezium-server-core/src/main/java/io/debezium/server/Any.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server; 7 | 8 | import io.debezium.engine.format.SerializationFormat; 9 | 10 | /** 11 | * A {@link SerializationFormat} defining the undefined serialization type. 12 | */ 13 | class Any implements SerializationFormat { 14 | } 15 | -------------------------------------------------------------------------------- /debezium-server-core/src/main/java/io/debezium/server/CustomConsumerBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server; 7 | 8 | import static java.lang.annotation.ElementType.FIELD; 9 | import static java.lang.annotation.ElementType.METHOD; 10 | import static java.lang.annotation.ElementType.PARAMETER; 11 | import static java.lang.annotation.ElementType.TYPE; 12 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 13 | 14 | import java.lang.annotation.Documented; 15 | import java.lang.annotation.Retention; 16 | import java.lang.annotation.Target; 17 | 18 | import jakarta.inject.Qualifier; 19 | 20 | @Qualifier 21 | @Target({ TYPE, METHOD, PARAMETER, FIELD }) 22 | @Retention(RUNTIME) 23 | @Documented 24 | public @interface CustomConsumerBuilder { 25 | } 26 | -------------------------------------------------------------------------------- /debezium-server-core/src/main/java/io/debezium/server/DebeziumServerConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server; 7 | 8 | import io.smallrye.config.ConfigMapping; 9 | import io.smallrye.config.WithDefault; 10 | 11 | @ConfigMapping(prefix = "debezium") 12 | public interface DebeziumServerConfig { 13 | Api api(); 14 | 15 | interface Api { 16 | @WithDefault("false") 17 | boolean enabled(); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /debezium-server-core/src/main/java/io/debezium/server/Main.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server; 7 | 8 | import io.quarkus.runtime.Quarkus; 9 | import io.quarkus.runtime.annotations.QuarkusMain; 10 | 11 | @QuarkusMain 12 | public class Main { 13 | 14 | public static void main(String... args) { 15 | Quarkus.run(args); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /debezium-server-core/src/main/java/io/debezium/server/StreamNameMapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server; 7 | 8 | /** 9 | * Transforms the name of the record destination to the target stream name. 10 | * 11 | * @author Jiri Pechanec 12 | * 13 | */ 14 | public interface StreamNameMapper { 15 | String map(String topic); 16 | } 17 | -------------------------------------------------------------------------------- /debezium-server-core/src/main/java/io/debezium/server/events/ConnectorCompletedEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.events; 7 | 8 | import java.util.Optional; 9 | 10 | /** 11 | * Fired when the connector was completed. Provides information about completion state, message 12 | * and optional stacktrace in case of error. 13 | * 14 | * @author Jiri Pechanec 15 | * 16 | */ 17 | public class ConnectorCompletedEvent { 18 | 19 | private final boolean success; 20 | private final String message; 21 | private final Throwable error; 22 | 23 | public ConnectorCompletedEvent(boolean success, String message, Throwable error) { 24 | this.success = success; 25 | this.message = message; 26 | this.error = error; 27 | } 28 | 29 | /** 30 | * 31 | * @return true if the connector was completed successfully 32 | */ 33 | public boolean isSuccess() { 34 | return success; 35 | } 36 | 37 | /** 38 | * 39 | * @return message associated with connection completion 40 | */ 41 | public String getMessage() { 42 | return message; 43 | } 44 | 45 | /** 46 | * 47 | * @return optional error in case the connector has not started successfully or was terminated with an error 48 | */ 49 | public Optional getError() { 50 | return Optional.ofNullable(error); 51 | } 52 | 53 | @Override 54 | public String toString() { 55 | return "ConnectorCompletedEvent [success=" + success + ", message=" + message + ", error=" + error + "]"; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /debezium-server-core/src/main/java/io/debezium/server/events/ConnectorStartedEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.events; 7 | 8 | /** 9 | * Fired when the connector is started. The initialization is completed but the execution task 10 | * is not started yet. 11 | * 12 | * @author Jiri Pechanec 13 | * 14 | */ 15 | public class ConnectorStartedEvent { 16 | } 17 | -------------------------------------------------------------------------------- /debezium-server-core/src/main/java/io/debezium/server/events/ConnectorStoppedEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.events; 7 | 8 | /** 9 | * Fired when the connector is stopped but the final execution completion state is not yet determined. 10 | * 11 | * @author Jiri Pechanec 12 | * 13 | */ 14 | public class ConnectorStoppedEvent { 15 | } 16 | -------------------------------------------------------------------------------- /debezium-server-core/src/main/java/io/debezium/server/events/PollingStartedEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.events; 7 | 8 | /** 9 | * Fired right before creating polling loop. 10 | * 11 | * @author vjuranek 12 | */ 13 | public class PollingStartedEvent { 14 | } 15 | -------------------------------------------------------------------------------- /debezium-server-core/src/main/java/io/debezium/server/events/PollingStoppedEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.events; 7 | 8 | /** 9 | * Fired once the polling loop is finished. 10 | * 11 | * @author vjuranek 12 | */ 13 | public class PollingStoppedEvent { 14 | } 15 | -------------------------------------------------------------------------------- /debezium-server-core/src/main/java/io/debezium/server/events/TaskStartedEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.events; 7 | 8 | /** 9 | * Fired right after the connector execution code is started. 10 | * 11 | * @author Jiri Pechanec 12 | * 13 | */ 14 | public class TaskStartedEvent { 15 | } 16 | -------------------------------------------------------------------------------- /debezium-server-core/src/main/java/io/debezium/server/events/TaskStoppedEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.events; 7 | 8 | /** 9 | * Fired right after the connector execution code is stopped. 10 | * 11 | * @author Jiri Pechanec 12 | * 13 | */ 14 | public class TaskStoppedEvent { 15 | } 16 | -------------------------------------------------------------------------------- /debezium-server-core/src/main/java/io/debezium/server/rest/DebeziumServerAPI.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.rest; 7 | 8 | import jakarta.ws.rs.ApplicationPath; 9 | import jakarta.ws.rs.core.Application; 10 | 11 | @ApplicationPath("/api") 12 | public class DebeziumServerAPI extends Application { 13 | } 14 | -------------------------------------------------------------------------------- /debezium-server-core/src/main/java/io/debezium/server/rest/signal/DSSignal.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.rest.signal; 7 | 8 | import java.util.Map; 9 | 10 | public record DSSignal(String id, String type, String data, Map additionalData) { 11 | 12 | } 13 | -------------------------------------------------------------------------------- /debezium-server-core/src/main/java/io/debezium/server/rest/signal/SignalResource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.rest.signal; 7 | 8 | import jakarta.inject.Inject; 9 | import jakarta.validation.constraints.NotNull; 10 | import jakarta.ws.rs.POST; 11 | import jakarta.ws.rs.Path; 12 | import jakarta.ws.rs.core.Response; 13 | 14 | import io.debezium.engine.DebeziumEngine; 15 | import io.debezium.server.DebeziumServer; 16 | import io.debezium.server.DebeziumServerConfig; 17 | 18 | @Path("/signals") 19 | public class SignalResource { 20 | 21 | @Inject 22 | DebeziumServerConfig config; 23 | 24 | @Inject 25 | DebeziumServer server; 26 | 27 | @POST 28 | public Response post(@NotNull DSSignal dsSignal) { 29 | var signaler = server.getSignaler(); 30 | if (signaler == null || !config.api().enabled()) { 31 | return Response.status(Response.Status.SERVICE_UNAVAILABLE).build(); 32 | } 33 | 34 | var signal = toSignal(dsSignal); 35 | signaler.signal(signal); 36 | return Response.accepted().build(); 37 | } 38 | 39 | private DebeziumEngine.Signal toSignal(DSSignal dsSignal) { 40 | return new DebeziumEngine.Signal(dsSignal.id(), dsSignal.type(), dsSignal.data(), dsSignal.additionalData()); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /debezium-server-core/src/main/resources/META-INF/beans.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-core/src/main/resources/META-INF/beans.xml -------------------------------------------------------------------------------- /debezium-server-core/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | debezium.api.enabled=false 2 | # TODO: this should be eventually removed when entire config is mapped 3 | smallrye.config.mapping.validate-unknown=false -------------------------------------------------------------------------------- /debezium-server-core/src/main/resources/default_banner.txt: -------------------------------------------------------------------------------- 1 | __ __ _ 2 | ____/ /___ / /_ ___ ____ (_)__ __ ____ ___ 3 | / __ // _ \ / __ \ / _ \/_ / / // / / // __ `__ \ 4 | / /_/ // __// /_/ // __/ / /_ / // /_/ // / / / / / 5 | \__,_/ \___//_.___/ \___/ /___//_/ \__,_//_/ /_/ /_/ 6 | 7 | 8 | -------------------------------------------------------------------------------- /debezium-server-core/src/test/java/io/debezium/server/DebeziumServerApicurioProfile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server; 7 | 8 | import java.util.Collections; 9 | import java.util.List; 10 | 11 | import io.debezium.testing.testcontainers.ApicurioTestResourceLifeCycleManager; 12 | import io.quarkus.test.junit.QuarkusTestProfile; 13 | 14 | public class DebeziumServerApicurioProfile implements QuarkusTestProfile { 15 | 16 | @Override 17 | public List testResources() { 18 | return Collections.singletonList(new TestResourceEntry(ApicurioTestResourceLifeCycleManager.class)); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /debezium-server-core/src/test/java/io/debezium/server/DebeziumServerConnectFormatProfile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import io.debezium.embedded.Connect; 12 | import io.quarkus.test.junit.QuarkusTestProfile; 13 | 14 | public class DebeziumServerConnectFormatProfile implements QuarkusTestProfile { 15 | 16 | @Override 17 | public Map getConfigOverrides() { 18 | Map config = new HashMap(); 19 | config.put("debezium.format.key", Connect.class.getSimpleName().toLowerCase()); 20 | config.put("debezium.format.value", Connect.class.getSimpleName().toLowerCase()); 21 | config.put("debezium.format.header", Connect.class.getSimpleName().toLowerCase()); 22 | return config; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /debezium-server-core/src/test/java/io/debezium/server/DebeziumServerFileConfigProviderProfile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server; 7 | 8 | import java.net.URL; 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | 12 | import io.quarkus.test.junit.QuarkusTestProfile; 13 | 14 | public class DebeziumServerFileConfigProviderProfile implements QuarkusTestProfile { 15 | 16 | @Override 17 | public Map getConfigOverrides() { 18 | Map config = new HashMap(); 19 | URL secretFile = DebeziumServerFileConfigProviderProfile.class.getClassLoader().getResource("secrets_test.txt"); 20 | 21 | config.put("debezium.source.database.user", "\\${file:" + secretFile.getPath() + ":user}"); 22 | 23 | config.put("debezium.source.config.providers", "file"); 24 | config.put("debezium.source.config.providers.file.class", "org.apache.kafka.common.config.provider.FileConfigProvider"); 25 | 26 | return config; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /debezium-server-core/src/test/java/io/debezium/server/DebeziumServerSchemaRegistryProfile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server; 7 | 8 | import java.util.Collections; 9 | import java.util.List; 10 | 11 | import io.debezium.testing.testcontainers.SchemaRegistryTestResourceLifecycleManager; 12 | import io.quarkus.test.junit.QuarkusTestProfile; 13 | 14 | public class DebeziumServerSchemaRegistryProfile implements QuarkusTestProfile { 15 | 16 | @Override 17 | public List testResources() { 18 | return Collections.singletonList(new TestResourceEntry(SchemaRegistryTestResourceLifecycleManager.class)); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /debezium-server-core/src/test/java/io/debezium/server/DebeziumServerWithSchemaRegistryIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server; 7 | 8 | import static org.assertj.core.api.Assertions.assertThat; 9 | 10 | import java.time.Duration; 11 | 12 | import jakarta.enterprise.event.Observes; 13 | import jakarta.inject.Inject; 14 | 15 | import org.awaitility.Awaitility; 16 | import org.junit.jupiter.api.Test; 17 | import org.junit.jupiter.api.condition.EnabledIfSystemProperty; 18 | 19 | import io.debezium.server.events.ConnectorCompletedEvent; 20 | import io.debezium.server.events.ConnectorStartedEvent; 21 | import io.debezium.testing.testcontainers.PostgresTestResourceLifecycleManager; 22 | import io.debezium.util.Testing; 23 | import io.quarkus.test.common.QuarkusTestResource; 24 | import io.quarkus.test.junit.QuarkusTest; 25 | import io.quarkus.test.junit.TestProfile; 26 | 27 | @QuarkusTest 28 | @QuarkusTestResource(PostgresTestResourceLifecycleManager.class) 29 | @TestProfile(DebeziumServerSchemaRegistryProfile.class) 30 | @EnabledIfSystemProperty(named = "debezium.format.key", matches = "protobuf") 31 | @EnabledIfSystemProperty(named = "debezium.format.value", matches = "protobuf") 32 | public class DebeziumServerWithSchemaRegistryIT { 33 | 34 | private static final int MESSAGE_COUNT = 4; 35 | @Inject 36 | DebeziumServer server; 37 | 38 | { 39 | Testing.Files.delete(TestConfigSource.OFFSET_STORE_PATH); 40 | } 41 | 42 | void setupDependencies(@Observes ConnectorStartedEvent event) { 43 | if (!TestConfigSource.isItTest()) { 44 | return; 45 | } 46 | 47 | } 48 | 49 | void connectorCompleted(@Observes ConnectorCompletedEvent event) throws Exception { 50 | if (!event.isSuccess()) { 51 | throw (Exception) event.getError().get(); 52 | } 53 | } 54 | 55 | @Test 56 | public void testPostgresWithProtobuf() throws Exception { 57 | Testing.Print.enable(); 58 | final TestConsumer testConsumer = (TestConsumer) server.getConsumer(); 59 | Awaitility.await().atMost(Duration.ofSeconds(TestConfigSource.waitForSeconds())) 60 | .until(() -> (testConsumer.getValues().size() >= MESSAGE_COUNT)); 61 | assertThat(testConsumer.getValues().size()).isEqualTo(MESSAGE_COUNT); 62 | assertThat(testConsumer.getValues().get(0)).isInstanceOf(byte[].class); 63 | assertThat(testConsumer.getValues().get(0)).isNotNull(); 64 | assertThat(((byte[]) testConsumer.getValues().get(0))[0]).isEqualTo((byte) 0); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /debezium-server-core/src/test/java/io/debezium/server/Images.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server; 7 | 8 | /** 9 | * The list of container images used in testing 10 | */ 11 | public class Images { 12 | 13 | private static final String PRAVEGA_VERSION = "0.13.0"; 14 | private static final String PULSAR_VERSION = "2.5.2"; 15 | private static final String MILVUS_VERSION = "v2.5.4"; 16 | private static final String QDRANT_VERSION = "v1.14.0"; 17 | 18 | public static final String PRAVEGA_IMAGE = "mirror.gcr.io/pravega/pravega:" + PRAVEGA_VERSION; 19 | public static final String REDIS_IMAGE = "mirror.gcr.io/library/redis"; 20 | public static final String PUB_SUB_EMULATOR_IMAGE = "gcr.io/google.com/cloudsdktool/cloud-sdk:380.0.0-emulators"; 21 | public static final String KAFKA_IMAGE = "mirror.gcr.io/confluentinc/cp-kafka:5.4.3"; 22 | public static final String PULSAR_IMAGE = "mirror.gcr.io/apachepulsar/pulsar:" + PULSAR_VERSION; 23 | public static final String MILVUS_IMAGE = "mirror.gcr.io/milvusdb/milvus:" + MILVUS_VERSION; 24 | public static final String QDRANT_IMAGE = "qdrant/qdrant:" + QDRANT_VERSION; 25 | public static final String INFINISPAN_IMAGE = "quay.io/infinispan/server:" + System.getProperty("tag.infinispan", "latest"); 26 | public static final String LOCALSTACK_IMAGE = "mirror.gcr.io/localstack/localstack"; 27 | public static final String RABBITMQ_IMAGE = "mirror.gcr.io/library/rabbitmq:3.12.9-management"; 28 | public static final String ROCKETMQ_IMAGE = "mirror.gcr.io/apache/rocketmq"; 29 | public static final String WIREMOCK_IMAGE = "mirror.gcr.io/wiremock/wiremock:3.2.0"; 30 | public static final String NATS_IMAGE = "mirror.gcr.io/library/nats:latest"; 31 | public static final String NATS_STREAMING_IMAGE = "mirror.gcr.io/library/nats-streaming:latest"; 32 | } 33 | -------------------------------------------------------------------------------- /debezium-server-core/src/test/java/io/debezium/server/TestConsumer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server; 7 | 8 | import java.util.ArrayList; 9 | import java.util.Collections; 10 | import java.util.List; 11 | 12 | import jakarta.annotation.PostConstruct; 13 | import jakarta.annotation.PreDestroy; 14 | import jakarta.enterprise.context.Dependent; 15 | import jakarta.inject.Named; 16 | 17 | import io.debezium.DebeziumException; 18 | import io.debezium.engine.ChangeEvent; 19 | import io.debezium.engine.DebeziumEngine; 20 | import io.debezium.engine.DebeziumEngine.RecordCommitter; 21 | import io.debezium.util.Testing; 22 | 23 | @Dependent 24 | @Named("test") 25 | public class TestConsumer implements DebeziumEngine.ChangeConsumer> { 26 | 27 | final List values = Collections.synchronizedList(new ArrayList<>()); 28 | 29 | @PostConstruct 30 | void init() { 31 | Testing.print("Test consumer constructed"); 32 | } 33 | 34 | @PreDestroy 35 | void close() { 36 | Testing.print("Test consumer destroyed"); 37 | } 38 | 39 | @Override 40 | public void handleBatch(List> records, RecordCommitter> committer) 41 | throws InterruptedException { 42 | records.forEach(record -> { 43 | Testing.print(record); 44 | values.add(record.value()); 45 | try { 46 | committer.markProcessed(record); 47 | } 48 | catch (InterruptedException e) { 49 | throw new DebeziumException(e); 50 | } 51 | }); 52 | } 53 | 54 | public List getValues() { 55 | return values; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /debezium-server-core/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- 1 | io.debezium.server.TestConfigSource -------------------------------------------------------------------------------- /debezium-server-core/src/test/resources/application.properties: -------------------------------------------------------------------------------- 1 | debezium.api.enabled=true 2 | quarkus.kubernetes-client.devservices.enabled=false -------------------------------------------------------------------------------- /debezium-server-core/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | %d{ISO8601} %-5p %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext} %m [%c]%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 16 | 17 | 18 | 19 | 22 | 23 | 24 | 27 | 28 | 29 | 32 | 33 | 34 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /debezium-server-core/src/test/resources/secrets_test.txt: -------------------------------------------------------------------------------- 1 | user=postgres -------------------------------------------------------------------------------- /debezium-server-core/src/test/resources/testcontainers.properties: -------------------------------------------------------------------------------- 1 | hub.image.name.prefix=mirror.gcr.io/ 2 | -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/config/application.properties.cassandra.redis.example: -------------------------------------------------------------------------------- 1 | # Sink 2 | debezium.sink.type=redis 3 | debezium.sink.redis.address=localhost:12001 4 | 5 | # DSE Cassandra Connector 6 | debezium.source.connector.class=io.debezium.connector.dse.Dse6816Connector 7 | debezium.source.cassandra.node.id=node02 8 | debezium.source.cassandra.hosts=127.0.0.1 9 | debezium.source.cassandra.port=9042 10 | debezium.source.cassandra.config=/data/dse-6.8.16/resources/cassandra/conf/cassandra.yaml 11 | debezium.source.cassandra.driver.config.file=${user.dir}/config/lib/cassandra/driver.conf 12 | debezium.source.commit.log.relocation.dir=/data/dse-6.8.16/dse-data/relocdir 13 | debezium.source.commit.log.real.time.processing.enabled=true 14 | debezium.source.commit.log.marked.complete.poll.interval.ms=2000 15 | debezium.source.offset.storage=io.debezium.server.redis.RedisOffsetBackingStore 16 | debezium.source.topic.prefix=dse684 17 | ## internal http server 18 | debezium.source.http.port=8040 19 | 20 | # Transforms 21 | debezium.transforms=AddPrefix,EnvelopeTransformation 22 | debezium.transforms.AddPrefix.type=org.apache.kafka.connect.transforms.RegexRouter 23 | debezium.transforms.AddPrefix.regex=.* 24 | debezium.transforms.AddPrefix.replacement=data:\$0 25 | debezium.transforms.EnvelopeTransformation.type=io.debezium.connector.cassandra.transforms.EnvelopeTransformation 26 | 27 | debezium.sink.redis.memory.threshold.percentage=0 28 | 29 | # Quarkus 30 | quarkus.log.level=INFO 31 | quarkus.log.console.json=false 32 | # Uncomment to change Debezium port from 8080 [default] to any other port 33 | quarkus.http.port=8980 34 | quarkus.log.file.enable=true -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/config/application.properties.example: -------------------------------------------------------------------------------- 1 | debezium.sink.type=kinesis 2 | debezium.sink.kinesis.region=eu-central-1 3 | debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnector 4 | debezium.source.offset.storage.file.filename=data/offsets.dat 5 | debezium.source.offset.flush.interval.ms=0 6 | debezium.source.database.hostname=localhost 7 | debezium.source.database.port=5432 8 | debezium.source.database.user=postgres 9 | debezium.source.database.password=postgres 10 | debezium.source.database.dbname=postgres 11 | debezium.source.topic.prefix=tutorial 12 | debezium.source.schema.include.list=inventory 13 | quarkus.log.console.json=false 14 | -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/config/application.yaml.example: -------------------------------------------------------------------------------- 1 | debezium: 2 | sink: 3 | type: kinesis 4 | kinesis: 5 | region: eu-central-1 6 | source: 7 | connector: 8 | class: io.debezium.connector.postgresql.PostgresConnector 9 | offset: 10 | storage: 11 | file: 12 | filename: data/offsets.dat 13 | flush: 14 | interval: 15 | ms: 0 16 | database: 17 | hostname: localhost 18 | port: 5432 19 | user: postgres 20 | password: postgres 21 | dbname: postgres 22 | topic: 23 | prefix: tutorial 24 | schema: 25 | include: 26 | list: inventory 27 | 28 | quarkus: 29 | log: 30 | console: 31 | json: false 32 | -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/config/lib/cassandra/driver.conf: -------------------------------------------------------------------------------- 1 | datastax-java-driver { 2 | advanced.auth-provider { 3 | class = PlainTextAuthProvider 4 | username = dbz_user 5 | password = secret 6 | } 7 | } -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/config/metrics.yml: -------------------------------------------------------------------------------- 1 | startDelaySeconds: 0 2 | ssl: false 3 | lowercaseOutputName: false 4 | lowercaseOutputLabelNames: false 5 | rules: 6 | - pattern: "kafka.producer]+)><>([^:]+)" 7 | name: "kafka_producer_metrics_$2" 8 | type: GAUGE 9 | labels: 10 | client: "$1" 11 | - pattern: "kafka.producer]+), node-id=([^>]+)><>([^:]+)" 12 | name: "kafka_producer_node_metrics_$3" 13 | type: GAUGE 14 | labels: 15 | client: "$1" 16 | node: "$2" 17 | - pattern: "kafka.producer]+), topic=([^>]+)><>([^:]+)" 18 | name: "kafka_producer_topic_metrics_$3" 19 | type: GAUGE 20 | labels: 21 | client: "$1" 22 | topic: "$2" 23 | - pattern: "kafka.connect([^:]+):" 24 | name: "kafka_connect_worker_metrics_$1" 25 | type: GAUGE 26 | - pattern: "kafka.connect<>([^:]+)" 27 | name: "kafka_connect_metrics_$2" 28 | type: GAUGE 29 | labels: 30 | client: "$1" 31 | - pattern: "debezium.([^:]+)]+)><>RowsScanned" 32 | name: "debezium_metrics_RowsScanned" 33 | type: GAUGE 34 | labels: 35 | plugin: "$1" 36 | name: "$3" 37 | context: "$2" 38 | table: "$4" 39 | - pattern: "debezium.([^:]+)]+)>([^:]+)" 40 | name: "debezium_metrics_$6" 41 | type: GAUGE 42 | labels: 43 | plugin: "$1" 44 | name: "$2" 45 | task: "$3" 46 | context: "$4" 47 | database: "$5" 48 | - pattern: "debezium.([^:]+)]+)>([^:]+)" 49 | name: "debezium_metrics_$5" 50 | type: GAUGE 51 | labels: 52 | plugin: "$1" 53 | name: "$2" 54 | task: "$3" 55 | context: "$4" 56 | - pattern: "debezium.([^:]+)]+)>([^:]+)" 57 | name: "debezium_metrics_$4" 58 | type: GAUGE 59 | labels: 60 | plugin: "$1" 61 | name: "$3" 62 | context: "$2" 63 | -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/connectors/debezium-connector-cassandra-3/extra_class_path.sh: -------------------------------------------------------------------------------- 1 | export EXTRA_CLASS_PATH="$EXTRA_CONNECTOR_DIR/*"$PATH_SEP -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/connectors/debezium-connector-cassandra-3/jdk_java_options.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export JDK_JAVA_OPTIONS="--add-exports java.base/jdk.internal.misc=ALL-UNNAMED \ 3 | --add-exports java.base/jdk.internal.ref=ALL-UNNAMED \ 4 | --add-exports java.base/sun.nio.ch=ALL-UNNAMED \ 5 | --add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED \ 6 | --add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED \ 7 | --add-exports java.rmi/sun.rmi.server=ALL-UNNAMED \ 8 | --add-exports java.sql/java.sql=ALL-UNNAMED \ 9 | --add-opens java.base/java.lang.module=ALL-UNNAMED \ 10 | --add-opens java.base/jdk.internal.loader=ALL-UNNAMED \ 11 | --add-opens java.base/jdk.internal.ref=ALL-UNNAMED \ 12 | --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED \ 13 | --add-opens java.base/jdk.internal.math=ALL-UNNAMED \ 14 | --add-opens java.base/jdk.internal.module=ALL-UNNAMED \ 15 | --add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED \ 16 | --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED \ 17 | --add-opens=java.base/java.io=ALL-UNNAMED \ 18 | --add-opens java.base/sun.nio.ch=ALL-UNNAMED \ 19 | --add-opens java.base/jdk.internal.misc=ALL-UNNAMED" -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/connectors/debezium-connector-cassandra-4/extra_class_path.sh: -------------------------------------------------------------------------------- 1 | export EXTRA_CLASS_PATH="$EXTRA_CONNECTOR_DIR/*"$PATH_SEP -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/connectors/debezium-connector-cassandra-4/jdk_java_options.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export JDK_JAVA_OPTIONS="--add-exports java.base/jdk.internal.misc=ALL-UNNAMED \ 3 | --add-exports java.base/jdk.internal.ref=ALL-UNNAMED \ 4 | --add-exports java.base/sun.nio.ch=ALL-UNNAMED \ 5 | --add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED \ 6 | --add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED \ 7 | --add-exports java.rmi/sun.rmi.server=ALL-UNNAMED \ 8 | --add-exports java.sql/java.sql=ALL-UNNAMED \ 9 | --add-exports java.base/java.lang.ref=ALL-UNNAMED \ 10 | --add-exports jdk.unsupported/sun.misc=ALL-UNNAMED \ 11 | --add-opens java.base/java.lang.module=ALL-UNNAMED \ 12 | --add-opens java.base/jdk.internal.loader=ALL-UNNAMED \ 13 | --add-opens java.base/jdk.internal.ref=ALL-UNNAMED \ 14 | --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED \ 15 | --add-opens java.base/jdk.internal.math=ALL-UNNAMED \ 16 | --add-opens java.base/jdk.internal.module=ALL-UNNAMED \ 17 | --add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED \ 18 | --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED \ 19 | --add-opens java.base/java.io=ALL-UNNAMED \ 20 | --add-opens java.base/sun.nio.ch=ALL-UNNAMED \ 21 | --add-opens java.base/jdk.internal.misc=ALL-UNNAMED \ 22 | --add-opens java.base/java.lang=ALL-UNNAMED \ 23 | --add-opens java.base/java.lang.reflect=ALL-UNNAMED \ 24 | --add-opens java.base/java.util=ALL-UNNAMED \ 25 | --add-opens java.base/java.nio=ALL-UNNAMED \ 26 | --add-opens java.base/java.util.concurrent=ALL-UNNAMED" -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/connectors/debezium-connector-cassandra-5/extra_class_path.sh: -------------------------------------------------------------------------------- 1 | export EXTRA_CLASS_PATH="$EXTRA_CONNECTOR_DIR/*"$PATH_SEP -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/connectors/debezium-connector-cassandra-5/jdk_java_options.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export JDK_JAVA_OPTIONS="--add-exports java.base/jdk.internal.misc=ALL-UNNAMED \ 3 | --add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED \ 4 | --add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED \ 5 | --add-exports java.rmi/sun.rmi.server=ALL-UNNAMED \ 6 | --add-exports java.sql/java.sql=ALL-UNNAMED \ 7 | --add-exports java.base/java.lang.ref=ALL-UNNAMED \ 8 | --add-exports jdk.unsupported/sun.misc=ALL-UNNAMED \ 9 | --add-opens java.base/java.lang.module=ALL-UNNAMED \ 10 | --add-opens java.base/jdk.internal.loader=ALL-UNNAMED \ 11 | --add-opens java.base/jdk.internal.ref=ALL-UNNAMED \ 12 | --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED \ 13 | --add-opens java.base/jdk.internal.math=ALL-UNNAMED \ 14 | --add-opens java.base/jdk.internal.module=ALL-UNNAMED \ 15 | --add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED \ 16 | --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED \ 17 | --add-opens java.base/sun.nio.ch=ALL-UNNAMED \ 18 | --add-opens java.base/java.io=ALL-UNNAMED \ 19 | --add-opens java.base/java.lang=ALL-UNNAMED \ 20 | --add-opens java.base/java.lang.reflect=ALL-UNNAMED \ 21 | --add-opens java.base/java.util=ALL-UNNAMED \ 22 | --add-opens java.base/java.nio=ALL-UNNAMED" 23 | -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/connectors/debezium-connector-dse/extra_class_path.sh: -------------------------------------------------------------------------------- 1 | export EXTRA_CLASS_PATH="lib/slf4j-jboss-logmanager-2.0.0.Final.jar"$PATH_SEP"$DSE_HOME/resources/cassandra/lib/*"$PATH_SEP"$EXTRA_CONNECTOR_DIR/*"$PATH_SEP"$DSE_HOME/resources/cassandra/conf"$PATH_SEP"$DSE_HOME/lib/*"$PATH_SEP"$DSE_HOME/resources/solr/lib/*"$PATH_SEP -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/connectors/debezium-connector-dse/jdk_java_options.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export JDK_JAVA_OPTIONS="--add-exports java.base/jdk.internal.misc=ALL-UNNAMED \ 3 | --add-exports java.base/jdk.internal.ref=ALL-UNNAMED \ 4 | --add-exports java.base/sun.nio.ch=ALL-UNNAMED \ 5 | --add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED \ 6 | --add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED \ 7 | --add-exports java.rmi/sun.rmi.server=ALL-UNNAMED \ 8 | --add-exports java.sql/java.sql=ALL-UNNAMED \ 9 | --add-opens java.base/java.lang.module=ALL-UNNAMED \ 10 | --add-opens java.base/jdk.internal.loader=ALL-UNNAMED \ 11 | --add-opens java.base/jdk.internal.ref=ALL-UNNAMED \ 12 | --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED \ 13 | --add-opens java.base/jdk.internal.math=ALL-UNNAMED \ 14 | --add-opens java.base/jdk.internal.module=ALL-UNNAMED \ 15 | --add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED \ 16 | --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED \ 17 | --add-opens=java.base/java.io=ALL-UNNAMED \ 18 | --add-opens java.base/sun.nio.ch=ALL-UNNAMED \ 19 | --add-opens java.base/jdk.internal.misc=ALL-UNNAMED" -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/jmx/enable_jmx.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # To enable JMX functionality, export the JMX_HOST and JMX_PORT environment variables. 3 | # Modify the jmxremote.access and jmxremote.password files accordingly. 4 | if [ -n "${JMX_HOST}" -a -n "${JMX_PORT}" ]; then 5 | export JAVA_OPTS="-Dcom.sun.management.jmxremote.ssl=false \ 6 | -Dcom.sun.management.jmxremote.port=${JMX_PORT} \ 7 | -Dcom.sun.management.jmxremote.rmi.port=${JMX_PORT} \ 8 | -Dcom.sun.management.jmxremote.local.only=false \ 9 | -Djava.rmi.server.hostname=${JMX_HOST} \ 10 | -Dcom.sun.management.jmxremote.verbose=true" 11 | 12 | if [ -f "jmx/jmxremote.access" -a -f "jmx/jmxremote.password" ]; then 13 | chmod 600 jmx/jmxremote.password 14 | export JAVA_OPTS="${JAVA_OPTS} -Dcom.sun.management.jmxremote.authenticate=true \ 15 | -Dcom.sun.management.jmxremote.access.file=jmx/jmxremote.access \ 16 | -Dcom.sun.management.jmxremote.password.file=jmx/jmxremote.password" 17 | else 18 | export JAVA_OPT="${JAVA_OPTS} -Dcom.sun.management.jmxremote.authenticate=false" 19 | fi 20 | fi -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/jmx/jmxremote.access: -------------------------------------------------------------------------------- 1 | monitor readonly 2 | admin readwrite -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/jmx/jmxremote.password: -------------------------------------------------------------------------------- 1 | admin admin123 2 | monitor monitor123 -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/lib_metrics/enable_exporter.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # To enable Prometheus JMX exporter, set JMX_EXPORTER_PORT environment variable 3 | 4 | if [ -n "${JMX_EXPORTER_PORT}" ]; then 5 | JMX_EXPORTER_CONFIG=${JMX_EXPORTER_CONFIG:-"config/metrics.yml"} 6 | JMX_EXPORTER_AGENT_JAR=$(find lib_metrics -name "jmx_prometheus_javaagent-*.jar") 7 | export JAVA_OPTS="-javaagent:${JMX_EXPORTER_AGENT_JAR}=0.0.0.0:${JMX_EXPORTER_PORT}:${JMX_EXPORTER_CONFIG} ${JAVA_OPTS}" 8 | fi -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/run.bat: -------------------------------------------------------------------------------- 1 | 2 | SET PATH_SEP=; 3 | SET JAVA_BINARY=%JAVA_HOME%\bin\java 4 | 5 | for %%i in (debezium-server-*runner.jar) do set RUNNER=%%~i 6 | echo %RUNNER% 7 | SET LIB_PATH=lib\* 8 | @REM Configuration files and directories that need to be on the classpath 9 | SET LIB_CONFIG=config\lib 10 | IF %ENABLE_DEBEZIUM_SCRIPTING%=="true" LIB_PATH=%LIB_PATH%%PATH_SEP%lib_opt\* 11 | call "%JAVA_BINARY%" %DEBEZIUM_OPTS% %JAVA_OPTS% -cp %RUNNER%%PATH_SEP%%LIB_CONFIG_PATH%%PATH_SEP%%LIB_PATH% io.debezium.server.Main 12 | -------------------------------------------------------------------------------- /debezium-server-dist/src/main/resources/distro/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # In order to run Debezium for the Cassandra database, export the CASSANDRA_VERSION environment variable 4 | # with one of the following values: v3, v4, dse. 5 | # 6 | # Copyright Debezium Authors. 7 | # 8 | # Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | LIB_PATH="lib/*" 11 | # Configuration files and directories that need to be on the classpath 12 | LIB_CONFIG_PATH="config/lib" 13 | 14 | if [ "$OSTYPE" = "msys" ] || [ "$OSTYPE" = "cygwin" ]; then 15 | PATH_SEP=";" 16 | else 17 | PATH_SEP=":" 18 | fi 19 | 20 | if [ -n "$EXTRA_CONNECTOR" ]; then 21 | EXTRA_CONNECTOR=${EXTRA_CONNECTOR,,} 22 | export EXTRA_CONNECTOR_DIR="connectors/debezium-connector-${EXTRA_CONNECTOR}" 23 | 24 | echo "Connector - ${EXTRA_CONNECTOR} loaded from ${EXTRA_CONNECTOR_DIR}" 25 | 26 | if [ -f "${EXTRA_CONNECTOR_DIR}/jdk_java_options.sh" ]; then 27 | source "${EXTRA_CONNECTOR_DIR}/jdk_java_options.sh" 28 | fi 29 | 30 | EXTRA_CLASS_PATH="" 31 | if [ -f "${EXTRA_CONNECTOR_DIR}/extra_class_path.sh" ]; then 32 | source "${EXTRA_CONNECTOR_DIR}/extra_class_path.sh" 33 | LIB_PATH=$EXTRA_CLASS_PATH$LIB_PATH 34 | fi 35 | fi 36 | 37 | if [ -z "$JAVA_HOME" ]; then 38 | JAVA_BINARY="java" 39 | else 40 | JAVA_BINARY="$JAVA_HOME/bin/java" 41 | fi 42 | 43 | RUNNER=$(ls debezium-server-*runner.jar) 44 | 45 | ENABLE_DEBEZIUM_SCRIPTING=${ENABLE_DEBEZIUM_SCRIPTING:-false} 46 | if [[ "${ENABLE_DEBEZIUM_SCRIPTING}" == "true" ]]; then 47 | LIB_PATH=$LIB_PATH$PATH_SEP"lib_opt/*" 48 | fi 49 | 50 | source ./jmx/enable_jmx.sh 51 | source ./lib_metrics/enable_exporter.sh 52 | 53 | exec "$JAVA_BINARY" $DEBEZIUM_OPTS $JAVA_OPTS -cp \ 54 | $RUNNER$PATH_SEP$LIB_CONFIG_PATH$PATH_SEP$LIB_PATH io.debezium.server.Main -------------------------------------------------------------------------------- /debezium-server-dist/src/test/resources/redis/application.properties: -------------------------------------------------------------------------------- 1 | # Docker file used by GitHub actions to verify the debezium-server-dist works. 2 | debezium.sink.type=redis 3 | debezium.sink.redis.address=redis:6379 4 | debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnector 5 | debezium.source.offset.storage.file.filename=data/offsets.dat 6 | debezium.source.offset.flush.interval.ms=0 7 | debezium.source.database.hostname=postgres 8 | debezium.source.database.port=5432 9 | debezium.source.database.user=postgres 10 | debezium.source.database.password=postgres 11 | debezium.source.database.dbname=postgres 12 | debezium.source.topic.prefix=tutorial 13 | debezium.source.schema.include.list=inventory 14 | quarkus.log.console.json=false 15 | -------------------------------------------------------------------------------- /debezium-server-dist/src/test/resources/redis/docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Docker file used by GitHub actions to verify the debezium-server-dist works. 2 | services: 3 | postgres: 4 | image: quay.io/debezium/example-postgres:3.0 5 | ports: 6 | - "5432:5432" 7 | environment: 8 | - POSTGRES_USER=postgres 9 | - POSTGRES_PASSWORD=postgres 10 | networks: 11 | - backend 12 | 13 | redis: 14 | image: bitnami/redis:7.0 15 | ports: 16 | - "6379:6379" 17 | environment: 18 | - ALLOW_EMPTY_PASSWORD=yes 19 | networks: 20 | - backend 21 | networks: 22 | backend: 23 | name: debezium-backend -------------------------------------------------------------------------------- /debezium-server-eventhubs/src/main/java/io/debezium/server/eventhubs/EventDataBatchProxy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.eventhubs; 7 | 8 | import com.azure.messaging.eventhubs.EventData; 9 | import com.azure.messaging.eventhubs.EventDataBatch; 10 | import com.azure.messaging.eventhubs.EventHubProducerClient; 11 | import com.azure.messaging.eventhubs.models.CreateBatchOptions; 12 | 13 | /** 14 | * Proxy class/wrapper for EventDataBatch. Will create an inner EventDataBatch when data is being emitted to a specific 15 | * partition. 16 | */ 17 | public class EventDataBatchProxy { 18 | private EventDataBatch batch; 19 | private final EventHubProducerClient producer; 20 | private final CreateBatchOptions batchOptions; 21 | 22 | public EventDataBatchProxy(EventHubProducerClient producer, CreateBatchOptions batchOptions) { 23 | this.producer = producer; 24 | this.batchOptions = batchOptions; 25 | } 26 | 27 | public boolean tryAdd(final EventData eventData) { 28 | if (this.batch == null) { 29 | this.batch = producer.createBatch(this.batchOptions); 30 | } 31 | 32 | return batch.tryAdd(eventData); 33 | } 34 | 35 | public int getCount() { 36 | if (this.batch == null) { 37 | return 0; 38 | } 39 | 40 | return batch.getCount(); 41 | } 42 | 43 | public void emit() { 44 | if (this.batch == null) { 45 | return; 46 | } 47 | 48 | producer.send(this.batch); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /debezium-server-eventhubs/src/main/resources/META-INF/beans.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-eventhubs/src/main/resources/META-INF/beans.xml -------------------------------------------------------------------------------- /debezium-server-eventhubs/src/test/java/io/debezium/server/eventhubs/EventHubsTestConfigSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.eventhubs; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; 12 | 13 | import io.debezium.server.TestConfigSource; 14 | 15 | public class EventHubsTestConfigSource extends TestConfigSource { 16 | 17 | static final String EVENTHUBS_CONNECTION_STRING_SYSTEM_PROPERTY_NAME = "eventhubs.connection.string"; 18 | static final String EVENTHUBS_NAME_SYSTEM_PROPERTY_NAME = "eventhubs.hub.name"; 19 | static final String CONNECTION_STRING_FORMAT = "%s;EntityPath=%s"; 20 | 21 | public EventHubsTestConfigSource() { 22 | Map eventHubsTest = new HashMap<>(); 23 | 24 | // event hubs sink config 25 | eventHubsTest.put("debezium.sink.type", "eventhubs"); 26 | eventHubsTest.put("debezium.sink.eventhubs.connectionstring", getEventHubsConnectionString()); 27 | eventHubsTest.put("debezium.sink.eventhubs.hubname", getEventHubsName()); 28 | 29 | // postgresql source config 30 | 31 | eventHubsTest.put("debezium.source.connector.class", "io.debezium.connector.postgresql.PostgresConnector"); 32 | 33 | eventHubsTest.put("debezium.source." + StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG, 34 | OFFSET_STORE_PATH.toAbsolutePath().toString()); 35 | eventHubsTest.put("debezium.source.offset.flush.interval.ms", "0"); 36 | eventHubsTest.put("debezium.source.topic.prefix", "testc"); 37 | eventHubsTest.put("debezium.source.schema.include.list", "inventory"); 38 | eventHubsTest.put("debezium.source.table.include.list", "inventory.customers"); 39 | 40 | config = eventHubsTest; 41 | } 42 | 43 | @Override 44 | public int getOrdinal() { 45 | // Configuration property precedence is based on ordinal values and since we override the 46 | // properties in TestConfigSource, we should give this a higher priority. 47 | return super.getOrdinal() + 1; 48 | } 49 | 50 | public static String getEventHubsConnectionString() { 51 | return System.getProperty(EVENTHUBS_CONNECTION_STRING_SYSTEM_PROPERTY_NAME); 52 | } 53 | 54 | public static String getEventHubsName() { 55 | return System.getProperty(EVENTHUBS_NAME_SYSTEM_PROPERTY_NAME); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /debezium-server-eventhubs/src/test/java/io/debezium/server/eventhubs/EventHubsWithPartitionKeyProfile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.eventhubs; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import io.quarkus.test.junit.QuarkusTestProfile; 12 | 13 | public class EventHubsWithPartitionKeyProfile implements QuarkusTestProfile { 14 | 15 | @Override 16 | public Map getConfigOverrides() { 17 | Map config = new HashMap(); 18 | 19 | config.put("debezium.sink.eventhubs.partitionkey", "my-fixed-partition-key"); 20 | 21 | return config; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /debezium-server-eventhubs/src/test/java/io/debezium/server/eventhubs/EventHubsWithPartitionRouterProfile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.eventhubs; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import io.quarkus.test.junit.QuarkusTestProfile; 12 | 13 | public class EventHubsWithPartitionRouterProfile implements QuarkusTestProfile { 14 | 15 | @Override 16 | public Map getConfigOverrides() { 17 | Map config = new HashMap(); 18 | 19 | config.put("debezium.transforms", "PartitionRouter"); 20 | config.put("debezium.transforms.PartitionRouter.type", "io.debezium.transforms.partitions.PartitionRouting"); 21 | config.put("debezium.transforms.PartitionRouter.partition.payload.fields", "source.db"); 22 | config.put("debezium.transforms.PartitionRouter.partition.topic.num", "5"); 23 | 24 | return config; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /debezium-server-eventhubs/src/test/java/io/debezium/server/eventhubs/EventHubsWithStaticPartitionIdProfile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.eventhubs; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import io.quarkus.test.junit.QuarkusTestProfile; 12 | 13 | public class EventHubsWithStaticPartitionIdProfile implements QuarkusTestProfile { 14 | 15 | @Override 16 | public Map getConfigOverrides() { 17 | Map config = new HashMap(); 18 | 19 | config.put("debezium.sink.eventhubs.partitionid", "0"); 20 | 21 | return config; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /debezium-server-eventhubs/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- 1 | io.debezium.server.eventhubs.EventHubsTestConfigSource -------------------------------------------------------------------------------- /debezium-server-eventhubs/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | %d{ISO8601} %-5p %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext} %m [%c]%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 16 | 17 | 18 | 19 | 22 | 23 | 24 | 27 | 28 | 29 | 32 | 33 | 34 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /debezium-server-http/src/main/java/io/debezium/server/http/Authenticator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.http; 7 | 8 | import java.net.http.HttpRequest; 9 | import java.util.UUID; 10 | 11 | public interface Authenticator { 12 | void setAuthorizationHeader(HttpRequest.Builder httpRequestBuilder, String bodyContent, UUID messageId); 13 | 14 | boolean authenticate() throws InterruptedException; 15 | } 16 | -------------------------------------------------------------------------------- /debezium-server-http/src/main/java/io/debezium/server/http/jwt/JWTAuthorizationInitialRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.http.jwt; 7 | 8 | import com.fasterxml.jackson.databind.PropertyNamingStrategies; 9 | import com.fasterxml.jackson.databind.annotation.JsonNaming; 10 | 11 | @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) 12 | public class JWTAuthorizationInitialRequest { 13 | private String username; 14 | private String password; 15 | private long tokenExpiryInMinutes; 16 | private long refreshTokenExpiryInMinutes; 17 | 18 | public JWTAuthorizationInitialRequest() { 19 | 20 | } 21 | 22 | public JWTAuthorizationInitialRequest(String username, String password, long tokenExpiryInMinutes, long refreshTokenExpiryInMinutes) { 23 | this.username = username; 24 | this.password = password; 25 | this.tokenExpiryInMinutes = tokenExpiryInMinutes; 26 | this.refreshTokenExpiryInMinutes = refreshTokenExpiryInMinutes; 27 | } 28 | 29 | public String getUsername() { 30 | return username; 31 | } 32 | 33 | public void setUsername(String username) { 34 | this.username = username; 35 | } 36 | 37 | public String getPassword() { 38 | return password; 39 | } 40 | 41 | public void setPassword(String password) { 42 | this.password = password; 43 | } 44 | 45 | public long getTokenExpiryInMinutes() { 46 | return tokenExpiryInMinutes; 47 | } 48 | 49 | public void setTokenExpiryInMinutes(long tokenExpiryInMinutes) { 50 | this.tokenExpiryInMinutes = tokenExpiryInMinutes; 51 | } 52 | 53 | public long getRefreshTokenExpiryInMinutes() { 54 | return refreshTokenExpiryInMinutes; 55 | } 56 | 57 | public void setRefreshTokenExpiryInMinutes(long refreshTokenExpiryInMinutes) { 58 | this.refreshTokenExpiryInMinutes = refreshTokenExpiryInMinutes; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /debezium-server-http/src/main/java/io/debezium/server/http/jwt/JWTAuthorizationRefreshRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.http.jwt; 7 | 8 | import com.fasterxml.jackson.databind.PropertyNamingStrategies; 9 | import com.fasterxml.jackson.databind.annotation.JsonNaming; 10 | 11 | @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) 12 | public class JWTAuthorizationRefreshRequest { 13 | private String jwtRefreshToken; 14 | private long tokenExpiryInMinutes; 15 | private long refreshTokenExpiryInMinutes; 16 | 17 | public JWTAuthorizationRefreshRequest() { 18 | 19 | } 20 | 21 | public JWTAuthorizationRefreshRequest(String jwtRefreshToken, long tokenExpiryInMinutes, long refreshTokenExpiryInMinutes) { 22 | this.jwtRefreshToken = jwtRefreshToken; 23 | this.tokenExpiryInMinutes = tokenExpiryInMinutes; 24 | this.refreshTokenExpiryInMinutes = refreshTokenExpiryInMinutes; 25 | } 26 | 27 | public String getJwtRefreshToken() { 28 | return jwtRefreshToken; 29 | } 30 | 31 | public void setJwtRefreshToken(String jwtRefreshToken) { 32 | this.jwtRefreshToken = jwtRefreshToken; 33 | } 34 | 35 | public long getTokenExpiryInMinutes() { 36 | return tokenExpiryInMinutes; 37 | } 38 | 39 | public void setTokenExpiryInMinutes(long tokenExpiryInMinutes) { 40 | this.tokenExpiryInMinutes = tokenExpiryInMinutes; 41 | } 42 | 43 | public long getRefreshTokenExpiryInMinutes() { 44 | return refreshTokenExpiryInMinutes; 45 | } 46 | 47 | public void setRefreshTokenExpiryInMinutes(long refreshTokenExpiryInMinutes) { 48 | this.refreshTokenExpiryInMinutes = refreshTokenExpiryInMinutes; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /debezium-server-http/src/main/java/io/debezium/server/http/jwt/JWTAuthorizationResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.http.jwt; 7 | 8 | import com.fasterxml.jackson.databind.PropertyNamingStrategies; 9 | import com.fasterxml.jackson.databind.annotation.JsonNaming; 10 | 11 | @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) 12 | public class JWTAuthorizationResponse { 13 | private long expiresIn; 14 | private String jwt; 15 | private String jwtRefreshToken; 16 | private long refreshTokenExpiresIn; 17 | 18 | public JWTAuthorizationResponse() { 19 | 20 | } 21 | 22 | public JWTAuthorizationResponse(long expiresIn, String jwt, String jwtRefreshToken, long refreshTokenExpiresIn) { 23 | this.expiresIn = expiresIn; 24 | this.jwt = jwt; 25 | this.jwtRefreshToken = jwtRefreshToken; 26 | this.refreshTokenExpiresIn = refreshTokenExpiresIn; 27 | } 28 | 29 | public long getExpiresIn() { 30 | return expiresIn; 31 | } 32 | 33 | public void setExpiresIn(long expiresIn) { 34 | this.expiresIn = expiresIn; 35 | } 36 | 37 | public String getJwt() { 38 | return jwt; 39 | } 40 | 41 | public void setJwt(String jwt) { 42 | this.jwt = jwt; 43 | } 44 | 45 | public String getJwtRefreshToken() { 46 | return jwtRefreshToken; 47 | } 48 | 49 | public void setJwtRefreshToken(String jwtRefreshToken) { 50 | this.jwtRefreshToken = jwtRefreshToken; 51 | } 52 | 53 | public long getRefreshTokenExpiresIn() { 54 | return refreshTokenExpiresIn; 55 | } 56 | 57 | public void setRefreshTokenExpiresIn(long refreshTokenExpiresIn) { 58 | this.refreshTokenExpiresIn = refreshTokenExpiresIn; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /debezium-server-http/src/main/java/io/debezium/server/http/webhooks/StandardWebhooksAuthenticatorBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.http.webhooks; 7 | 8 | import org.eclipse.microprofile.config.Config; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import io.debezium.DebeziumException; 13 | 14 | public class StandardWebhooksAuthenticatorBuilder { 15 | private static final Logger LOGGER = LoggerFactory.getLogger(StandardWebhooksAuthenticatorBuilder.class); 16 | 17 | private static final String PROP_SECRET = "webhook.secret"; 18 | 19 | private String secret; 20 | 21 | public static StandardWebhooksAuthenticatorBuilder fromConfig(Config config, String prop_prefix) { 22 | StandardWebhooksAuthenticatorBuilder builder = new StandardWebhooksAuthenticatorBuilder(); 23 | 24 | builder.setSecret(config.getValue(prop_prefix + PROP_SECRET, String.class)); 25 | return builder; 26 | } 27 | 28 | public StandardWebhooksAuthenticatorBuilder setSecret(String secret) { 29 | this.secret = secret; 30 | return this; 31 | } 32 | 33 | public StandardWebhooksAuthenticator build() { 34 | if (secret == null) { 35 | String msg = "Cannot build StandardWebhooksAuthenticator. Secret must be set."; 36 | LOGGER.error(msg); 37 | throw new DebeziumException(msg); 38 | } 39 | 40 | return new StandardWebhooksAuthenticator(secret); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /debezium-server-http/src/main/resources/META-INF/beans.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-http/src/main/resources/META-INF/beans.xml -------------------------------------------------------------------------------- /debezium-server-http/src/test/java/io/debezium/server/http/HttpTestConfigSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.http; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; 12 | 13 | import io.debezium.embedded.async.ConvertingAsyncEngineBuilderFactory; 14 | import io.debezium.server.TestConfigSource; 15 | 16 | public class HttpTestConfigSource extends TestConfigSource { 17 | 18 | public HttpTestConfigSource() { 19 | Map httpTest = new HashMap<>(); 20 | 21 | httpTest.put("debezium.sink.type", "http"); 22 | httpTest.put("debezium.format.value", "cloudevents"); // Need to explicitly pass in the cloudevents format 23 | 24 | httpTest.put("debezium.source.connector.class", "io.debezium.connector.postgresql.PostgresConnector"); 25 | httpTest.put("debezium.source." + StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG, OFFSET_STORE_PATH.toAbsolutePath().toString()); 26 | httpTest.put("debezium.source.offset.flush.interval.ms", "0"); 27 | httpTest.put("debezium.source.topic.prefix", "testc"); 28 | httpTest.put("debezium.source.schema.include.list", "inventory"); 29 | httpTest.put("debezium.source.table.include.list", "inventory.customers"); 30 | httpTest.put("debezium.transforms", "addheader"); 31 | httpTest.put("debezium.transforms.addheader.type", "org.apache.kafka.connect.transforms.InsertHeader"); 32 | httpTest.put("debezium.transforms.addheader.header", "headerKey"); 33 | httpTest.put("debezium.transforms.addheader.value.literal", "headerValue"); 34 | httpTest.put("debezium.engine.factory", ConvertingAsyncEngineBuilderFactory.class.getName()); 35 | 36 | config = httpTest; 37 | } 38 | 39 | @Override 40 | public int getOrdinal() { 41 | // Configuration property precedence is based on ordinal values and since we override the 42 | // properties in TestConfigSource, we should give this a higher priority. 43 | return super.getOrdinal() + 1; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /debezium-server-http/src/test/java/io/debezium/server/http/jwt/JWTAuthenticatorBuilderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.http.jwt; 7 | 8 | import static org.mockito.ArgumentMatchers.any; 9 | import static org.mockito.ArgumentMatchers.eq; 10 | import static org.mockito.Mockito.mock; 11 | import static org.mockito.Mockito.when; 12 | 13 | import java.net.URI; 14 | import java.net.URISyntaxException; 15 | import java.util.Map; 16 | import java.util.Optional; 17 | 18 | import org.eclipse.microprofile.config.Config; 19 | import org.junit.jupiter.api.Test; 20 | 21 | public class JWTAuthenticatorBuilderTest { 22 | 23 | @Test 24 | public void verifyBuild() throws URISyntaxException { 25 | JWTAuthenticatorBuilder builder = new JWTAuthenticatorBuilder(); 26 | builder.setAuthUri(new URI("http://test.com/auth/authenticate")); 27 | builder.setRefreshUri(new URI("http://test.com/auth/refreshToken")); 28 | builder.setUsername("testUser"); 29 | builder.setPassword("testPassword"); 30 | 31 | JWTAuthenticator authenticator = builder.build(); 32 | } 33 | 34 | @Test 35 | public void verifyBuildFromConfig() throws URISyntaxException { 36 | Map configValues = Map.of("debezium.sink.http.jwt.url", "http://test.com/", 37 | "debezium.sink.http.jwt.username", "testUser", 38 | "debezium.sink.http.jwt.password", "testPassword"); 39 | 40 | Config result = mock(Config.class); 41 | 42 | for (Map.Entry entry : configValues.entrySet()) { 43 | Object value = entry.getValue(); 44 | when(result.getValue(eq(entry.getKey()), any())).thenReturn(value); 45 | when(result.getOptionalValue(eq(entry.getKey()), any())).thenReturn(Optional.of(value)); 46 | } 47 | 48 | JWTAuthenticatorBuilder builder = JWTAuthenticatorBuilder.fromConfig(result, "debezium.sink.http."); 49 | builder.setRefreshUri(new URI("http://test.com")); 50 | builder.setUsername("testUser"); 51 | builder.setPassword("testPassword"); 52 | 53 | JWTAuthenticator authenticator = builder.build(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /debezium-server-http/src/test/java/io/debezium/server/http/webhooks/StandardWebhooksAuthenticatorBuilderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.http.webhooks; 7 | 8 | import static org.mockito.ArgumentMatchers.any; 9 | import static org.mockito.ArgumentMatchers.eq; 10 | import static org.mockito.Mockito.mock; 11 | import static org.mockito.Mockito.when; 12 | 13 | import java.net.URISyntaxException; 14 | import java.util.Map; 15 | import java.util.Optional; 16 | 17 | import org.eclipse.microprofile.config.Config; 18 | import org.junit.jupiter.api.Assertions; 19 | import org.junit.jupiter.api.Test; 20 | 21 | public class StandardWebhooksAuthenticatorBuilderTest { 22 | 23 | @Test 24 | public void verifyBuild() throws URISyntaxException { 25 | StandardWebhooksAuthenticatorBuilder builder = new StandardWebhooksAuthenticatorBuilder(); 26 | builder.setSecret("whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw"); 27 | 28 | Assertions.assertDoesNotThrow(() -> { 29 | builder.build(); 30 | }); 31 | } 32 | 33 | @Test 34 | public void verifyBuildFromConfig() throws URISyntaxException { 35 | Map configValues = Map.of("debezium.sink.http.authentication.webhook.secret", "whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw"); 36 | 37 | Config result = mock(Config.class); 38 | 39 | for (Map.Entry entry : configValues.entrySet()) { 40 | Object value = entry.getValue(); 41 | when(result.getValue(eq(entry.getKey()), any())).thenReturn(value); 42 | when(result.getOptionalValue(eq(entry.getKey()), any())).thenReturn(Optional.of(value)); 43 | } 44 | 45 | StandardWebhooksAuthenticatorBuilder builder = StandardWebhooksAuthenticatorBuilder.fromConfig(result, "debezium.sink.http.authentication."); 46 | 47 | Assertions.assertDoesNotThrow(() -> { 48 | builder.build(); 49 | }); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /debezium-server-http/src/test/java/io/debezium/server/http/webhooks/StandardWebhooksAuthenticatorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.http.webhooks; 7 | 8 | import java.net.URI; 9 | import java.net.URISyntaxException; 10 | import java.net.http.HttpHeaders; 11 | import java.net.http.HttpRequest; 12 | import java.time.Clock; 13 | import java.time.Instant; 14 | import java.time.ZoneOffset; 15 | import java.util.Optional; 16 | import java.util.UUID; 17 | 18 | import org.junit.Test; 19 | import org.junit.jupiter.api.Assertions; 20 | 21 | public class StandardWebhooksAuthenticatorTest { 22 | 23 | @Test 24 | public void addAuthorizationHeader() throws URISyntaxException { 25 | Clock clock = Clock.fixed(Instant.ofEpochSecond(1234), ZoneOffset.UTC); 26 | UUID messageId = UUID.fromString("22bd292a-71ab-46fe-a460-8632d6754ac6"); 27 | StandardWebhooksAuthenticator authenticator = new StandardWebhooksAuthenticator( 28 | "whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw", clock); 29 | 30 | URI testURI = new URI("http://example.com"); 31 | String testEventContent = "{\"hello\":\"world\"}"; 32 | HttpRequest.Builder builder = HttpRequest.newBuilder(testURI); 33 | builder.POST(HttpRequest.BodyPublishers.ofString(testEventContent)); 34 | authenticator.setAuthorizationHeader(builder, testEventContent, messageId); 35 | HttpRequest request = builder.build(); 36 | 37 | HttpHeaders headers = request.headers(); 38 | 39 | Optional idHeader = headers.firstValue("webhook-id"); 40 | Assertions.assertTrue(idHeader.isPresent()); 41 | Assertions.assertEquals("msg_22bd292a-71ab-46fe-a460-8632d6754ac6", idHeader.get()); 42 | 43 | Optional timestampHeader = headers.firstValue("webhook-timestamp"); 44 | Assertions.assertTrue(timestampHeader.isPresent()); 45 | Assertions.assertEquals("1234", timestampHeader.get()); 46 | 47 | Optional signatureHeader = headers.firstValue("webhook-signature"); 48 | Assertions.assertTrue(signatureHeader.isPresent()); 49 | String[] sigParts = signatureHeader.get().split(","); 50 | Assertions.assertEquals(2, sigParts.length); 51 | Assertions.assertEquals("v1", sigParts[0]); 52 | 53 | // https://www.standardwebhooks.com/verify 54 | String expected = "v1,qCVBRIv6rKQVhSJBAmUSE9GkdCdPe2j6xzzkm89UcoA="; 55 | 56 | Assertions.assertEquals(expected, signatureHeader.get()); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /debezium-server-http/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- 1 | io.debezium.server.http.HttpTestConfigSource -------------------------------------------------------------------------------- /debezium-server-http/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | %d{ISO8601} %-5p %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext} %m [%c]%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 16 | 17 | 18 | 19 | 22 | 23 | 24 | 27 | 28 | 29 | 32 | 33 | 34 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /debezium-server-infinispan/src/main/resources/META-INF/beans.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-infinispan/src/main/resources/META-INF/beans.xml -------------------------------------------------------------------------------- /debezium-server-infinispan/src/test/java/io/debezium/server/infinispan/InfinispanTestConfigSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.infinispan; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; 12 | 13 | import io.debezium.server.TestConfigSource; 14 | 15 | public class InfinispanTestConfigSource extends TestConfigSource { 16 | 17 | public static final String CACHE_NAME = "debezium_test"; 18 | public static final String USER_NAME = "admin"; 19 | public static final String PASSWORD = "secret"; 20 | public static final String CONFIG_FILE = "infinispan-local.xml"; 21 | 22 | public InfinispanTestConfigSource() { 23 | Map infinispanTest = new HashMap<>(); 24 | 25 | infinispanTest.put("debezium.sink.type", "infinispan"); 26 | infinispanTest.put("debezium.sink.infinispan.cache", CACHE_NAME); 27 | infinispanTest.put("debezium.sink.infinispan.user", USER_NAME); 28 | infinispanTest.put("debezium.sink.infinispan.password", PASSWORD); 29 | infinispanTest.put("debezium.source.connector.class", "io.debezium.connector.postgresql.PostgresConnector"); 30 | infinispanTest.put("debezium.source." + StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG, OFFSET_STORE_PATH.toAbsolutePath().toString()); 31 | infinispanTest.put("debezium.source.offset.flush.interval.ms", "0"); 32 | infinispanTest.put("debezium.source.topic.prefix", "testc"); 33 | infinispanTest.put("debezium.source.schema.include.list", "inventory"); 34 | infinispanTest.put("debezium.source.table.include.list", "inventory.customers"); 35 | 36 | config = infinispanTest; 37 | } 38 | 39 | @Override 40 | public int getOrdinal() { 41 | // Configuration property precedence is based on ordinal values and since we override the 42 | // properties in TestConfigSource, we should give this a higher priority. 43 | return super.getOrdinal() + 1; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /debezium-server-infinispan/src/test/java/io/debezium/server/infinispan/InfinispanTestResourceLifecycleManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.infinispan; 7 | 8 | import java.util.Map; 9 | import java.util.concurrent.ConcurrentHashMap; 10 | import java.util.concurrent.atomic.AtomicBoolean; 11 | 12 | import org.infinispan.client.hotrod.impl.ConfigurationProperties; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | import org.testcontainers.containers.BindMode; 16 | import org.testcontainers.containers.GenericContainer; 17 | 18 | import io.debezium.server.Images; 19 | import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; 20 | 21 | public class InfinispanTestResourceLifecycleManager implements QuarkusTestResourceLifecycleManager { 22 | 23 | private static final Logger LOGGER = LoggerFactory.getLogger(InfinispanTestResourceLifecycleManager.class); 24 | public static final int PORT = ConfigurationProperties.DEFAULT_HOTROD_PORT; 25 | public static final String CONFIG_PATH = "/etc/infinispan-local.xml"; 26 | private static final GenericContainer container = new GenericContainer<>(Images.INFINISPAN_IMAGE) 27 | .withExposedPorts(PORT) 28 | .withClasspathResourceMapping(InfinispanTestConfigSource.CONFIG_FILE, CONFIG_PATH, BindMode.READ_ONLY) 29 | .withCommand("-c", CONFIG_PATH) 30 | .withEnv("USER", InfinispanTestConfigSource.USER_NAME) 31 | .withEnv("PASS", InfinispanTestConfigSource.PASSWORD); 32 | 33 | private static final AtomicBoolean running = new AtomicBoolean(false); 34 | 35 | private static synchronized void init() { 36 | if (!running.get()) { 37 | container.start(); 38 | running.set(true); 39 | } 40 | } 41 | 42 | public static String getHost() { 43 | return container.getHost(); 44 | } 45 | 46 | public static int getPort() { 47 | return container.getMappedPort(PORT); 48 | } 49 | 50 | @Override 51 | public Map start() { 52 | init(); 53 | 54 | Map params = new ConcurrentHashMap<>(); 55 | params.put("debezium.sink.infinispan.server.host", getHost()); 56 | params.put("debezium.sink.infinispan.server.port", String.valueOf(getPort())); 57 | return params; 58 | } 59 | 60 | @Override 61 | public void stop() { 62 | try { 63 | container.stop(); 64 | } 65 | catch (Exception e) { 66 | // ignored 67 | } 68 | running.set(false); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /debezium-server-infinispan/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- 1 | io.debezium.server.infinispan.InfinispanTestConfigSource -------------------------------------------------------------------------------- /debezium-server-infinispan/src/test/resources/infinispan-local.xml: -------------------------------------------------------------------------------- 1 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /debezium-server-instructlab/README.md: -------------------------------------------------------------------------------- 1 | # InstructLab 2 | 3 | InstructLab uses a novel synthetic data-based alignment tuning method for Large Language Models (LLMs). 4 | The **InstructLab** sink is designed to improve the training experience with taxonomies, by taking a stream of structured data from a source like a relational database, and applying transformations to create seeded examples for a taxonomy. 5 | 6 | ## Taxonomies 7 | 8 | In InstructLab, taxonomies define structured categories that help classify and organize knowledge or responses within the AI model. 9 | They act like labels or tags that help the model understand and respond to different topics or domains efficiently. 10 | 11 | Taxonomies serve several purposes: 12 | 13 | 1. They are used to guide how different pieces of knowledge are structured and retrieved. 14 | 2. They ensure that the AI provides relevant and context-aware responses. 15 | 3. They can be customized based on specific needs, allowing better control over how the AI categorizes and understands various types of questions. 16 | 17 | ## Training 18 | 19 | The `qna.yml` file is a **Question and Answer YAML** used by InstructLab to define structured FAQ-style data. 20 | It helps train the AI by providing predefined questions and their expected answers. 21 | 22 | The structure of the `qna.yml` file typically contains several pieces of information: 23 | 24 | 1. A list of questions that a user might ask. 25 | 2. Various answers the AI should provide. 26 | 3. Metadata (optional) that contains additional information like categories, intents, or tags. 27 | 28 | The `qna.yml` file therefore provides the following benefits: 29 | 30 | 1. Helps to train the AI with structured knowledge. 31 | 2. Ensures consistency with its responses. 32 | 3. Improves the AI's ability to answer domain-specific questions. 33 | 34 | ## Debezium InstructLab Integration 35 | 36 | The Debezium Server integration is designed to provide a common framework where structured data can be passed through a series of transformations to generate a series of questions and answers. 37 | The transformations amend the current event by adding headers, that the sink adapter consumes. 38 | The series of questions, answers, and context details are then appended to a specific taxonomy `qna.yml` file automatically. 39 | This eases the maintenance of these **Question and Answer YAML** files and allows driving the training data in real time from changes consumed by Debezium. 40 | 41 | Once the taxonomy tree has been created with various `qna.yml` files, InstructLab can be told to train the model based on that knowledge. 42 | -------------------------------------------------------------------------------- /debezium-server-instructlab/src/main/resources/META-INF/beans.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-instructlab/src/main/resources/META-INF/beans.xml -------------------------------------------------------------------------------- /debezium-server-instructlab/src/test/java/io/debezium/server/instructlab/MappingValueTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.instructlab; 7 | 8 | import static org.assertj.core.api.Assertions.assertThat; 9 | 10 | import org.junit.jupiter.api.Test; 11 | 12 | import io.debezium.server.instructlab.InstructLabSinkConsumer.MappingValue; 13 | 14 | /** 15 | * Unit tests for the {@link MappingValue} class. 16 | * 17 | * @author Chris Cranford 18 | */ 19 | public class MappingValueTest { 20 | 21 | @Test 22 | public void testMappingValueFromConstant() { 23 | final MappingValue value = MappingValue.from("/domain/path1/path2/abc"); 24 | assertThat(value.isHeader()).isFalse(); 25 | assertThat(value.isField()).isFalse(); 26 | assertThat(value.isConstant()).isTrue(); 27 | assertThat(value.getValue()).isEqualTo("/domain/path1/path2/abc"); 28 | } 29 | 30 | @Test 31 | public void testMappingValueFromHeader() { 32 | final MappingValue value = MappingValue.from("header:h1"); 33 | assertThat(value.isHeader()).isTrue(); 34 | assertThat(value.isField()).isFalse(); 35 | assertThat(value.isConstant()).isFalse(); 36 | assertThat(value.getValue()).isEqualTo("h1"); 37 | } 38 | 39 | @Test 40 | public void testMappingValueFromEmptyHeader() { 41 | final MappingValue value = MappingValue.from("header:"); 42 | assertThat(value.isHeader()).isTrue(); 43 | assertThat(value.isField()).isFalse(); 44 | assertThat(value.isConstant()).isFalse(); 45 | assertThat(value.getValue()).isEmpty(); 46 | } 47 | 48 | @Test 49 | public void testMappingValueFromFieldAllTopics() { 50 | final MappingValue value = MappingValue.from("value:xyz"); 51 | assertThat(value.isHeader()).isFalse(); 52 | assertThat(value.isField()).isTrue(); 53 | assertThat(value.isConstant()).isFalse(); 54 | assertThat(value.getValue()).isEqualTo("xyz"); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /debezium-server-instructlab/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- 1 | io.debezium.server.instructlab.InstructLabTestConfigSource -------------------------------------------------------------------------------- /debezium-server-instructlab/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | %d{ISO8601} %-5p %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext} %m [%c]%n 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 15 | 16 | 17 | 18 | 21 | 22 | 23 | 26 | 27 | 28 | 31 | 32 | 33 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /debezium-server-kafka/src/main/resources/META-INF/beans.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-kafka/src/main/resources/META-INF/beans.xml -------------------------------------------------------------------------------- /debezium-server-kafka/src/test/java/io/debezium/server/kafka/KafkaTestResourceLifecycleManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.kafka; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import org.testcontainers.containers.KafkaContainer; 12 | import org.testcontainers.utility.DockerImageName; 13 | 14 | import io.debezium.server.Images; 15 | import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; 16 | 17 | /** 18 | * Manages the lifecycle of a Kafka cluster test resource. 19 | * 20 | * @author Alfusainey Jallow 21 | */ 22 | public class KafkaTestResourceLifecycleManager implements QuarkusTestResourceLifecycleManager { 23 | 24 | private static KafkaContainer kafkaContainer = new KafkaContainer( 25 | DockerImageName.parse(Images.KAFKA_IMAGE).asCompatibleSubstituteFor("confluentinc/cp-kafka")); 26 | 27 | @Override 28 | public Map start() { 29 | kafkaContainer.start(); 30 | return new HashMap<>(); 31 | } 32 | 33 | @Override 34 | public void stop() { 35 | kafkaContainer.stop(); 36 | } 37 | 38 | public static String getBootstrapServers() { 39 | // if container is already started, start() will return early 40 | kafkaContainer.start(); 41 | return kafkaContainer.getBootstrapServers(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /debezium-server-kafka/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- 1 | io.debezium.server.kafka.KafkaTestConfigSource -------------------------------------------------------------------------------- /debezium-server-kinesis/src/main/resources/META-INF/beans.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-kinesis/src/main/resources/META-INF/beans.xml -------------------------------------------------------------------------------- /debezium-server-kinesis/src/test/java/io/debezium/server/kinesis/KinesisTestConfigSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.kinesis; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; 12 | 13 | import io.debezium.server.TestConfigSource; 14 | 15 | public class KinesisTestConfigSource extends TestConfigSource { 16 | 17 | public static final String KINESIS_REGION = "eu-central-1"; 18 | 19 | public KinesisTestConfigSource() { 20 | Map kinesisTest = new HashMap<>(); 21 | 22 | kinesisTest.put("debezium.sink.type", "kinesis"); 23 | kinesisTest.put("debezium.sink.kinesis.region", KINESIS_REGION); 24 | kinesisTest.put("debezium.source.connector.class", "io.debezium.connector.postgresql.PostgresConnector"); 25 | kinesisTest.put("debezium.source." + StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG, OFFSET_STORE_PATH.toAbsolutePath().toString()); 26 | kinesisTest.put("debezium.source.offset.flush.interval.ms", "0"); 27 | kinesisTest.put("debezium.source.topic.prefix", "testc"); 28 | kinesisTest.put("debezium.source.schema.include.list", "inventory"); 29 | kinesisTest.put("debezium.source.table.include.list", "inventory.customers"); 30 | 31 | config = kinesisTest; 32 | } 33 | 34 | @Override 35 | public int getOrdinal() { 36 | // Configuration property precedence is based on ordinal values and since we override the 37 | // properties in TestConfigSource, we should give this a higher priority. 38 | return super.getOrdinal() + 1; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /debezium-server-kinesis/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- 1 | io.debezium.server.kinesis.KinesisTestConfigSource -------------------------------------------------------------------------------- /debezium-server-kinesis/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | %d{ISO8601} %-5p %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext} %m [%c]%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 16 | 17 | 18 | 19 | 22 | 23 | 24 | 27 | 28 | 29 | 32 | 33 | 34 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /debezium-server-milvus/src/main/resources/META-INF/beans.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-milvus/src/main/resources/META-INF/beans.xml -------------------------------------------------------------------------------- /debezium-server-milvus/src/test/java/io/debezium/server/milvus/MilvusTestConfigSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.milvus; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; 12 | 13 | import io.debezium.server.TestConfigSource; 14 | 15 | public class MilvusTestConfigSource extends TestConfigSource { 16 | 17 | public MilvusTestConfigSource() { 18 | Map milvusTest = new HashMap<>(); 19 | 20 | milvusTest.put("debezium.sink.type", "milvus"); 21 | milvusTest.put("debezium.source.connector.class", "io.debezium.connector.postgresql.PostgresConnector"); 22 | milvusTest.put("debezium.source." + StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG, 23 | OFFSET_STORE_PATH.toAbsolutePath().toString()); 24 | milvusTest.put("debezium.source.offset.flush.interval.ms", "0"); 25 | milvusTest.put("debezium.source.topic.prefix", "testc"); 26 | milvusTest.put("debezium.source.schema.include.list", "inventory"); 27 | milvusTest.put("debezium.source.table.include.list", "inventory.t_vector"); 28 | milvusTest.put("debezium.sink.milvus.unwind.json", "true"); 29 | 30 | config = milvusTest; 31 | } 32 | 33 | @Override 34 | public int getOrdinal() { 35 | // Configuration property precedence is based on ordinal values and since we override the 36 | // properties in TestConfigSource, we should give this a higher priority. 37 | return super.getOrdinal() + 1; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /debezium-server-milvus/src/test/java/io/debezium/server/milvus/VectorPostgresTestResourceLifecycleManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.milvus; 7 | 8 | import java.util.Map; 9 | 10 | import io.debezium.DebeziumException; 11 | import io.debezium.connector.postgresql.connection.PostgresConnection; 12 | import io.debezium.jdbc.JdbcConfiguration; 13 | import io.debezium.testing.testcontainers.PostgresTestResourceLifecycleManager; 14 | 15 | /** 16 | * PostgreSQL Test resource with enabled vector extension 17 | */ 18 | public class VectorPostgresTestResourceLifecycleManager extends PostgresTestResourceLifecycleManager { 19 | 20 | @Override 21 | public Map start() { 22 | final var params = super.start(); 23 | 24 | final JdbcConfiguration config = JdbcConfiguration.create() 25 | .with("hostname", PostgresTestResourceLifecycleManager.POSTGRES_HOST) 26 | .with("port", params.get("debezium.source.database.port")) 27 | .with("user", PostgresTestResourceLifecycleManager.POSTGRES_USER) 28 | .with("password", PostgresTestResourceLifecycleManager.POSTGRES_PASSWORD) 29 | .with("dbname", "postgres") 30 | .build(); 31 | 32 | try (PostgresConnection connection = new PostgresConnection(config, "Debezium Milvus Test")) { 33 | connection.execute( 34 | "CREATE SCHEMA IF NOT EXISTS pgvector", 35 | "CREATE EXTENSION IF NOT EXISTS vector SCHEMA pgvector", 36 | "CREATE TABLE inventory.t_vector (pk INT8 PRIMARY KEY, value VARCHAR(32), f_vector pgvector.vector(3), f_json JSON);", 37 | "INSERT INTO inventory.t_vector VALUES (1, 'one', '[1.1, 1.2, 1.3]', '{}'::JSON)", 38 | "INSERT INTO inventory.t_vector VALUES (2, 'two', '[2.1, 2.2, 2.3]', '{}'::JSON)"); 39 | } 40 | catch (Exception e) { 41 | throw new DebeziumException(e); 42 | } 43 | 44 | return params; 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /debezium-server-milvus/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- 1 | io.debezium.server.milvus.MilvusTestConfigSource -------------------------------------------------------------------------------- /debezium-server-milvus/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | %d{ISO8601} %-5p %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext} %m [%c]%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 16 | 17 | 18 | 19 | 22 | 23 | 24 | 27 | 28 | 29 | 32 | 33 | 34 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /debezium-server-nats-jetstream/src/test/java/io/debezium/server/nats/jetstream/NatsJetStreamTestConfigSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.nats.jetstream; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; 12 | 13 | import io.debezium.server.TestConfigSource; 14 | 15 | public class NatsJetStreamTestConfigSource extends TestConfigSource { 16 | 17 | public NatsJetStreamTestConfigSource() { 18 | Map natsJetStreamTest = new HashMap<>(); 19 | natsJetStreamTest.put("debezium.sink.type", "nats-jetstream"); 20 | natsJetStreamTest.put("debezium.sink.nats-jetstream.url", 21 | NatsJetStreamTestResourceLifecycleManager.getNatsContainerUrl()); 22 | natsJetStreamTest.put("debezium.sink.nats-jetstream.create-stream", "true"); 23 | natsJetStreamTest.put("debezium.sink.nats-jetstream.subjects", "testc.inventory.customers"); 24 | natsJetStreamTest.put("debezium.source.connector.class", "io.debezium.connector.postgresql.PostgresConnector"); 25 | natsJetStreamTest.put("debezium.source.topic.prefix", "testc"); 26 | natsJetStreamTest.put("debezium.source.schema.include.list", "inventory"); 27 | natsJetStreamTest.put("debezium.source.table.include.list", "inventory.customers"); 28 | natsJetStreamTest.put("debezium.source." + StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG, 29 | OFFSET_STORE_PATH.toAbsolutePath().toString()); 30 | natsJetStreamTest.put("debezium.source.offset.flush.interval.ms", "0"); 31 | 32 | config = natsJetStreamTest; 33 | } 34 | 35 | @Override 36 | public int getOrdinal() { 37 | // Configuration property precedence is based on ordinal values and since we override the 38 | // properties in TestConfigSource, we should give this a higher priority. 39 | return super.getOrdinal() + 1; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /debezium-server-nats-jetstream/src/test/java/io/debezium/server/nats/jetstream/NatsJetStreamTestResourceLifecycleManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.nats.jetstream; 7 | 8 | import java.util.Map; 9 | import java.util.concurrent.ConcurrentHashMap; 10 | import java.util.concurrent.atomic.AtomicBoolean; 11 | 12 | import org.testcontainers.containers.GenericContainer; 13 | import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; 14 | 15 | import io.debezium.server.Images; 16 | import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; 17 | 18 | /** 19 | * Manages the lifecycle of a NATS Streaming test resource. 20 | * 21 | * @author Thiago Avancini 22 | */ 23 | public class NatsJetStreamTestResourceLifecycleManager implements QuarkusTestResourceLifecycleManager { 24 | 25 | public static final int NATS_PORT = 4222; 26 | 27 | private static final AtomicBoolean running = new AtomicBoolean(false); 28 | private static final GenericContainer container = new GenericContainer<>(Images.NATS_IMAGE) 29 | .withExposedPorts(NATS_PORT) 30 | .withCommand("-js") 31 | .waitingFor(new LogMessageWaitStrategy().withRegEx(".*Server is ready.*")); 32 | 33 | private static synchronized void start(boolean ignored) { 34 | if (!running.get()) { 35 | container.start(); 36 | running.set(true); 37 | } 38 | } 39 | 40 | @Override 41 | public Map start() { 42 | start(true); 43 | Map params = new ConcurrentHashMap<>(); 44 | return params; 45 | } 46 | 47 | @Override 48 | public void stop() { 49 | try { 50 | container.stop(); 51 | } 52 | catch (Exception e) { 53 | // ignored 54 | } 55 | running.set(false); 56 | } 57 | 58 | public static String getNatsContainerUrl() { 59 | start(true); 60 | return String.format("nats://%s:%d", container.getHost(), container.getFirstMappedPort()); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /debezium-server-nats-jetstream/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- 1 | io.debezium.server.nats.jetstream.NatsJetStreamTestConfigSource 2 | -------------------------------------------------------------------------------- /debezium-server-nats-jetstream/src/test/resources/testcontainers.properties: -------------------------------------------------------------------------------- 1 | hub.image.name.prefix=mirror.gcr.io/ 2 | -------------------------------------------------------------------------------- /debezium-server-nats-streaming/src/test/java/io/debezium/server/nats/streaming/NatsStreamingTestConfigSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.nats.streaming; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; 12 | 13 | import io.debezium.server.TestConfigSource; 14 | 15 | public class NatsStreamingTestConfigSource extends TestConfigSource { 16 | 17 | public NatsStreamingTestConfigSource() { 18 | Map natsStreamingTest = new HashMap<>(); 19 | 20 | natsStreamingTest.put("debezium.sink.type", "nats-streaming"); 21 | natsStreamingTest.put("debezium.sink.nats-streaming.url", 22 | NatsStreamingTestResourceLifecycleManager.getNatsStreamingContainerUrl()); 23 | natsStreamingTest.put("debezium.sink.nats-streaming.cluster.id", "debezium"); 24 | natsStreamingTest.put("debezium.sink.nats-streaming.client.id", "debezium-sink"); 25 | natsStreamingTest.put("debezium.source.connector.class", "io.debezium.connector.postgresql.PostgresConnector"); 26 | natsStreamingTest.put("debezium.source.topic.prefix", "testc"); 27 | natsStreamingTest.put("debezium.source.schema.include.list", "inventory"); 28 | natsStreamingTest.put("debezium.source.table.include.list", "inventory.customers"); 29 | natsStreamingTest.put("debezium.source." + StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG, 30 | OFFSET_STORE_PATH.toAbsolutePath().toString()); 31 | natsStreamingTest.put("debezium.source.offset.flush.interval.ms", "0"); 32 | 33 | config = natsStreamingTest; 34 | } 35 | 36 | @Override 37 | public int getOrdinal() { 38 | // Configuration property precedence is based on ordinal values and since we override the 39 | // properties in TestConfigSource, we should give this a higher priority. 40 | return super.getOrdinal() + 1; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /debezium-server-nats-streaming/src/test/java/io/debezium/server/nats/streaming/NatsStreamingTestResourceLifecycleManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.nats.streaming; 7 | 8 | import java.util.Map; 9 | import java.util.concurrent.ConcurrentHashMap; 10 | import java.util.concurrent.atomic.AtomicBoolean; 11 | 12 | import org.testcontainers.containers.GenericContainer; 13 | import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; 14 | 15 | import io.debezium.server.Images; 16 | import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; 17 | 18 | /** 19 | * Manages the lifecycle of a NATS Streaming test resource. 20 | * 21 | * @author Thiago Avancini 22 | */ 23 | public class NatsStreamingTestResourceLifecycleManager implements QuarkusTestResourceLifecycleManager { 24 | 25 | public static final int NATS_STREAMING_PORT = 4222; 26 | 27 | private static final AtomicBoolean running = new AtomicBoolean(false); 28 | private static final GenericContainer container = new GenericContainer<>(Images.NATS_STREAMING_IMAGE) 29 | .withExposedPorts(NATS_STREAMING_PORT) 30 | .withCommand("-SD", "-cid", "debezium") 31 | .waitingFor(new LogMessageWaitStrategy().withRegEx(".*Server is ready.*")); 32 | 33 | private static synchronized void start(boolean ignored) { 34 | if (!running.get()) { 35 | container.start(); 36 | running.set(true); 37 | } 38 | } 39 | 40 | @Override 41 | public Map start() { 42 | start(true); 43 | Map params = new ConcurrentHashMap<>(); 44 | return params; 45 | } 46 | 47 | @Override 48 | public void stop() { 49 | try { 50 | container.stop(); 51 | } 52 | catch (Exception e) { 53 | // ignored 54 | } 55 | running.set(false); 56 | } 57 | 58 | public static String getNatsStreamingContainerUrl() { 59 | start(true); 60 | return String.format("nats://%s:%d", container.getContainerIpAddress(), container.getFirstMappedPort()); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /debezium-server-nats-streaming/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- 1 | io.debezium.server.nats.streaming.NatsStreamingTestConfigSource 2 | -------------------------------------------------------------------------------- /debezium-server-nats-streaming/src/test/resources/testcontainers.properties: -------------------------------------------------------------------------------- 1 | hub.image.name.prefix=mirror.gcr.io/ 2 | -------------------------------------------------------------------------------- /debezium-server-pravega/src/main/java/io/debezium/server/pravega/PravegaSink.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.pravega; 7 | 8 | import io.debezium.engine.ChangeEvent; 9 | import io.debezium.engine.DebeziumEngine.ChangeConsumer; 10 | 11 | public interface PravegaSink extends ChangeConsumer>, AutoCloseable { 12 | 13 | } 14 | -------------------------------------------------------------------------------- /debezium-server-pravega/src/main/resources/META-INF/beans.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-pravega/src/main/resources/META-INF/beans.xml -------------------------------------------------------------------------------- /debezium-server-pravega/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- 1 | io.debezium.server.TestConfigSource -------------------------------------------------------------------------------- /debezium-server-pubsub/src/main/resources/META-INF/beans.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-pubsub/src/main/resources/META-INF/beans.xml -------------------------------------------------------------------------------- /debezium-server-pubsub/src/test/java/io/debezium/server/pubsub/PubSubLiteTestConfigSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.pubsub; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; 12 | 13 | import io.debezium.server.TestConfigSource; 14 | 15 | public class PubSubLiteTestConfigSource extends TestConfigSource { 16 | 17 | public PubSubLiteTestConfigSource() { 18 | Map pubsubLiteTest = new HashMap<>(); 19 | 20 | pubsubLiteTest.put("debezium.sink.type", "pubsublite"); 21 | pubsubLiteTest.put("debezium.source.connector.class", "io.debezium.connector.postgresql.PostgresConnector"); 22 | pubsubLiteTest.put("debezium.source." + StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG, 23 | OFFSET_STORE_PATH.toAbsolutePath().toString()); 24 | pubsubLiteTest.put("debezium.source.offset.flush.interval.ms", "0"); 25 | pubsubLiteTest.put("debezium.source.topic.prefix", "testc"); 26 | pubsubLiteTest.put("debezium.source.schema.include.list", "inventory"); 27 | pubsubLiteTest.put("debezium.source.table.include.list", "inventory.customers"); 28 | pubsubLiteTest.put("debezium.transforms", "addheader"); 29 | pubsubLiteTest.put("debezium.transforms.addheader.type", "org.apache.kafka.connect.transforms.InsertHeader"); 30 | pubsubLiteTest.put("debezium.transforms.addheader.header", "headerKey"); 31 | pubsubLiteTest.put("debezium.transforms.addheader.value.literal", "headerValue"); 32 | 33 | config = pubsubLiteTest; 34 | } 35 | 36 | @Override 37 | public int getOrdinal() { 38 | // Configuration property precedence is based on ordinal values and since we override the 39 | // properties in TestConfigSource, we should give this a higher priority. 40 | return super.getOrdinal() + 1; 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /debezium-server-pubsub/src/test/java/io/debezium/server/pubsub/PubSubTestConfigSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.pubsub; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; 12 | 13 | import io.debezium.server.TestConfigSource; 14 | 15 | public class PubSubTestConfigSource extends TestConfigSource { 16 | 17 | public PubSubTestConfigSource() { 18 | Map pubsubTest = new HashMap<>(); 19 | 20 | pubsubTest.put("debezium.sink.type", "pubsub"); 21 | pubsubTest.put("debezium.source.connector.class", "io.debezium.connector.postgresql.PostgresConnector"); 22 | pubsubTest.put("debezium.source." + StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG, 23 | OFFSET_STORE_PATH.toAbsolutePath().toString()); 24 | pubsubTest.put("debezium.source.offset.flush.interval.ms", "0"); 25 | pubsubTest.put("debezium.source.topic.prefix", "testc"); 26 | pubsubTest.put("debezium.source.schema.include.list", "inventory"); 27 | pubsubTest.put("debezium.source.table.include.list", "inventory.customers"); 28 | pubsubTest.put("debezium.transforms", "addheader"); 29 | pubsubTest.put("debezium.transforms.addheader.type", "org.apache.kafka.connect.transforms.InsertHeader"); 30 | pubsubTest.put("debezium.transforms.addheader.header", "headerKey"); 31 | pubsubTest.put("debezium.transforms.addheader.value.literal", "headerValue"); 32 | 33 | config = pubsubTest; 34 | } 35 | 36 | @Override 37 | public int getOrdinal() { 38 | // Configuration property precedence is based on ordinal values and since we override the 39 | // properties in TestConfigSource, we should give this a higher priority. 40 | return super.getOrdinal() + 1; 41 | } 42 | } -------------------------------------------------------------------------------- /debezium-server-pubsub/src/test/java/io/debezium/server/pubsub/PubSubTestResourceLifecycleManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.pubsub; 7 | 8 | import java.util.Map; 9 | import java.util.concurrent.ConcurrentHashMap; 10 | 11 | import org.testcontainers.containers.PubSubEmulatorContainer; 12 | import org.testcontainers.utility.DockerImageName; 13 | 14 | import io.debezium.server.Images; 15 | import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; 16 | 17 | public class PubSubTestResourceLifecycleManager implements QuarkusTestResourceLifecycleManager { 18 | 19 | public PubSubEmulatorContainer emulator = new PubSubEmulatorContainer( 20 | DockerImageName.parse(Images.PUB_SUB_EMULATOR_IMAGE)); 21 | private static String endpoint; 22 | 23 | @Override 24 | public Map start() { 25 | emulator.start(); 26 | 27 | Map params = new ConcurrentHashMap<>(); 28 | endpoint = emulator.getEmulatorEndpoint(); 29 | params.put("debezium.sink.pubsub.address", endpoint); 30 | return params; 31 | } 32 | 33 | @Override 34 | public void stop() { 35 | try { 36 | if (emulator != null) { 37 | emulator.stop(); 38 | } 39 | } 40 | catch (Exception e) { 41 | // ignored 42 | } 43 | } 44 | 45 | public static String getEmulatorEndpoint() { 46 | return endpoint; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /debezium-server-pubsub/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- 1 | io.debezium.server.pubsub.PubSubTestConfigSource -------------------------------------------------------------------------------- /debezium-server-pubsub/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | %d{ISO8601} %-5p %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext} %m [%c]%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 16 | 17 | 18 | 19 | 22 | 23 | 24 | 27 | 28 | 29 | 32 | 33 | 34 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /debezium-server-pulsar/src/main/resources/META-INF/beans.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-pulsar/src/main/resources/META-INF/beans.xml -------------------------------------------------------------------------------- /debezium-server-pulsar/src/test/java/io/debezium/server/pulsar/PulsarTestConfigSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.pulsar; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; 12 | 13 | import io.debezium.server.TestConfigSource; 14 | 15 | public class PulsarTestConfigSource extends TestConfigSource { 16 | 17 | public PulsarTestConfigSource() { 18 | Map pulsarTest = new HashMap<>(); 19 | 20 | pulsarTest.put("debezium.sink.type", "pulsar"); 21 | pulsarTest.put("debezium.source.connector.class", "io.debezium.connector.postgresql.PostgresConnector"); 22 | pulsarTest.put("debezium.source." + StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG, 23 | OFFSET_STORE_PATH.toAbsolutePath().toString()); 24 | pulsarTest.put("debezium.source.offset.flush.interval.ms", "0"); 25 | pulsarTest.put("debezium.source.topic.prefix", "testc"); 26 | pulsarTest.put("debezium.source.schema.include.list", "inventory"); 27 | pulsarTest.put("debezium.source.table.include.list", "inventory.customers,inventory.nokey"); 28 | pulsarTest.put("debezium.transforms", "addheader"); 29 | pulsarTest.put("debezium.transforms.addheader.type", "org.apache.kafka.connect.transforms.InsertHeader"); 30 | pulsarTest.put("debezium.transforms.addheader.header", "headerKey"); 31 | pulsarTest.put("debezium.transforms.addheader.value.literal", "headerValue"); 32 | 33 | config = pulsarTest; 34 | } 35 | 36 | @Override 37 | public int getOrdinal() { 38 | // Configuration property precedence is based on ordinal values and since we override the 39 | // properties in TestConfigSource, we should give this a higher priority. 40 | return super.getOrdinal() + 1; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /debezium-server-pulsar/src/test/java/io/debezium/server/pulsar/PulsarTestResourceLifecycleManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | 7 | package io.debezium.server.pulsar; 8 | 9 | import java.time.Duration; 10 | import java.util.Map; 11 | import java.util.concurrent.ConcurrentHashMap; 12 | 13 | import org.testcontainers.containers.BindMode; 14 | import org.testcontainers.containers.GenericContainer; 15 | import org.testcontainers.containers.wait.strategy.Wait; 16 | 17 | import io.debezium.server.Images; 18 | import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; 19 | 20 | public class PulsarTestResourceLifecycleManager implements QuarkusTestResourceLifecycleManager { 21 | 22 | public static final int PULSAR_PORT = 6650; 23 | public static final int PULSAR_HTTP_PORT = 8080; 24 | 25 | private static final GenericContainer container = new GenericContainer<>(Images.PULSAR_IMAGE) 26 | .withStartupTimeout(Duration.ofSeconds(90)) 27 | .waitingFor(Wait.forLogMessage(".*messaging service is ready.*", 1)) 28 | .withCommand("bin/pulsar", "standalone") 29 | .withClasspathResourceMapping("/docker/conf/", "/pulsar/conf", BindMode.READ_ONLY) 30 | .withExposedPorts(PULSAR_PORT, PULSAR_HTTP_PORT); 31 | 32 | @Override 33 | public Map start() { 34 | container.start(); 35 | 36 | Map params = new ConcurrentHashMap<>(); 37 | params.put("debezium.sink.pulsar.client.serviceUrl", getPulsarServiceUrl()); 38 | 39 | return params; 40 | } 41 | 42 | @Override 43 | public void stop() { 44 | try { 45 | if (container != null) { 46 | container.stop(); 47 | } 48 | } 49 | catch (Exception e) { 50 | // ignored 51 | } 52 | } 53 | 54 | public static String getPulsarServiceUrl() { 55 | return "pulsar://localhost:" + container.getMappedPort(PULSAR_PORT); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /debezium-server-pulsar/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- 1 | io.debezium.server.pulsar.PulsarTestConfigSource -------------------------------------------------------------------------------- /debezium-server-pulsar/src/test/resources/docker/conf/bkenv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | # Set JAVA_HOME here to override the environment setting 22 | # JAVA_HOME= 23 | 24 | # default settings for starting bookkeeper 25 | 26 | # Configuration file of settings used in bookie server 27 | BOOKIE_CONF=${BOOKIE_CONF:-"$BK_HOME/conf/bookkeeper.conf"} 28 | 29 | # Log4j configuration file 30 | # BOOKIE_LOG_CONF= 31 | 32 | # Logs location 33 | # BOOKIE_LOG_DIR= 34 | 35 | # Memory size options 36 | BOOKIE_MEM=${BOOKIE_MEM:-${PULSAR_MEM:-"-Xms2g -Xmx2g -XX:MaxDirectMemorySize=2g"}} 37 | 38 | # Garbage collection options 39 | BOOKIE_GC=${BOOKIE_GC:-"-XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB"} 40 | 41 | # Extra options to be passed to the jvm 42 | BOOKIE_EXTRA_OPTS="${BOOKIE_EXTRA_OPTS} ${BOOKIE_MEM} ${BOOKIE_GC} -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.maxCapacity.default=1000 -Dio.netty.recycler.linkCapacity=1024" 43 | 44 | # Add extra paths to the bookkeeper classpath 45 | # BOOKIE_EXTRA_CLASSPATH= 46 | 47 | #Folder where the Bookie server PID file should be stored 48 | #BOOKIE_PID_DIR= 49 | 50 | #Wait time before forcefully kill the Bookie server instance, if the stop is not successful 51 | #BOOKIE_STOP_TIMEOUT= 52 | 53 | #Entry formatter class to format entries. 54 | #ENTRY_FORMATTER_CLASS= 55 | -------------------------------------------------------------------------------- /debezium-server-pulsar/src/test/resources/docker/conf/client.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | # Configuration for pulsar-client and pulsar-admin CLI tools 21 | 22 | # URL for Pulsar REST API (for admin operations) 23 | # For TLS: 24 | # webServiceUrl=https://localhost:8443/ 25 | webServiceUrl=http://localhost:8080/ 26 | 27 | # URL for Pulsar Binary Protocol (for produce and consume operations) 28 | # For TLS: 29 | # brokerServiceUrl=pulsar+ssl://localhost:6651/ 30 | brokerServiceUrl=pulsar://localhost:6650/ 31 | 32 | # Authentication plugin to authenticate with servers 33 | # e.g. for TLS 34 | # authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls 35 | authPlugin= 36 | 37 | # Parameters passed to authentication plugin. 38 | # A comma separated list of key:value pairs. 39 | # Keys depend on the configured authPlugin. 40 | # e.g. for TLS 41 | # authParams=tlsCertFile:/path/to/client-cert.pem,tlsKeyFile:/path/to/client-key.pem 42 | authParams= 43 | 44 | # Allow TLS connections to servers whose certificate cannot be 45 | # be verified to have been signed by a trusted certificate 46 | # authority. 47 | tlsAllowInsecureConnection=false 48 | 49 | # Whether server hostname must match the common name of the certificate 50 | # the server is using. 51 | tlsEnableHostnameVerification=false 52 | 53 | # Path for the trusted TLS certificate file. 54 | # This cert is used to verify that any cert presented by a server 55 | # is signed by a certificate authority. If this verification 56 | # fails, then the cert is untrusted and the connection is dropped. 57 | tlsTrustCertsFilePath= 58 | 59 | # Enable TLS with KeyStore type configuration in broker. 60 | useKeyStoreTls=false 61 | 62 | # TLS KeyStore type configuration: JKS, PKCS12 63 | tlsTrustStoreType=JKS 64 | 65 | # TLS TrustStore path 66 | tlsTrustStorePath= 67 | 68 | # TLS TrustStore password 69 | tlsTrustStorePassword= 70 | -------------------------------------------------------------------------------- /debezium-server-pulsar/src/test/resources/docker/conf/filesystem_offload_core_site.xml: -------------------------------------------------------------------------------- 1 | 21 | 22 | 23 | 24 | fs.defaultFS 25 | 26 | 27 | 28 | hadoop.tmp.dir 29 | pulsar 30 | 31 | 32 | io.file.buffer.size 33 | 4096 34 | 35 | 36 | io.seqfile.compress.blocksize 37 | 1000000 38 | 39 | 40 | io.seqfile.compression.type 41 | BLOCK 42 | 43 | 44 | io.map.index.interval 45 | 128 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /debezium-server-pulsar/src/test/resources/docker/conf/functions-logging/console_logging_config.ini: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | [loggers] 21 | keys=root 22 | 23 | [handlers] 24 | keys=stream_handler 25 | 26 | [formatters] 27 | keys=formatter 28 | 29 | [logger_root] 30 | level=INFO 31 | handlers=stream_handler 32 | 33 | [handler_stream_handler] 34 | class=StreamHandler 35 | level=INFO 36 | formatter=formatter 37 | args=(sys.stdout,) 38 | 39 | [formatter_formatter] 40 | format=[%(asctime)s] [%(levelname)s] %(filename)s: %(message)s 41 | datefmt=%Y-%m-%d %H:%M:%S %z -------------------------------------------------------------------------------- /debezium-server-pulsar/src/test/resources/docker/conf/functions-logging/logging_config.ini: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | [loggers] 21 | keys=root 22 | 23 | [handlers] 24 | keys=rotating_file_handler 25 | 26 | [formatters] 27 | keys=formatter 28 | 29 | [logger_root] 30 | level=INFO 31 | handlers=rotating_file_handler 32 | 33 | [handler_rotating_file_handler] 34 | class=log.CreatePathRotatingFileHandler 35 | level=INFO 36 | formatter=formatter 37 | args=(os.getenv("LOG_FILE",""), 5, 10 * 1024 * 1024) 38 | 39 | [formatter_formatter] 40 | format=[%(asctime)s] [%(levelname)s] %(filename)s: %(message)s 41 | datefmt=%Y-%m-%d %H:%M:%S %z -------------------------------------------------------------------------------- /debezium-server-pulsar/src/test/resources/docker/conf/global_zookeeper.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | # The number of milliseconds of each tick 21 | tickTime=2000 22 | # The number of ticks that the initial 23 | # synchronization phase can take 24 | initLimit=10 25 | # The number of ticks that can pass between 26 | # sending a request and getting an acknowledgement 27 | syncLimit=5 28 | # the directory where the snapshot is stored. 29 | dataDir=data/global-zookeeper 30 | # the port at which the clients will connect 31 | clientPort=2184 32 | 33 | # the port at which the admin will listen 34 | admin.enableServer=true 35 | admin.serverPort=9991 36 | 37 | # the maximum number of client connections. 38 | # increase this if you need to handle more clients 39 | #maxClientCnxns=60 40 | # 41 | # Be sure to read the maintenance section of the 42 | # administrator guide before turning on autopurge. 43 | # 44 | # http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance 45 | # 46 | # The number of snapshots to retain in dataDir 47 | autopurge.snapRetainCount=3 48 | # Purge task interval in hours 49 | # Set to "0" to disable auto purge feature 50 | autopurge.purgeInterval=1 51 | -------------------------------------------------------------------------------- /debezium-server-pulsar/src/test/resources/docker/conf/log4j2-scripts/filter.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | // This is a sample file which can be used by log4j2 to: only log debug-statement which has text: pulsar-topic-name 20 | var result = false; 21 | var topicName = "pulsar-topic-name"; 22 | /* 23 | * Find more logEvent attributes at : 24 | * https://github.com/apache/logging-log4j2/blob/dbd2d252a1b4139a9bd9eb213c89f28498db6dcf/log4j-core/src/main/java/org/apache/logging/log4j/core/LogEvent.java 25 | */ 26 | if (logEvent.getLevel() == "DEBUG"){ 27 | if(logEvent.getMessage().getFormattedMessage().indexOf(topicName)!=-1) { 28 | result = true; 29 | } 30 | } else { 31 | result = true; 32 | } 33 | result; -------------------------------------------------------------------------------- /debezium-server-pulsar/src/test/resources/docker/conf/presto/config.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | node.id=ffffffff-ffff-ffff-ffff-ffffffffffff 21 | node.environment=test 22 | http-server.http.port=8081 23 | 24 | discovery-server.enabled=true 25 | discovery.uri=http://localhost:8081 26 | 27 | exchange.http-client.max-connections=1000 28 | exchange.http-client.max-connections-per-server=1000 29 | exchange.http-client.connect-timeout=1m 30 | exchange.http-client.idle-timeout=1m 31 | 32 | scheduler.http-client.max-connections=1000 33 | scheduler.http-client.max-connections-per-server=1000 34 | scheduler.http-client.connect-timeout=1m 35 | scheduler.http-client.idle-timeout=1m 36 | 37 | query.client.timeout=5m 38 | query.min-expire-age=30m 39 | 40 | presto.version=testversion 41 | distributed-joins-enabled=true 42 | node-scheduler.include-coordinator=true 43 | -------------------------------------------------------------------------------- /debezium-server-pulsar/src/test/resources/docker/conf/presto/jvm.config: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | -server 21 | -Xmx16G 22 | -XX:+UseG1GC 23 | -XX:G1HeapRegionSize=32M 24 | -XX:+UseGCOverheadLimit 25 | -XX:+ExplicitGCInvokesConcurrent 26 | -XX:+HeapDumpOnOutOfMemoryError 27 | -XX:+ExitOnOutOfMemoryError 28 | -------------------------------------------------------------------------------- /debezium-server-pulsar/src/test/resources/docker/conf/presto/log.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | com.facebook.presto=INFO 21 | com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory=WARN 22 | com.ning.http.client=WARN 23 | com.facebook.presto.server.PluginManager=DEBUG 24 | -------------------------------------------------------------------------------- /debezium-server-pulsar/src/test/resources/docker/conf/pulsar_env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | # Set JAVA_HOME here to override the environment setting 22 | # JAVA_HOME= 23 | 24 | # default settings for starting pulsar broker 25 | 26 | # Log4j configuration file 27 | # PULSAR_LOG_CONF= 28 | 29 | # Logs location 30 | # PULSAR_LOG_DIR= 31 | 32 | # Configuration file of settings used in broker server 33 | # PULSAR_BROKER_CONF= 34 | 35 | # Configuration file of settings used in bookie server 36 | # PULSAR_BOOKKEEPER_CONF= 37 | 38 | # Configuration file of settings used in zookeeper server 39 | # PULSAR_ZK_CONF= 40 | 41 | # Configuration file of settings used in global zookeeper server 42 | # PULSAR_GLOBAL_ZK_CONF= 43 | 44 | # Extra options to be passed to the jvm 45 | PULSAR_MEM=${PULSAR_MEM:-"-Xms2g -Xmx2g -XX:MaxDirectMemorySize=4g"} 46 | 47 | # Garbage collection options 48 | PULSAR_GC=${PULSAR_GC:-"-XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB"} 49 | 50 | # Extra options to be passed to the jvm 51 | PULSAR_EXTRA_OPTS=${PULSAR_EXTRA_OPTS:-" -Dpulsar.allocator.exit_on_oom=true -Dio.netty.recycler.maxCapacity.default=1000 -Dio.netty.recycler.linkCapacity=1024"} 52 | 53 | # Add extra paths to the bookkeeper classpath 54 | # PULSAR_EXTRA_CLASSPATH= 55 | 56 | #Folder where the Bookie server PID file should be stored 57 | #PULSAR_PID_DIR= 58 | 59 | #Wait time before forcefully kill the pulser server instance, if the stop is not successful 60 | #PULSAR_STOP_TIMEOUT= 61 | 62 | -------------------------------------------------------------------------------- /debezium-server-pulsar/src/test/resources/docker/conf/pulsar_tools_env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | 21 | # Set JAVA_HOME here to override the environment setting 22 | # JAVA_HOME= 23 | 24 | # default settings for starting pulsar broker 25 | 26 | # Log4j configuration file 27 | # PULSAR_LOG_CONF= 28 | 29 | # Logs location 30 | # PULSAR_LOG_DIR= 31 | 32 | # Configuration file of settings used in broker server 33 | # PULSAR_BROKER_CONF= 34 | 35 | # Configuration file of settings used in bookie server 36 | # PULSAR_BOOKKEEPER_CONF= 37 | 38 | # Configuration file of settings used in zookeeper server 39 | # PULSAR_ZK_CONF= 40 | 41 | # Configuration file of settings used in global zookeeper server 42 | # PULSAR_GLOBAL_ZK_CONF= 43 | 44 | # Extra options to be passed to the jvm 45 | PULSAR_MEM="-Xmx128m -XX:MaxDirectMemorySize=128m" 46 | 47 | # Garbage collection options 48 | PULSAR_GC=" -client " 49 | 50 | # Extra options to be passed to the jvm 51 | PULSAR_EXTRA_OPTS="${PULSAR_EXTRA_OPTS} ${PULSAR_MEM} ${PULSAR_GC} -Dio.netty.leakDetectionLevel=disabled" 52 | 53 | # Add extra paths to the bookkeeper classpath 54 | # PULSAR_EXTRA_CLASSPATH= 55 | 56 | #Folder where the Bookie server PID file should be stored 57 | #PULSAR_PID_DIR= 58 | 59 | #Wait time before forcefully kill the pulser server instance, if the stop is not successful 60 | #PULSAR_STOP_TIMEOUT= 61 | -------------------------------------------------------------------------------- /debezium-server-pulsar/src/test/resources/docker/conf/schema_example.conf: -------------------------------------------------------------------------------- 1 | { 2 | "type": "STRING", 3 | "schema": "", 4 | "properties": { 5 | "key1" : "value1" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /debezium-server-pulsar/src/test/resources/docker/conf/zookeeper.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | # The number of milliseconds of each tick 21 | tickTime=2000 22 | # The number of ticks that the initial 23 | # synchronization phase can take 24 | initLimit=10 25 | # The number of ticks that can pass between 26 | # sending a request and getting an acknowledgement 27 | syncLimit=5 28 | # the directory where the snapshot is stored. 29 | dataDir=data/zookeeper 30 | # the port at which the clients will connect 31 | clientPort=2181 32 | 33 | # the port at which the admin will listen 34 | admin.enableServer=true 35 | admin.serverPort=9990 36 | 37 | # the maximum number of client connections. 38 | # increase this if you need to handle more clients 39 | #maxClientCnxns=60 40 | # 41 | # Be sure to read the maintenance section of the 42 | # administrator guide before turning on autopurge. 43 | # 44 | # http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance 45 | # 46 | # The number of snapshots to retain in dataDir 47 | autopurge.snapRetainCount=3 48 | # Purge task interval in hours 49 | # Set to "0" to disable auto purge feature 50 | autopurge.purgeInterval=1 51 | 52 | # Requires updates to be synced to media of the transaction log before finishing 53 | # processing the update. If this option is set to 'no', ZooKeeper will not require 54 | # updates to be synced to the media. 55 | # WARNING: it's not recommended to run a production ZK cluster with forceSync disabled. 56 | forceSync=yes 57 | -------------------------------------------------------------------------------- /debezium-server-pulsar/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | %d{ISO8601} %-5p %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext} %m [%c]%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 16 | 17 | 18 | 19 | 22 | 23 | 24 | 27 | 28 | 29 | 32 | 33 | 34 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /debezium-server-qdrant/src/main/resources/META-INF/beans.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-qdrant/src/main/resources/META-INF/beans.xml -------------------------------------------------------------------------------- /debezium-server-qdrant/src/test/java/io/debezium/server/qdrant/QdrantTestConfigSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.qdrant; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; 12 | 13 | import io.debezium.server.TestConfigSource; 14 | 15 | public class QdrantTestConfigSource extends TestConfigSource { 16 | 17 | public QdrantTestConfigSource() { 18 | Map qdrantTest = new HashMap<>(); 19 | 20 | qdrantTest.put("debezium.sink.type", "qdrant"); 21 | qdrantTest.put("debezium.source.connector.class", "io.debezium.connector.postgresql.PostgresConnector"); 22 | qdrantTest.put("debezium.source." + StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG, 23 | OFFSET_STORE_PATH.toAbsolutePath().toString()); 24 | qdrantTest.put("debezium.source.offset.flush.interval.ms", "0"); 25 | qdrantTest.put("debezium.source.topic.prefix", "testc"); 26 | qdrantTest.put("debezium.source.schema.include.list", "inventory"); 27 | qdrantTest.put("debezium.source.table.include.list", "inventory.t_vector"); 28 | 29 | config = qdrantTest; 30 | } 31 | 32 | @Override 33 | public int getOrdinal() { 34 | // Configuration property precedence is based on ordinal values and since we override the 35 | // properties in TestConfigSource, we should give this a higher priority. 36 | return super.getOrdinal() + 1; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /debezium-server-qdrant/src/test/java/io/debezium/server/qdrant/QdrantTestResourceLifecycleManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | 7 | package io.debezium.server.qdrant; 8 | 9 | import java.time.Duration; 10 | import java.util.Map; 11 | import java.util.concurrent.ConcurrentHashMap; 12 | 13 | import org.testcontainers.qdrant.QdrantContainer; 14 | import org.testcontainers.utility.DockerImageName; 15 | 16 | import io.debezium.DebeziumException; 17 | import io.debezium.server.Images; 18 | import io.grpc.Grpc; 19 | import io.grpc.InsecureChannelCredentials; 20 | import io.qdrant.client.QdrantClient; 21 | import io.qdrant.client.QdrantGrpcClient; 22 | import io.qdrant.client.grpc.Collections.Distance; 23 | import io.qdrant.client.grpc.Collections.VectorParams; 24 | import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; 25 | 26 | public class QdrantTestResourceLifecycleManager implements QuarkusTestResourceLifecycleManager { 27 | 28 | public static final int QDRANT_RPC_PORT = 6334; 29 | 30 | @SuppressWarnings("resource") 31 | private static final QdrantContainer container = new QdrantContainer(DockerImageName.parse(Images.QDRANT_IMAGE).asCompatibleSubstituteFor("qdrant/qdrant")) 32 | .withStartupTimeout(Duration.ofSeconds(90)); 33 | 34 | @Override 35 | public Map start() { 36 | container.start(); 37 | createQdrantCollections(); 38 | 39 | Map params = new ConcurrentHashMap<>(); 40 | params.put("debezium.sink.qdrant.host", "localhost"); 41 | params.put("debezium.sink.qdrant.port", getPort()); 42 | 43 | return params; 44 | } 45 | 46 | @Override 47 | public void stop() { 48 | try { 49 | if (container != null) { 50 | container.stop(); 51 | container.close(); 52 | } 53 | } 54 | catch (Exception e) { 55 | // ignored 56 | } 57 | } 58 | 59 | private void createQdrantCollections() { 60 | try (var client = new QdrantClient(QdrantGrpcClient.newBuilder( 61 | Grpc.newChannelBuilder("%s:%s".formatted("localhost", getPort()), InsecureChannelCredentials.create()) 62 | .build(), 63 | true) 64 | .build());) { 65 | client.createCollectionAsync(QdrantIT.COLLECTION_NAME, 66 | VectorParams.newBuilder().setDistance(Distance.Euclid).setSize(3).build()).get(); 67 | } 68 | catch (Exception e) { 69 | throw new DebeziumException(e); 70 | } 71 | } 72 | 73 | protected String getPort() { 74 | return Integer.toString(container.getMappedPort(QDRANT_RPC_PORT)); 75 | } 76 | 77 | } 78 | -------------------------------------------------------------------------------- /debezium-server-qdrant/src/test/java/io/debezium/server/qdrant/VectorPostgresTestResourceLifecycleManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.qdrant; 7 | 8 | import java.util.Map; 9 | 10 | import io.debezium.DebeziumException; 11 | import io.debezium.connector.postgresql.connection.PostgresConnection; 12 | import io.debezium.jdbc.JdbcConfiguration; 13 | import io.debezium.testing.testcontainers.PostgresTestResourceLifecycleManager; 14 | 15 | /** 16 | * PostgreSQL Test resource with enabled vector extension 17 | */ 18 | public class VectorPostgresTestResourceLifecycleManager extends PostgresTestResourceLifecycleManager { 19 | 20 | @Override 21 | public Map start() { 22 | final var params = super.start(); 23 | 24 | final JdbcConfiguration config = JdbcConfiguration.create() 25 | .with("hostname", PostgresTestResourceLifecycleManager.POSTGRES_HOST) 26 | .with("port", params.get("debezium.source.database.port")) 27 | .with("user", PostgresTestResourceLifecycleManager.POSTGRES_USER) 28 | .with("password", PostgresTestResourceLifecycleManager.POSTGRES_PASSWORD) 29 | .with("dbname", "postgres") 30 | .build(); 31 | 32 | try (PostgresConnection connection = new PostgresConnection(config, "Debezium Qdrant Test")) { 33 | connection.execute( 34 | "CREATE SCHEMA IF NOT EXISTS pgvector", 35 | "CREATE EXTENSION IF NOT EXISTS vector SCHEMA pgvector", 36 | "CREATE TABLE inventory.t_vector (pk INT8 PRIMARY KEY, value VARCHAR(32), f_vector pgvector.vector(3), f_json JSON);", 37 | "INSERT INTO inventory.t_vector VALUES (1, 'one', '[1.1, 1.2, 1.3]', '{}'::JSON)", 38 | "INSERT INTO inventory.t_vector VALUES (2, 'two', '[2.1, 2.2, 2.3]', '{}'::JSON)"); 39 | } 40 | catch (Exception e) { 41 | throw new DebeziumException(e); 42 | } 43 | 44 | return params; 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /debezium-server-qdrant/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- 1 | io.debezium.server.qdrant.QdrantTestConfigSource -------------------------------------------------------------------------------- /debezium-server-qdrant/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | %d{ISO8601} %-5p %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext} %m [%c]%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 16 | 17 | 18 | 19 | 22 | 23 | 24 | 27 | 28 | 29 | 32 | 33 | 34 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /debezium-server-rabbitmq/src/main/resources/META-INF/beans.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-rabbitmq/src/main/resources/META-INF/beans.xml -------------------------------------------------------------------------------- /debezium-server-rabbitmq/src/test/java/io/debezium/server/rabbitmq/RabbitMqContainer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.rabbitmq; 7 | 8 | import java.time.Duration; 9 | 10 | import org.testcontainers.containers.GenericContainer; 11 | import org.testcontainers.containers.wait.strategy.Wait; 12 | import org.testcontainers.utility.DockerImageName; 13 | 14 | import io.debezium.server.Images; 15 | 16 | /** 17 | * RabbitMQ container 18 | */ 19 | public class RabbitMqContainer extends GenericContainer { 20 | 21 | private static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse(Images.RABBITMQ_IMAGE); 22 | public static final int BROKER_PORT = 5672; 23 | public static final int STREAM_PORT = 5552; 24 | 25 | public RabbitMqContainer() { 26 | super(DEFAULT_IMAGE_NAME); 27 | withExposedPorts(BROKER_PORT, STREAM_PORT, 15672); 28 | 29 | this.waitStrategy = Wait.forLogMessage(".*Server startup complete.*", 1).withStartupTimeout(Duration.ofSeconds(60)); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /debezium-server-rabbitmq/src/test/java/io/debezium/server/rabbitmq/RabbitMqStreamTestResourceLifecycleManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.rabbitmq; 7 | 8 | import java.io.IOException; 9 | import java.util.Map; 10 | import java.util.concurrent.ConcurrentHashMap; 11 | import java.util.concurrent.atomic.AtomicBoolean; 12 | 13 | import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; 14 | 15 | /** 16 | * Manages the lifecycle of a RabbitMQ cluster test resource. 17 | */ 18 | public class RabbitMqStreamTestResourceLifecycleManager implements QuarkusTestResourceLifecycleManager { 19 | 20 | public static final int PORT = 5552; 21 | public static RabbitMqContainer container = new RabbitMqContainer(); 22 | private static final AtomicBoolean running = new AtomicBoolean(false); 23 | 24 | private static synchronized void init() throws IOException, InterruptedException { 25 | if (!running.get()) { 26 | 27 | container.start(); 28 | container.execInContainer("rabbitmq-plugins", "enable", "--all"); 29 | running.set(true); 30 | } 31 | } 32 | 33 | @Override 34 | public Map start() { 35 | try { 36 | init(); 37 | } 38 | catch (IOException e) { 39 | throw new RuntimeException(e); 40 | } 41 | catch (InterruptedException e) { 42 | throw new RuntimeException(e); 43 | } 44 | Map params = new ConcurrentHashMap<>(); 45 | params.put("debezium.sink.rabbitmqstream.host", container.getHost()); 46 | params.put("debezium.sink.rabbitmqstream.port", String.valueOf(getPort())); 47 | return params; 48 | } 49 | 50 | @Override 51 | public void stop() { 52 | try { 53 | if (container != null) { 54 | container.stop(); 55 | } 56 | } 57 | catch (Exception e) { 58 | // ignored 59 | } 60 | } 61 | 62 | public static int getPort() { 63 | return container.getMappedPort(PORT); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /debezium-server-rabbitmq/src/test/java/io/debezium/server/rabbitmq/RabbitMqTestConfigSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.rabbitmq; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; 12 | 13 | import io.debezium.server.TestConfigSource; 14 | 15 | public class RabbitMqTestConfigSource extends TestConfigSource { 16 | 17 | public static final String TOPIC_NAME = "testc.inventory.customers"; 18 | 19 | public RabbitMqTestConfigSource() { 20 | 21 | final Map rabbitmqConfig = new HashMap<>(); 22 | String sinkType = System.getProperty("debezium.sink.type"); 23 | if ("rabbitmqstream".equals(sinkType)) { 24 | rabbitmqConfig.put("debezium.sink.type", "rabbitmqstream"); 25 | } 26 | else { 27 | rabbitmqConfig.put("debezium.sink.type", "rabbitmq"); 28 | } 29 | 30 | rabbitmqConfig.put("debezium.source.connector.class", "io.debezium.connector.postgresql.PostgresConnector"); 31 | rabbitmqConfig.put("debezium.source." + StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG, OFFSET_STORE_PATH.toAbsolutePath().toString()); 32 | rabbitmqConfig.put("debezium.source.offset.flush.interval.ms", "0"); 33 | rabbitmqConfig.put("debezium.source.topic.prefix", "testc"); 34 | rabbitmqConfig.put("debezium.source.schema.include.list", "inventory"); 35 | rabbitmqConfig.put("debezium.source.table.include.list", "inventory.customers"); 36 | rabbitmqConfig.put("debezium.sink.rabbitmq.routingKey.source", "topic"); 37 | config = rabbitmqConfig; 38 | } 39 | 40 | @Override 41 | public int getOrdinal() { 42 | // Configuration property precedence is based on ordinal values and since we override the 43 | // properties in TestConfigSource, we should give this a higher priority. 44 | return super.getOrdinal() + 1; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /debezium-server-rabbitmq/src/test/java/io/debezium/server/rabbitmq/RabbitMqTestResourceLifecycleManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.rabbitmq; 7 | 8 | import java.io.IOException; 9 | import java.util.Map; 10 | import java.util.concurrent.ConcurrentHashMap; 11 | import java.util.concurrent.atomic.AtomicBoolean; 12 | 13 | import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; 14 | 15 | /** 16 | * Manages the lifecycle of a RabbitMQ cluster test resource. 17 | */ 18 | public class RabbitMqTestResourceLifecycleManager implements QuarkusTestResourceLifecycleManager { 19 | 20 | public static final int PORT = 5672; 21 | public static RabbitMqContainer container = new RabbitMqContainer(); 22 | private static final AtomicBoolean running = new AtomicBoolean(false); 23 | 24 | private static synchronized void init() throws IOException, InterruptedException { 25 | if (!running.get()) { 26 | 27 | container.start(); 28 | container.execInContainer("rabbitmq-plugins", "enable", "--all"); 29 | running.set(true); 30 | } 31 | } 32 | 33 | @Override 34 | public Map start() { 35 | try { 36 | init(); 37 | } 38 | catch (IOException | InterruptedException e) { 39 | throw new RuntimeException(e); 40 | } 41 | Map params = new ConcurrentHashMap<>(); 42 | params.put("debezium.sink.rabbitmq.connection.host", container.getHost()); 43 | params.put("debezium.sink.rabbitmq.connection.port", String.valueOf(getPort())); 44 | return params; 45 | } 46 | 47 | @Override 48 | public void stop() { 49 | try { 50 | if (container != null) { 51 | container.stop(); 52 | } 53 | } 54 | catch (Exception e) { 55 | // ignored 56 | } 57 | } 58 | 59 | public static int getPort() { 60 | return container.getMappedPort(PORT); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /debezium-server-rabbitmq/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- 1 | io.debezium.server.rabbitmq.RabbitMqTestConfigSource -------------------------------------------------------------------------------- /debezium-server-rabbitmq/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | %d{ISO8601} %-5p %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext} %m [%c]%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 16 | 17 | 18 | 19 | 22 | 23 | 24 | 27 | 28 | 29 | 32 | 33 | 34 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /debezium-server-redis/src/main/java/io/debezium/server/redis/RedisOffsetBackingStore.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.redis; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | /** 12 | * Deprecated and replaced with {@link io.debezium.storage.redis.offset.RedisOffsetBackingStore} 13 | * 14 | */ 15 | 16 | @Deprecated 17 | public class RedisOffsetBackingStore extends io.debezium.storage.redis.offset.RedisOffsetBackingStore { 18 | 19 | private static final Logger LOGGER = LoggerFactory.getLogger(RedisOffsetBackingStore.class); 20 | 21 | public RedisOffsetBackingStore() { 22 | LOGGER.warn("Class '{}' is deprecated and scheduled for removal, please use '{}'", 23 | RedisOffsetBackingStore.class.getName(), 24 | io.debezium.storage.redis.offset.RedisOffsetBackingStore.class.getName()); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /debezium-server-redis/src/main/java/io/debezium/server/redis/RedisSchemaHistory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.redis; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | /** 12 | * Deprecated and replaced with {@link io.debezium.storage.redis.history.RedisSchemaHistory} 13 | * 14 | */ 15 | 16 | @Deprecated 17 | public final class RedisSchemaHistory extends io.debezium.storage.redis.history.RedisSchemaHistory { 18 | 19 | private static final Logger LOGGER = LoggerFactory.getLogger(RedisOffsetBackingStore.class); 20 | 21 | public RedisSchemaHistory() { 22 | LOGGER.warn("Class '{}' is deprecated and scheduled for removal, please use '{}'", 23 | RedisSchemaHistory.class.getName(), 24 | io.debezium.storage.redis.history.RedisSchemaHistory.class.getName()); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /debezium-server-redis/src/test/java/io/debezium/server/redis/RedisOffsetTestProfile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.redis; 7 | 8 | import java.util.Arrays; 9 | import java.util.HashMap; 10 | import java.util.List; 11 | import java.util.Map; 12 | 13 | import io.debezium.testing.testcontainers.PostgresTestResourceLifecycleManager; 14 | import io.quarkus.test.junit.QuarkusTestProfile; 15 | 16 | public class RedisOffsetTestProfile implements QuarkusTestProfile { 17 | 18 | @Override 19 | public List testResources() { 20 | return Arrays.asList(new TestResourceEntry(PostgresTestResourceLifecycleManager.class)); 21 | } 22 | 23 | @Override 24 | public Map getConfigOverrides() { 25 | Map config = new HashMap(); 26 | config.put("debezium.source.connector.class", "io.debezium.connector.postgresql.PostgresConnector"); 27 | config.put("debezium.source.offset.storage", "io.debezium.server.redis.RedisOffsetBackingStore"); 28 | config.put("debezium.source.offset.flush.interval.ms", "0"); 29 | return config; 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /debezium-server-redis/src/test/java/io/debezium/server/redis/RedisSSLOptionsStreamTestProfile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.redis; 7 | 8 | import java.net.URL; 9 | import java.util.Arrays; 10 | import java.util.HashMap; 11 | import java.util.List; 12 | import java.util.Map; 13 | 14 | import io.debezium.testing.testcontainers.PostgresTestResourceLifecycleManager; 15 | import io.quarkus.test.junit.QuarkusTestProfile; 16 | 17 | public class RedisSSLOptionsStreamTestProfile implements QuarkusTestProfile { 18 | 19 | @Override 20 | public List testResources() { 21 | return Arrays.asList( 22 | new TestResourceEntry(PostgresTestResourceLifecycleManager.class), 23 | new TestResourceEntry(RedisSSLTestResourceLifecycleManager.class)); 24 | } 25 | 26 | public Map getConfigOverrides() { 27 | Map config = new HashMap(); 28 | URL keyStoreFile = RedisSSLOptionsStreamTestProfile.class.getClassLoader().getResource("ssl/client-keystore.p12"); 29 | URL trustStoreFile = RedisSSLOptionsStreamTestProfile.class.getClassLoader().getResource("ssl/client-truststore.p12"); 30 | 31 | // Instead of using javax.net.ssl properties (used in RedisSSLStreamIT), redis sink specific properties are used 32 | config.put("debezium.sink.redis.ssl.truststore.path", trustStoreFile.getPath()); 33 | config.put("debezium.sink.redis.ssl.truststore.password", "secret"); 34 | config.put("debezium.sink.redis.ssl.truststore.type", "PKCS12"); 35 | config.put("debezium.sink.redis.ssl.keystore.path", keyStoreFile.getPath()); 36 | config.put("debezium.sink.redis.ssl.keystore.password", "secret"); 37 | config.put("debezium.sink.redis.ssl.keystore.type", "PKCS12"); 38 | 39 | config.put("debezium.source.offset.storage", "io.debezium.server.redis.RedisOffsetBackingStore"); 40 | config.put("debezium.source.connector.class", "io.debezium.connector.postgresql.PostgresConnector"); 41 | return config; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /debezium-server-redis/src/test/java/io/debezium/server/redis/RedisSSLStreamIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.redis; 7 | 8 | import static org.junit.Assert.assertTrue; 9 | 10 | import java.util.Map; 11 | 12 | import org.junit.jupiter.api.Test; 13 | 14 | import io.quarkus.test.junit.QuarkusIntegrationTest; 15 | import io.quarkus.test.junit.TestProfile; 16 | 17 | import redis.clients.jedis.HostAndPort; 18 | import redis.clients.jedis.Jedis; 19 | 20 | /** 21 | * Integration tests for secured Redis 22 | * 23 | * @author Oren Elias 24 | */ 25 | @QuarkusIntegrationTest 26 | @TestProfile(RedisSSLStreamTestProfile.class) 27 | public class RedisSSLStreamIT { 28 | 29 | /** 30 | * Verifies that all the records of a PostgreSQL table are streamed to Redis 31 | */ 32 | @Test 33 | public void testRedisStream() throws Exception { 34 | HostAndPort address = HostAndPort.from(RedisSSLTestResourceLifecycleManager.getRedisContainerAddress()); 35 | Jedis jedis = new Jedis(address.getHost(), address.getPort(), true); 36 | final int MESSAGE_COUNT = 4; 37 | final String STREAM_NAME = "testc.inventory.customers"; 38 | final String HASH_NAME = "metadata:debezium:offsets"; 39 | 40 | TestUtils.awaitStreamLengthGte(jedis, STREAM_NAME, MESSAGE_COUNT); 41 | 42 | Long streamLength = jedis.xlen(STREAM_NAME); 43 | assertTrue("Redis Basic Stream Test Failed", streamLength == MESSAGE_COUNT); 44 | 45 | // wait until the offsets are re-written 46 | TestUtils.awaitHashSizeGte(jedis, HASH_NAME, 1); 47 | 48 | Map redisOffsets = jedis.hgetAll(HASH_NAME); 49 | assertTrue(redisOffsets.size() > 0); 50 | 51 | jedis.close(); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /debezium-server-redis/src/test/java/io/debezium/server/redis/RedisSSLStreamTestProfile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.redis; 7 | 8 | import java.net.URL; 9 | import java.util.Arrays; 10 | import java.util.HashMap; 11 | import java.util.List; 12 | import java.util.Map; 13 | 14 | import io.debezium.testing.testcontainers.PostgresTestResourceLifecycleManager; 15 | import io.quarkus.test.junit.QuarkusTestProfile; 16 | 17 | public class RedisSSLStreamTestProfile implements QuarkusTestProfile { 18 | 19 | @Override 20 | public List testResources() { 21 | return Arrays.asList( 22 | new TestResourceEntry(PostgresTestResourceLifecycleManager.class), 23 | new TestResourceEntry(RedisSSLTestResourceLifecycleManager.class)); 24 | } 25 | 26 | public Map getConfigOverrides() { 27 | 28 | Map config = new HashMap(); 29 | URL keyStoreFile = RedisSSLStreamTestProfile.class.getClassLoader().getResource("ssl/client-keystore.p12"); 30 | URL trustStoreFile = RedisSSLStreamTestProfile.class.getClassLoader().getResource("ssl/client-truststore.p12"); 31 | 32 | config.put("javax.net.ssl.keyStore", keyStoreFile.getPath()); 33 | config.put("javax.net.ssl.trustStore", trustStoreFile.getPath()); 34 | config.put("javax.net.ssl.keyStorePassword", "secret"); 35 | config.put("javax.net.ssl.trustStorePassword", "secret"); 36 | config.put("debezium.source.offset.storage", "io.debezium.server.redis.RedisOffsetBackingStore"); 37 | config.put("debezium.source.connector.class", "io.debezium.connector.postgresql.PostgresConnector"); 38 | return config; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /debezium-server-redis/src/test/java/io/debezium/server/redis/RedisSchemaHistoryTestProfile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.redis; 7 | 8 | import java.nio.file.Path; 9 | import java.util.Arrays; 10 | import java.util.HashMap; 11 | import java.util.List; 12 | import java.util.Map; 13 | 14 | import io.debezium.testing.testcontainers.MySqlTestResourceLifecycleManager; 15 | import io.debezium.util.Testing; 16 | import io.quarkus.test.junit.QuarkusTestProfile; 17 | 18 | public class RedisSchemaHistoryTestProfile implements QuarkusTestProfile { 19 | public static final String OFFSETS_FILE = "file-connector-offsets.txt"; 20 | public static final Path OFFSET_STORE_PATH = Testing.Files.createTestingPath(OFFSETS_FILE).toAbsolutePath(); 21 | public static final String OFFSET_STORAGE_FILE_FILENAME_CONFIG = "offset.storage.file.filename"; 22 | 23 | @Override 24 | public List testResources() { 25 | return Arrays.asList(new TestResourceEntry(MySqlTestResourceLifecycleManager.class)); 26 | } 27 | 28 | public Map getConfigOverrides() { 29 | Map config = new HashMap(); 30 | config.put("debezium.source." + OFFSET_STORAGE_FILE_FILENAME_CONFIG, OFFSET_STORE_PATH.toAbsolutePath().toString()); 31 | config.put("debezium.source.schema.history.internal", "io.debezium.server.redis.RedisSchemaHistory"); 32 | config.put("debezium.source.database.server.id", "12345"); 33 | return config; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /debezium-server-redis/src/test/java/io/debezium/server/redis/RedisStreamHeartbeatDisabledTestProfile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.redis; 7 | 8 | import java.util.Map; 9 | 10 | public class RedisStreamHeartbeatDisabledTestProfile extends RedisStreamTestProfile { 11 | @Override 12 | public Map getConfigOverrides() { 13 | Map config = super.getConfigOverrides(); 14 | 15 | // Enable heartbeats with a short interval 16 | config.put("debezium.source.heartbeat.interval.ms", "50"); 17 | 18 | // Set skip.heartbeat.messages to false to allow heartbeat messages to be stored 19 | config.put("debezium.sink.redis.skip.heartbeat.messages", "false"); 20 | 21 | return config; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /debezium-server-redis/src/test/java/io/debezium/server/redis/RedisStreamHeartbeatTestProfile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.redis; 7 | 8 | import java.util.Map; 9 | 10 | public class RedisStreamHeartbeatTestProfile extends RedisStreamTestProfile { 11 | @Override 12 | public Map getConfigOverrides() { 13 | Map config = super.getConfigOverrides(); 14 | 15 | // Enable heartbeats with a short interval 16 | config.put("debezium.source.heartbeat.interval.ms", "50"); 17 | 18 | // Ensure skip.heartbeat.messages is true (which is the default but we explicitly set it for clarity) 19 | config.put("debezium.sink.redis.skip.heartbeat.messages", "true"); 20 | 21 | return config; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /debezium-server-redis/src/test/java/io/debezium/server/redis/RedisStreamMemoryThresholdIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.redis; 7 | 8 | import static org.junit.Assert.assertTrue; 9 | 10 | import org.junit.jupiter.api.Test; 11 | 12 | import io.debezium.connector.postgresql.connection.PostgresConnection; 13 | import io.debezium.util.Testing; 14 | import io.quarkus.test.common.QuarkusTestResource; 15 | import io.quarkus.test.junit.QuarkusIntegrationTest; 16 | import io.quarkus.test.junit.TestProfile; 17 | 18 | import redis.clients.jedis.HostAndPort; 19 | import redis.clients.jedis.Jedis; 20 | 21 | @QuarkusIntegrationTest 22 | @TestProfile(RedisStreamMemoryThresholdTestProfile.class) 23 | @QuarkusTestResource(RedisTestResourceLifecycleManager.class) 24 | public class RedisStreamMemoryThresholdIT { 25 | 26 | @Test 27 | public void testRedisMemoryThreshold() throws Exception { 28 | Testing.Print.enable(); 29 | 30 | Jedis jedis = new Jedis(HostAndPort.from(RedisTestResourceLifecycleManager.getRedisContainerAddress())); 31 | final String STREAM_NAME = "testc.inventory.redis_test2"; 32 | final int TOTAL_RECORDS = 50; 33 | 34 | Testing.print("Setting Redis' maxmemory to 1M"); 35 | jedis.configSet("maxmemory", "1M"); 36 | 37 | PostgresConnection connection = TestUtils.getPostgresConnection(); 38 | connection.execute("CREATE TABLE inventory.redis_test2 " + 39 | "(id VARCHAR(100) PRIMARY KEY, " + 40 | "first_name VARCHAR(100), " + 41 | "last_name VARCHAR(100))"); 42 | connection.execute(String.format("INSERT INTO inventory.redis_test2 (id,first_name,last_name) " + 43 | "SELECT LEFT(i::text, 10), RANDOM()::text, RANDOM()::text FROM generate_series(1,%d) s(i)", TOTAL_RECORDS)); 44 | connection.commit(); 45 | 46 | Thread.sleep(1000); 47 | Testing.print("Entries in " + STREAM_NAME + ":" + jedis.xlen(STREAM_NAME)); 48 | assertTrue(jedis.xlen(STREAM_NAME) < TOTAL_RECORDS); 49 | 50 | Thread.sleep(1000); 51 | jedis.configSet("maxmemory", "0"); 52 | TestUtils.awaitStreamLengthGte(jedis, STREAM_NAME, TOTAL_RECORDS); 53 | 54 | long streamLength = jedis.xlen(STREAM_NAME); 55 | assertTrue("Redis Memory Threshold Test Failed", streamLength == TOTAL_RECORDS); 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /debezium-server-redis/src/test/java/io/debezium/server/redis/RedisStreamMemoryThresholdTestProfile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.redis; 7 | 8 | import java.util.Map; 9 | 10 | public class RedisStreamMemoryThresholdTestProfile extends RedisStreamTestProfile { 11 | 12 | @Override 13 | public Map getConfigOverrides() { 14 | Map config = super.getConfigOverrides(); 15 | config.put("debezium.sink.redis.memory.threshold.percentage", "75"); 16 | return config; 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /debezium-server-redis/src/test/java/io/debezium/server/redis/RedisStreamMessageIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.redis; 7 | 8 | import static org.junit.jupiter.api.Assertions.assertEquals; 9 | import static org.junit.jupiter.api.Assertions.assertTrue; 10 | 11 | import java.util.List; 12 | import java.util.Map; 13 | 14 | import org.junit.jupiter.api.Test; 15 | 16 | import io.debezium.util.Testing; 17 | import io.quarkus.test.common.QuarkusTestResource; 18 | import io.quarkus.test.junit.QuarkusIntegrationTest; 19 | import io.quarkus.test.junit.TestProfile; 20 | 21 | import redis.clients.jedis.HostAndPort; 22 | import redis.clients.jedis.Jedis; 23 | import redis.clients.jedis.StreamEntryID; 24 | import redis.clients.jedis.resps.StreamEntry; 25 | 26 | /** 27 | * Integration tests that verify basic reading from PostgreSQL database and writing to Redis stream 28 | * 29 | * @author ggaborg 30 | */ 31 | @QuarkusIntegrationTest 32 | @TestProfile(RedisStreamMessageTestProfile.class) 33 | @QuarkusTestResource(RedisTestResourceLifecycleManager.class) 34 | public class RedisStreamMessageIT { 35 | 36 | /** 37 | * Verifies that all the records of a PostgreSQL table are streamed to Redis in extended message format 38 | */ 39 | @Test 40 | public void testRedisStreamExtendedMessage() { 41 | Testing.Print.enable(); 42 | 43 | Jedis jedis = new Jedis(HostAndPort.from(RedisTestResourceLifecycleManager.getRedisContainerAddress())); 44 | final int MESSAGE_COUNT = 4; 45 | final String STREAM_NAME = "testc.inventory.customers"; 46 | 47 | TestUtils.awaitStreamLengthGte(jedis, STREAM_NAME, MESSAGE_COUNT); 48 | 49 | long streamLength = jedis.xlen(STREAM_NAME); 50 | assertEquals(MESSAGE_COUNT, streamLength, "Expected stream length of " + MESSAGE_COUNT); 51 | 52 | final List entries = jedis.xrange(STREAM_NAME, (StreamEntryID) null, (StreamEntryID) null); 53 | for (StreamEntry entry : entries) { 54 | Map map = entry.getFields(); 55 | assertEquals(3, map.size(), "Expected map of size 3"); 56 | assertTrue(map.get("key") != null && map.get("key").startsWith("{\"schema\":"), "Expected key's value starting with {\"schema\":..."); 57 | assertTrue(map.get("value") != null && map.get("value").startsWith("{\"schema\":"), "Expected value's value starting with {\"schema\":..."); 58 | assertTrue(map.containsKey("HEADERKEY")); 59 | } 60 | 61 | jedis.close(); 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /debezium-server-redis/src/test/java/io/debezium/server/redis/RedisStreamMessageTestProfile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.redis; 7 | 8 | import java.util.Map; 9 | 10 | public class RedisStreamMessageTestProfile extends RedisStreamTestProfile { 11 | 12 | @Override 13 | public Map getConfigOverrides() { 14 | Map config = super.getConfigOverrides(); 15 | config.put("debezium.sink.redis.message.format", "extended"); 16 | return config; 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /debezium-server-redis/src/test/java/io/debezium/server/redis/RedisStreamTestProfile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.redis; 7 | 8 | import java.nio.file.Path; 9 | import java.util.Arrays; 10 | import java.util.HashMap; 11 | import java.util.List; 12 | import java.util.Map; 13 | 14 | import io.debezium.testing.testcontainers.PostgresTestResourceLifecycleManager; 15 | import io.debezium.util.Testing; 16 | import io.quarkus.test.junit.QuarkusTestProfile; 17 | 18 | public class RedisStreamTestProfile implements QuarkusTestProfile { 19 | 20 | public static final String OFFSETS_FILE = "file-connector-offsets.txt"; 21 | public static final Path OFFSET_STORE_PATH = Testing.Files.createTestingPath(OFFSETS_FILE).toAbsolutePath(); 22 | public static final String OFFSET_STORAGE_FILE_FILENAME_CONFIG = "offset.storage.file.filename"; 23 | 24 | @Override 25 | public List testResources() { 26 | return Arrays.asList(new TestResourceEntry(PostgresTestResourceLifecycleManager.class)); 27 | } 28 | 29 | public Map getConfigOverrides() { 30 | Map config = new HashMap(); 31 | config.put("debezium.source.connector.class", "io.debezium.connector.postgresql.PostgresConnector"); 32 | config.put("debezium.source." + OFFSET_STORAGE_FILE_FILENAME_CONFIG, OFFSET_STORE_PATH.toAbsolutePath().toString()); 33 | config.put("debezium.sink.redis.memory.threshold.percentage", "0"); 34 | return config; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /debezium-server-redis/src/test/java/io/debezium/server/redis/TestUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.redis; 7 | 8 | import java.time.Duration; 9 | import java.util.function.Supplier; 10 | 11 | import org.awaitility.Awaitility; 12 | 13 | import io.debezium.connector.postgresql.connection.PostgresConnection; 14 | import io.debezium.jdbc.JdbcConfiguration; 15 | import io.debezium.server.TestConfigSource; 16 | import io.debezium.testing.testcontainers.PostgresTestResourceLifecycleManager; 17 | 18 | import redis.clients.jedis.Jedis; 19 | import redis.clients.jedis.exceptions.JedisConnectionException; 20 | 21 | public class TestUtils { 22 | 23 | private TestUtils() { 24 | } 25 | 26 | public static PostgresConnection getPostgresConnection() { 27 | return new PostgresConnection(JdbcConfiguration.create() 28 | .with("user", PostgresTestResourceLifecycleManager.POSTGRES_USER) 29 | .with("password", PostgresTestResourceLifecycleManager.POSTGRES_PASSWORD) 30 | .with("dbname", PostgresTestResourceLifecycleManager.POSTGRES_DBNAME) 31 | .with("hostname", PostgresTestResourceLifecycleManager.POSTGRES_HOST) 32 | .with("port", PostgresTestResourceLifecycleManager.getContainer().getMappedPort(PostgresTestResourceLifecycleManager.POSTGRES_PORT)) 33 | .build(), "Debezium Redis Test"); 34 | } 35 | 36 | public static void awaitStreamLengthGte(Jedis jedis, String streamName, int expectedLength) { 37 | waitBoolean(() -> jedis.xlen(streamName) >= expectedLength); 38 | } 39 | 40 | public static void awaitHashSizeGte(Jedis jedis, String hashName, int expectedSize) { 41 | waitBoolean(() -> { 42 | try { 43 | return jedis.hgetAll(hashName).size() >= expectedSize; 44 | } 45 | catch (JedisConnectionException e) { 46 | return false; 47 | } 48 | }); 49 | } 50 | 51 | public static void waitBoolean(Supplier bool) { 52 | Awaitility.await().atMost(Duration.ofSeconds(TestConfigSource.waitForSeconds())).until(() -> { 53 | return Boolean.TRUE.equals(bool.get()); 54 | }); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /debezium-server-redis/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-redis/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource -------------------------------------------------------------------------------- /debezium-server-redis/src/test/resources/ssl/ca.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDQDCCAiigAwIBAgIEZDf2TTANBgkqhkiG9w0BAQsFADA1MRMwEQYDVQQKDApS 3 | ZWRpcyBUZXN0MR4wHAYDVQQDDBVDZXJ0aWZpY2F0ZSBBdXRob3JpdHkwHhcNMjMw 4 | NDEzMTIzMjEzWhcNNDMwNDEzMTIzMjEzWjA1MRMwEQYDVQQKDApSZWRpcyBUZXN0 5 | MR4wHAYDVQQDDBVDZXJ0aWZpY2F0ZSBBdXRob3JpdHkwggEiMA0GCSqGSIb3DQEB 6 | AQUAA4IBDwAwggEKAoIBAQDzpg547h5TOzxBcO7fOgIA/57sh66xRljOWEEbgchD 7 | j3sGHVRO+yxijUCthLQHNrAa2V/9aOvJ8bKHLc/ABls8hu7Yl3svkXtPLkJKZhKk 8 | jx3F+E3Elm2gBQNn7+uvfVjo9Ycwhs1M0L6c7CibJdECagg4GinE4H3vEQQFw2UF 9 | rHJqHNvmwdv1fN3p/8wtDHpYSgDr7lepwCzDqlzlFoYXqiYQuqeCdSKMnsEMJbPa 10 | 1P47SsEw3xd4M9R+TnOtjkz1Z8ZAJscWOUfSllWLYtH7IyGHqSlKHrsCZa9XIWis 11 | WxfqWT/R5dlfpg5WzfBGWHDnWLrU+Er1a2pPGq7XhXP9AgMBAAGjWDBWMAwGA1Ud 12 | EwQFMAMBAf8wJwYDVR0jBCAwHoAUEstaaii7yKEFDcyReBcHYVYlmkChAIIEZDf2 13 | TTAdBgNVHQ4EFgQUEstaaii7yKEFDcyReBcHYVYlmkAwDQYJKoZIhvcNAQELBQAD 14 | ggEBAOWmtSgHtP7QsL28+2wDN00YsJdRGIWSaEVIS7zZOlToIAoa2zIaPdv/V740 15 | aEbZ19E/Z+/iWQbRuZoIHw5zqo5UBP3qcJzAIpcwPQu9R0mIQMMDaL2j0Eyao8U7 16 | YU9hRNNdUUFjvp3vPOFxtNG1WK5QeiiKW1aIa+VIGro/zONypkWhrvg43fLz2onD 17 | BFVjNduHjlz4B4/Y1anFYehS/7gVSlyU096UftQexjfGSYNoeZNCV6KDuU5hQLIN 18 | Y7Tquts2SUe5G51DFOehoyxmLGBchP8nbJjHf+7kFMP8poqSy0Arwq1gGlduZXXm 19 | 8NA4P+2ZMwEo0Lh+ynXtWu9TJbY= 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /debezium-server-redis/src/test/resources/ssl/client-keystore.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-redis/src/test/resources/ssl/client-keystore.p12 -------------------------------------------------------------------------------- /debezium-server-redis/src/test/resources/ssl/client-truststore.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-redis/src/test/resources/ssl/client-truststore.p12 -------------------------------------------------------------------------------- /debezium-server-redis/src/test/resources/ssl/keys.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-redis/src/test/resources/ssl/keys.p12 -------------------------------------------------------------------------------- /debezium-server-redis/src/test/resources/ssl/redis.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIC2DCCAcACBGQ39vYwDQYJKoZIhvcNAQELBQAwNTETMBEGA1UECgwKUmVkaXMg 3 | VGVzdDEeMBwGA1UEAwwVQ2VydGlmaWNhdGUgQXV0aG9yaXR5MB4XDTIzMDQxMzEy 4 | MzUwMloXDTQzMDQxMzEyMzUwMlowLDETMBEGA1UECgwKUmVkaXMgVGVzdDEVMBMG 5 | A1UEAwwMR2VuZXJpYy1jZXJ0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC 6 | AQEAicMUB3xNFlUohdqvTYiX+GJBPpwQnbdpUIqUtHDHjdM+c4wWxLhpsD5/gVmG 7 | WaUkBd4Hyfu9ck9yxeR4QX2OebALNzGVOt7J4sGs1QuSx3o4ma0huNybJHOviRCN 8 | ekONh3NGxP/zgPIqmkAgkHx8IzvdYwo6VVQGOh8pMirWFEWwtYwjNJG5d4l2dBlB 9 | VZ8mqqzsHkdXztJRe0j6aMU1GSRjaAQUFTuinl/Jg8Nfvd3YU4xrMq5C7Py7E1Hy 10 | ORZ/MXCmWAH19h2gqi4+Wti3PpetGfExD+LgmmCKt64QLe7PVPgqnfkUHnb8G/ic 11 | 6GmgptlWAcbZPiTjNTqUepxV9wIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQA5T42e 12 | vhoxanjO9+bznT8Ri9btI+uOoBK68nZppd/5k9rC9Yyva1hrO0gLDPn1d0E9KDGu 13 | abJ/c7mNtCFYvGz0U6dU7NnNUjMVB60xUaHMIRcM7qNI21HqRScpQ5TT+RZ6MqX2 14 | qGXWpg7n1it4CXj0VN8C0TSDo0wbeSVtqWzPoMRLQ92QJQ60uForby8MRGMhSVcL 15 | Co5riUpPdvA/XrG5WMY81rt6bDQP9KuU7w4yXrDnVZCNt+AiBoFRcc/1o4Ti/Tjx 16 | YpOxELVZpt+kRs4/Lr8Iyacwofy3sbvz2JhVPs5rU8boA+uffnih4Ut3lr1fSH7C 17 | dHSjZtSFLqp1QIn2 18 | -----END CERTIFICATE----- 19 | -------------------------------------------------------------------------------- /debezium-server-redis/src/test/resources/ssl/redis.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEAicMUB3xNFlUohdqvTYiX+GJBPpwQnbdpUIqUtHDHjdM+c4wW 3 | xLhpsD5/gVmGWaUkBd4Hyfu9ck9yxeR4QX2OebALNzGVOt7J4sGs1QuSx3o4ma0h 4 | uNybJHOviRCNekONh3NGxP/zgPIqmkAgkHx8IzvdYwo6VVQGOh8pMirWFEWwtYwj 5 | NJG5d4l2dBlBVZ8mqqzsHkdXztJRe0j6aMU1GSRjaAQUFTuinl/Jg8Nfvd3YU4xr 6 | Mq5C7Py7E1HyORZ/MXCmWAH19h2gqi4+Wti3PpetGfExD+LgmmCKt64QLe7PVPgq 7 | nfkUHnb8G/ic6GmgptlWAcbZPiTjNTqUepxV9wIDAQABAoIBAAyYtlWi44eVvNFU 8 | eDAOO9sd3NplOc26DhqruUIwh8armKvHFPPi6uMPK+h/b5310C/rJ+orpxxzMErx 9 | A5/1Y5gz5SqkUYFjCrG9xNhsKP0ta0x9/LXjEhqIq3XMeitCTwgiHcf4oWLagdO9 10 | 5lxfzZgJKsqaDlBsEO+ylrk8Gnwa6twTYK7ToYURjMPmNScyoXmhh6Hz+FvUDuWQ 11 | J0+Mzw+JPbUtCgItHzQ6RJZK/c+h0Nl3A2Nytm+RbRIOqEjhlcqh+zPLB1vXTOFm 12 | R47V+DgGR2yra0R3/fd65rDQkNXApJeNoiV0gSpJIzKPS0OnnWfmt7cVnASF/Hzs 13 | Z2pOgKECgYEAwZUPzy/IftZm7xs7lz14BY0zynrc/kgSjqj2b5LHubiy54fsjTT0 14 | v5JzOnhrrCkIme5MPGjCLv2iNPnvCyVrRn0oSZt4jjpswp8EhmJ3VLGulCVRPJlF 15 | WNAlZ457bYkUjN3fRdPecLqg0VFd3zTnMj34SxfTJOqdXHHDASpkg+ECgYEAti5s 16 | cldZvHfzI0yFXBznbSVKbVbK57UVIKExNGeNvi0aLM3XAv1y+qflbrPesUTrcAY0 17 | VY8SKvL4J6rLs6PluCD+W1YbApPedilGFim+BWZKrBtmZG/7j1+8S07+fTUj+tC2 18 | 3S536ZEQcCOXOWDEEW3D5w8P1MmxZD/k1sJkFNcCgYARyOUT7trntEDutLzLz0zo 19 | jal1b8Y/4lU6IBHL/Fs3F93v4Y/9BSF7PDQz6f5Sac6tfbQpVKKKeKjfXxr0FjL9 20 | 3d/Gq27u+jUlyEA9LGy0LkLQv4DwJVC2sSDOm8uBK7fknTm3C5kDGFJQCL6sFAst 21 | lX3rQDwYxyk/5f+5arCjYQKBgFnoaT64myGm9MgL4JjH7vJvf96cAcD+HfIfeuAI 22 | 5uve6FcKMr1YBWFscI8El7dWcvazKSq8U9P8WrmA0czS6vDG36CFywinnxcjYWJY 23 | sV2K0PlFAHA6z6Q/M/bagj+adSo3zLc1UppjLACbn+sCNWEAkbi7Ny3lZ/U4jx/p 24 | iWi7AoGBAIfYsB2gYot1/PcculT1PIoZCp0fuM+lIQEzXkzBP7hXkra4Q4+qipv8 25 | Og8OCh6ba0XMAsPcAXvW38nlWieNRdjI8OXRGlWOrKItbUSRw1MgNFKMDPwKdu12 26 | WF8Ya/4uPB3LzM19hPFYg9feCBA9Q4JOxiI5+09N4dq1OLbgDI6y 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /debezium-server-redis/src/test/resources/testcontainers.properties: -------------------------------------------------------------------------------- 1 | hub.image.name.prefix=mirror.gcr.io/ 2 | -------------------------------------------------------------------------------- /debezium-server-rocketmq/src/main/resources/META-INF/beans.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-rocketmq/src/main/resources/META-INF/beans.xml -------------------------------------------------------------------------------- /debezium-server-rocketmq/src/test/java/io/debezium/server/rocketmq/RocketMqTestConfigSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.rocketmq; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; 12 | 13 | import io.debezium.server.TestConfigSource; 14 | 15 | public class RocketMqTestConfigSource extends TestConfigSource { 16 | 17 | public static final String TOPIC_NAME = "testc-inventory-customers"; 18 | 19 | public RocketMqTestConfigSource() { 20 | 21 | final Map rocketmqConfig = new HashMap<>(); 22 | rocketmqConfig.put("debezium.sink.type", "rocketmq"); 23 | rocketmqConfig.put("debezium.source.connector.class", "io.debezium.connector.postgresql.PostgresConnector"); 24 | rocketmqConfig.put("debezium.source." + StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG, OFFSET_STORE_PATH.toAbsolutePath().toString()); 25 | rocketmqConfig.put("debezium.source.offset.flush.interval.ms", "0"); 26 | rocketmqConfig.put("debezium.source.topic.prefix", "testc"); 27 | rocketmqConfig.put("debezium.source.schema.include.list", "inventory"); 28 | rocketmqConfig.put("debezium.source.table.include.list", "inventory.customers"); 29 | rocketmqConfig.put("debezium.transforms", "Reroute,addheader"); 30 | rocketmqConfig.put("debezium.transforms.Reroute.type", "io.debezium.transforms.ByLogicalTableRouter"); 31 | rocketmqConfig.put("debezium.transforms.Reroute.topic.regex", "(.*)"); 32 | rocketmqConfig.put("debezium.transforms.Reroute.topic.replacement", TOPIC_NAME); 33 | rocketmqConfig.put("debezium.transforms.addheader.type", "org.apache.kafka.connect.transforms.InsertHeader"); 34 | rocketmqConfig.put("debezium.transforms.addheader.header", "headerKey"); 35 | rocketmqConfig.put("debezium.transforms.addheader.value.literal", "headerValue"); 36 | config = rocketmqConfig; 37 | } 38 | 39 | @Override 40 | public int getOrdinal() { 41 | // Configuration property precedence is based on ordinal values and since we override the 42 | // properties in TestConfigSource, we should give this a higher priority. 43 | return super.getOrdinal() + 1; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /debezium-server-rocketmq/src/test/java/io/debezium/server/rocketmq/RocketMqTestResourceLifecycleManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.rocketmq; 7 | 8 | import java.util.Map; 9 | import java.util.concurrent.ConcurrentHashMap; 10 | import java.util.concurrent.atomic.AtomicBoolean; 11 | 12 | import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; 13 | 14 | /** 15 | * Manages the lifecycle of a RocketMQ cluster test resource. 16 | */ 17 | public class RocketMqTestResourceLifecycleManager implements QuarkusTestResourceLifecycleManager { 18 | 19 | public static RocketMqContainer container = new RocketMqContainer(); 20 | private static final AtomicBoolean running = new AtomicBoolean(false); 21 | 22 | private static synchronized void init() { 23 | if (!running.get()) { 24 | container.start(); 25 | running.set(true); 26 | } 27 | } 28 | 29 | @Override 30 | public Map start() { 31 | init(); 32 | Map params = new ConcurrentHashMap<>(); 33 | params.put("debezium.sink.rocketmq.producer.name.srv.addr", getNameSrvAddr()); 34 | params.put("debezium.sink.rocketmq.producer.group", "producer-group"); 35 | return params; 36 | } 37 | 38 | @Override 39 | public void stop() { 40 | try { 41 | if (container != null) { 42 | container.stop(); 43 | } 44 | } 45 | catch (Exception e) { 46 | // ignored 47 | } 48 | } 49 | 50 | public static String getNameSrvAddr() { 51 | return container.getNamesrvAddr(); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /debezium-server-rocketmq/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- 1 | io.debezium.server.rocketmq.RocketMqTestConfigSource -------------------------------------------------------------------------------- /debezium-server-rocketmq/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | %d{ISO8601} %-5p %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext} %m [%c]%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 16 | 17 | 18 | 19 | 22 | 23 | 24 | 27 | 28 | 29 | 32 | 33 | 34 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /debezium-server-sqs/src/main/resources/META-INF/beans.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/debezium/debezium-server/4dea9a90cad5d44e489e09360eb9751e9b6a3a40/debezium-server-sqs/src/main/resources/META-INF/beans.xml -------------------------------------------------------------------------------- /debezium-server-sqs/src/test/java/io/debezium/server/sqs/SqsIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.sqs; 7 | 8 | import static io.debezium.server.sqs.SqsTestResourceLifecycleManager.getQueueUrl; 9 | import static io.debezium.server.sqs.SqsTestResourceLifecycleManager.sqsClient; 10 | 11 | import jakarta.enterprise.event.Observes; 12 | 13 | import org.junit.jupiter.api.Test; 14 | 15 | import io.debezium.server.events.ConnectorCompletedEvent; 16 | import io.debezium.testing.testcontainers.PostgresTestResourceLifecycleManager; 17 | import io.debezium.util.Testing; 18 | import io.quarkus.test.common.QuarkusTestResource; 19 | import io.quarkus.test.junit.QuarkusTest; 20 | 21 | import software.amazon.awssdk.services.sqs.model.Message; 22 | import software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest; 23 | import software.amazon.awssdk.services.sqs.model.ReceiveMessageResponse; 24 | 25 | /** 26 | * Integration test that verifies basic reading from PostgreSQL database and sending to SQS. 27 | * 28 | * @author V K 29 | */ 30 | @QuarkusTest 31 | @QuarkusTestResource(PostgresTestResourceLifecycleManager.class) 32 | @QuarkusTestResource(SqsTestResourceLifecycleManager.class) 33 | public class SqsIT { 34 | 35 | private static final int MESSAGE_COUNT = 4; 36 | 37 | { 38 | Testing.Files.delete(SqsTestConfigSource.OFFSET_STORE_PATH); 39 | Testing.Files.createTestingFile(SqsTestConfigSource.OFFSET_STORE_PATH); 40 | } 41 | 42 | void connectorCompleted(@Observes ConnectorCompletedEvent event) throws Exception { 43 | if (!event.isSuccess()) { 44 | throw (Exception) event.getError().get(); 45 | } 46 | } 47 | 48 | @Test 49 | public void testSqs() throws Exception { 50 | Testing.Print.enable(); 51 | 52 | final java.util.List messages = new java.util.ArrayList<>(); 53 | org.awaitility.Awaitility.await().atMost(java.time.Duration.ofSeconds(SqsTestConfigSource.waitForSeconds())).until(() -> { 54 | final ReceiveMessageResponse receiveMessageResponse = sqsClient() 55 | .receiveMessage(ReceiveMessageRequest.builder().queueUrl(getQueueUrl()).waitTimeSeconds(3).maxNumberOfMessages(MESSAGE_COUNT).build()); 56 | messages.addAll(receiveMessageResponse.messages()); 57 | 58 | return messages.size() >= MESSAGE_COUNT; 59 | }); 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /debezium-server-sqs/src/test/java/io/debezium/server/sqs/SqsTestConfigSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server.sqs; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; 12 | 13 | import io.debezium.server.TestConfigSource; 14 | 15 | /** 16 | * @author V K 17 | */ 18 | public class SqsTestConfigSource extends TestConfigSource { 19 | public SqsTestConfigSource() { 20 | Map sqsConfigMap = new HashMap<>(); 21 | 22 | sqsConfigMap.put("debezium.source.connector.class", "io.debezium.connector.postgresql.PostgresConnector"); 23 | sqsConfigMap.put("debezium.source." + StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG, OFFSET_STORE_PATH.toAbsolutePath().toString()); 24 | sqsConfigMap.put("debezium.source.offset.flush.interval.ms", "0"); 25 | sqsConfigMap.put("debezium.source.topic.prefix", "testc"); 26 | sqsConfigMap.put("debezium.source.schema.include.list", "inventory"); 27 | sqsConfigMap.put("debezium.source.table.include.list", "inventory.customers"); 28 | 29 | config = sqsConfigMap; 30 | } 31 | 32 | @Override 33 | public int getOrdinal() { 34 | // Configuration property precedence is based on ordinal values and since we override the 35 | // properties in TestConfigSource, we should give this a higher priority. 36 | return super.getOrdinal() + 1; 37 | } 38 | 39 | public static int waitForSeconds() { 40 | return 60; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /debezium-server-sqs/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- 1 | io.debezium.server.sqs.SqsTestConfigSource -------------------------------------------------------------------------------- /debezium-server-sqs/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | %d{ISO8601} %-5p %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext} %m [%c]%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 16 | 17 | 18 | 19 | 22 | 23 | 24 | 27 | 28 | 29 | 32 | 33 | 34 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /debezium-server-sqs/src/test/resources/testcontainers.properties: -------------------------------------------------------------------------------- 1 | hub.image.name.prefix=mirror.gcr.io/ 2 | -------------------------------------------------------------------------------- /debezium-system-tests/src/test/java/io/debezium/server/TestConsumer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Debezium Authors. 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package io.debezium.server; 7 | 8 | import java.util.ArrayList; 9 | import java.util.Collections; 10 | import java.util.List; 11 | 12 | import jakarta.annotation.PostConstruct; 13 | import jakarta.annotation.PreDestroy; 14 | import jakarta.enterprise.context.Dependent; 15 | import jakarta.inject.Named; 16 | 17 | import io.debezium.DebeziumException; 18 | import io.debezium.engine.ChangeEvent; 19 | import io.debezium.engine.DebeziumEngine; 20 | import io.debezium.engine.DebeziumEngine.RecordCommitter; 21 | import io.debezium.util.Testing; 22 | 23 | @Dependent 24 | @Named("test") 25 | public class TestConsumer implements DebeziumEngine.ChangeConsumer> { 26 | 27 | final List values = Collections.synchronizedList(new ArrayList<>()); 28 | 29 | @PostConstruct 30 | void init() { 31 | Testing.print("Test consumer constructed"); 32 | } 33 | 34 | @PreDestroy 35 | void close() { 36 | Testing.print("Test consumer destroyed"); 37 | } 38 | 39 | @Override 40 | public void handleBatch(List> records, RecordCommitter> committer) { 41 | 42 | records.forEach(record -> { 43 | Testing.print(record); 44 | values.add(record.value()); 45 | try { 46 | committer.markProcessed(record); 47 | } 48 | catch (InterruptedException e) { 49 | throw new DebeziumException(e); 50 | } 51 | }); 52 | } 53 | 54 | public List getValues() { 55 | return values; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /debezium-system-tests/src/test/resources/META-INF/services/org.eclipse.microprofile.config.spi.ConfigSource: -------------------------------------------------------------------------------- 1 | io.debezium.server.TestConfigSource -------------------------------------------------------------------------------- /debezium-system-tests/src/test/resources/application.properties: -------------------------------------------------------------------------------- 1 | quarkus.kubernetes-client.devservices.enabled=false -------------------------------------------------------------------------------- /debezium-system-tests/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | %d{ISO8601} %-5p %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext} %m [%c]%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 16 | 17 | 18 | 19 | 22 | 23 | 24 | 27 | 28 | 29 | 32 | 33 | 34 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /debezium-system-tests/src/test/resources/testcontainers.properties: -------------------------------------------------------------------------------- 1 | hub.image.name.prefix=mirror.gcr.io/ 2 | --------------------------------------------------------------------------------