├── .github ├── pull_request_template.md └── workflows │ ├── build.yml │ ├── prepare-release.yml │ └── publish.yml ├── .gitignore ├── .gitlab-ci.yml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── pom.xml └── src ├── main ├── assembly │ └── src.xml └── java │ └── com │ └── getindata │ └── kafka │ └── connect │ └── iceberg │ └── sink │ ├── IcebergCatalogFactory.java │ ├── IcebergChangeConsumer.java │ ├── IcebergChangeEvent.java │ ├── IcebergSink.java │ ├── IcebergSinkConfiguration.java │ ├── IcebergSinkTask.java │ ├── IcebergSinkVersion.java │ ├── IcebergUtil.java │ ├── converter │ ├── DeserializerFactory.java │ ├── ExtractNewRecordStateTransformationFactory.java │ ├── JsonConverterFactory.java │ ├── SinkRecordToIcebergChangeEventConverter.java │ └── SinkRecordToIcebergChangeEventConverterFactory.java │ └── tableoperator │ ├── BaseDeltaTaskWriter.java │ ├── CdcOperation.java │ ├── IcebergTableOperator.java │ ├── IcebergTableOperatorFactory.java │ ├── IcebergTableWriterFactory.java │ ├── PartitionedAppendWriter.java │ ├── PartitionedDeltaWriter.java │ └── UnpartitionedDeltaWriter.java └── test ├── java └── com │ └── getindata │ └── kafka │ └── connect │ └── iceberg │ └── sink │ ├── IcebergSinkSystemTest.java │ ├── TestIcebergUtil.java │ ├── tableoperator │ └── IcebergTableOperatorTest.java │ ├── testcontainers │ ├── DebeziumConnectContainer.java │ ├── KafkaContainer.java │ ├── PostgresContainer.java │ ├── S3MinioContainer.java │ └── SchemaRegistryContainer.java │ └── testresources │ ├── IcebergChangeEventBuilder.java │ ├── MinioTestHelper.java │ ├── PostgresTestHelper.java │ ├── SparkTestHelper.java │ └── TestConfig.java └── resources └── json ├── custom-partition-column.json ├── debezium-annotated-schema.json ├── debezium-metadata-schema.json ├── serde-update.json ├── serde-with-array.json ├── serde-with-array2.json ├── serde-with-schema.json ├── serde-with-schema2.json ├── serde-with-schema_geom.json └── unwrap-with-schema.json /.github/pull_request_template.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/.github/pull_request_template.md -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/.github/workflows/build.yml -------------------------------------------------------------------------------- /.github/workflows/prepare-release.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/.github/workflows/prepare-release.yml -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/.github/workflows/publish.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/.gitignore -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/.gitlab-ci.yml -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/CHANGELOG.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/README.md -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/pom.xml -------------------------------------------------------------------------------- /src/main/assembly/src.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/assembly/src.xml -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/IcebergCatalogFactory.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/IcebergCatalogFactory.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/IcebergChangeConsumer.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/IcebergChangeConsumer.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/IcebergChangeEvent.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/IcebergChangeEvent.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/IcebergSink.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/IcebergSink.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/IcebergSinkConfiguration.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/IcebergSinkConfiguration.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/IcebergSinkTask.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/IcebergSinkTask.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/IcebergSinkVersion.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/IcebergSinkVersion.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/IcebergUtil.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/IcebergUtil.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/converter/DeserializerFactory.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/converter/DeserializerFactory.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/converter/ExtractNewRecordStateTransformationFactory.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/converter/ExtractNewRecordStateTransformationFactory.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/converter/JsonConverterFactory.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/converter/JsonConverterFactory.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/converter/SinkRecordToIcebergChangeEventConverter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/converter/SinkRecordToIcebergChangeEventConverter.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/converter/SinkRecordToIcebergChangeEventConverterFactory.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/converter/SinkRecordToIcebergChangeEventConverterFactory.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/tableoperator/BaseDeltaTaskWriter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/tableoperator/BaseDeltaTaskWriter.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/tableoperator/CdcOperation.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/tableoperator/CdcOperation.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/tableoperator/IcebergTableOperator.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/tableoperator/IcebergTableOperator.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/tableoperator/IcebergTableOperatorFactory.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/tableoperator/IcebergTableOperatorFactory.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/tableoperator/IcebergTableWriterFactory.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/tableoperator/IcebergTableWriterFactory.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/tableoperator/PartitionedAppendWriter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/tableoperator/PartitionedAppendWriter.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/tableoperator/PartitionedDeltaWriter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/tableoperator/PartitionedDeltaWriter.java -------------------------------------------------------------------------------- /src/main/java/com/getindata/kafka/connect/iceberg/sink/tableoperator/UnpartitionedDeltaWriter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/main/java/com/getindata/kafka/connect/iceberg/sink/tableoperator/UnpartitionedDeltaWriter.java -------------------------------------------------------------------------------- /src/test/java/com/getindata/kafka/connect/iceberg/sink/IcebergSinkSystemTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/java/com/getindata/kafka/connect/iceberg/sink/IcebergSinkSystemTest.java -------------------------------------------------------------------------------- /src/test/java/com/getindata/kafka/connect/iceberg/sink/TestIcebergUtil.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/java/com/getindata/kafka/connect/iceberg/sink/TestIcebergUtil.java -------------------------------------------------------------------------------- /src/test/java/com/getindata/kafka/connect/iceberg/sink/tableoperator/IcebergTableOperatorTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/java/com/getindata/kafka/connect/iceberg/sink/tableoperator/IcebergTableOperatorTest.java -------------------------------------------------------------------------------- /src/test/java/com/getindata/kafka/connect/iceberg/sink/testcontainers/DebeziumConnectContainer.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/java/com/getindata/kafka/connect/iceberg/sink/testcontainers/DebeziumConnectContainer.java -------------------------------------------------------------------------------- /src/test/java/com/getindata/kafka/connect/iceberg/sink/testcontainers/KafkaContainer.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/java/com/getindata/kafka/connect/iceberg/sink/testcontainers/KafkaContainer.java -------------------------------------------------------------------------------- /src/test/java/com/getindata/kafka/connect/iceberg/sink/testcontainers/PostgresContainer.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/java/com/getindata/kafka/connect/iceberg/sink/testcontainers/PostgresContainer.java -------------------------------------------------------------------------------- /src/test/java/com/getindata/kafka/connect/iceberg/sink/testcontainers/S3MinioContainer.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/java/com/getindata/kafka/connect/iceberg/sink/testcontainers/S3MinioContainer.java -------------------------------------------------------------------------------- /src/test/java/com/getindata/kafka/connect/iceberg/sink/testcontainers/SchemaRegistryContainer.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/java/com/getindata/kafka/connect/iceberg/sink/testcontainers/SchemaRegistryContainer.java -------------------------------------------------------------------------------- /src/test/java/com/getindata/kafka/connect/iceberg/sink/testresources/IcebergChangeEventBuilder.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/java/com/getindata/kafka/connect/iceberg/sink/testresources/IcebergChangeEventBuilder.java -------------------------------------------------------------------------------- /src/test/java/com/getindata/kafka/connect/iceberg/sink/testresources/MinioTestHelper.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/java/com/getindata/kafka/connect/iceberg/sink/testresources/MinioTestHelper.java -------------------------------------------------------------------------------- /src/test/java/com/getindata/kafka/connect/iceberg/sink/testresources/PostgresTestHelper.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/java/com/getindata/kafka/connect/iceberg/sink/testresources/PostgresTestHelper.java -------------------------------------------------------------------------------- /src/test/java/com/getindata/kafka/connect/iceberg/sink/testresources/SparkTestHelper.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/java/com/getindata/kafka/connect/iceberg/sink/testresources/SparkTestHelper.java -------------------------------------------------------------------------------- /src/test/java/com/getindata/kafka/connect/iceberg/sink/testresources/TestConfig.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/java/com/getindata/kafka/connect/iceberg/sink/testresources/TestConfig.java -------------------------------------------------------------------------------- /src/test/resources/json/custom-partition-column.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/resources/json/custom-partition-column.json -------------------------------------------------------------------------------- /src/test/resources/json/debezium-annotated-schema.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/resources/json/debezium-annotated-schema.json -------------------------------------------------------------------------------- /src/test/resources/json/debezium-metadata-schema.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/resources/json/debezium-metadata-schema.json -------------------------------------------------------------------------------- /src/test/resources/json/serde-update.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/resources/json/serde-update.json -------------------------------------------------------------------------------- /src/test/resources/json/serde-with-array.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/resources/json/serde-with-array.json -------------------------------------------------------------------------------- /src/test/resources/json/serde-with-array2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/resources/json/serde-with-array2.json -------------------------------------------------------------------------------- /src/test/resources/json/serde-with-schema.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/resources/json/serde-with-schema.json -------------------------------------------------------------------------------- /src/test/resources/json/serde-with-schema2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/resources/json/serde-with-schema2.json -------------------------------------------------------------------------------- /src/test/resources/json/serde-with-schema_geom.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/resources/json/serde-with-schema_geom.json -------------------------------------------------------------------------------- /src/test/resources/json/unwrap-with-schema.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getindata/kafka-connect-iceberg-sink/HEAD/src/test/resources/json/unwrap-with-schema.json --------------------------------------------------------------------------------