├── .github ├── dependabot.yml └── workflows │ ├── ci.yml │ ├── publish.yml │ └── release.yml ├── .gitignore ├── LICENSE ├── README.md ├── gradle.properties ├── gradle ├── checkstyle.gradle ├── config │ └── checkstyle │ │ └── suppressions.xml ├── jacoco.gradle ├── publish.gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── integration-tests ├── build.gradle └── src │ ├── avro │ └── test-event-with-all-spark-compatible-types.avsc │ └── integration-test │ ├── java │ └── com │ │ └── github │ │ └── tenxfuturetechnologies │ │ └── kafkaconnecticeberg │ │ ├── containers │ │ ├── KafkaConnectContainer.java │ │ ├── KafkaContainer.java │ │ ├── MinioContainer.java │ │ └── SchemaRegistryContainer.java │ │ ├── sink │ │ ├── AbstractBaseIT.java │ │ └── IcebergS3SinkAvroIT.java │ │ └── util │ │ ├── Constants.java │ │ ├── MinioTestHelper.java │ │ ├── SinkConfigBuilder.java │ │ └── SparkTestHelper.java │ └── resources │ └── logback-test.xml ├── settings.gradle └── sink ├── build.gradle └── src ├── main ├── java │ └── com │ │ └── github │ │ └── tenxfuturetechnologies │ │ └── kafkaconnecticeberg │ │ └── sink │ │ ├── IcebergSinkConnector.java │ │ ├── IcebergSinkConnectorConfig.java │ │ ├── IcebergSinkTask.java │ │ ├── RecordWriter.java │ │ ├── converter │ │ ├── RecordConverter.java │ │ └── SchemaConverter.java │ │ ├── iceberg │ │ ├── DefaultIcebergWriter.java │ │ ├── DefaultIcebergWriterFactory.java │ │ ├── DefaultPartitionedFanoutWriter.java │ │ ├── IcebergCatalogFactory.java │ │ ├── IcebergTableManager.java │ │ └── IcebergUtils.java │ │ ├── partition │ │ ├── PartitionTransformSpec.java │ │ └── PartitionTransformSpecFactory.java │ │ ├── schema │ │ └── RecordSchemaCompatibilityTracker.java │ │ ├── spi │ │ ├── IcebergWriter.java │ │ └── IcebergWriterFactory.java │ │ └── util │ │ └── Version.java └── resources │ ├── kafka-connect-iceberg-sink.properties │ └── manifest.json └── test └── java └── com └── github └── tenxfuturetechnologies └── kafkaconnecticeberg └── sink ├── IcebergSinkConnectorConfigTest.java ├── converter ├── RecordConverterTest.java └── SchemaConverterTest.java ├── iceberg └── IcebergUtilsTest.java ├── partition ├── PartitionTransformSpecFactoryTest.java └── PartitionTransformSpecTest.java └── schema └── RecordSchemaCompatibilityTrackerTest.java /.github/dependabot.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/.github/dependabot.yml -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/.github/workflows/ci.yml -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/.github/workflows/publish.yml -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/.github/workflows/release.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/README.md -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/gradle.properties -------------------------------------------------------------------------------- /gradle/checkstyle.gradle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/gradle/checkstyle.gradle -------------------------------------------------------------------------------- /gradle/config/checkstyle/suppressions.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/gradle/config/checkstyle/suppressions.xml -------------------------------------------------------------------------------- /gradle/jacoco.gradle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/gradle/jacoco.gradle -------------------------------------------------------------------------------- /gradle/publish.gradle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/gradle/publish.gradle -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/gradle/wrapper/gradle-wrapper.properties -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/gradlew -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/gradlew.bat -------------------------------------------------------------------------------- /integration-tests/build.gradle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/integration-tests/build.gradle -------------------------------------------------------------------------------- /integration-tests/src/avro/test-event-with-all-spark-compatible-types.avsc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/integration-tests/src/avro/test-event-with-all-spark-compatible-types.avsc -------------------------------------------------------------------------------- /integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/containers/KafkaConnectContainer.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/containers/KafkaConnectContainer.java -------------------------------------------------------------------------------- /integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/containers/KafkaContainer.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/containers/KafkaContainer.java -------------------------------------------------------------------------------- /integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/containers/MinioContainer.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/containers/MinioContainer.java -------------------------------------------------------------------------------- /integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/containers/SchemaRegistryContainer.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/containers/SchemaRegistryContainer.java -------------------------------------------------------------------------------- /integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/AbstractBaseIT.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/AbstractBaseIT.java -------------------------------------------------------------------------------- /integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/IcebergS3SinkAvroIT.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/IcebergS3SinkAvroIT.java -------------------------------------------------------------------------------- /integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/util/Constants.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/util/Constants.java -------------------------------------------------------------------------------- /integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/util/MinioTestHelper.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/util/MinioTestHelper.java -------------------------------------------------------------------------------- /integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/util/SinkConfigBuilder.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/util/SinkConfigBuilder.java -------------------------------------------------------------------------------- /integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/util/SparkTestHelper.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/integration-tests/src/integration-test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/util/SparkTestHelper.java -------------------------------------------------------------------------------- /integration-tests/src/integration-test/resources/logback-test.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/integration-tests/src/integration-test/resources/logback-test.xml -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/settings.gradle -------------------------------------------------------------------------------- /sink/build.gradle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/build.gradle -------------------------------------------------------------------------------- /sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/IcebergSinkConnector.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/IcebergSinkConnector.java -------------------------------------------------------------------------------- /sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/IcebergSinkConnectorConfig.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/IcebergSinkConnectorConfig.java -------------------------------------------------------------------------------- /sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/IcebergSinkTask.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/IcebergSinkTask.java -------------------------------------------------------------------------------- /sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/RecordWriter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/RecordWriter.java -------------------------------------------------------------------------------- /sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/converter/RecordConverter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/converter/RecordConverter.java -------------------------------------------------------------------------------- /sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/converter/SchemaConverter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/converter/SchemaConverter.java -------------------------------------------------------------------------------- /sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/iceberg/DefaultIcebergWriter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/iceberg/DefaultIcebergWriter.java -------------------------------------------------------------------------------- /sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/iceberg/DefaultIcebergWriterFactory.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/iceberg/DefaultIcebergWriterFactory.java -------------------------------------------------------------------------------- /sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/iceberg/DefaultPartitionedFanoutWriter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/iceberg/DefaultPartitionedFanoutWriter.java -------------------------------------------------------------------------------- /sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/iceberg/IcebergCatalogFactory.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/iceberg/IcebergCatalogFactory.java -------------------------------------------------------------------------------- /sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/iceberg/IcebergTableManager.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/iceberg/IcebergTableManager.java -------------------------------------------------------------------------------- /sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/iceberg/IcebergUtils.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/iceberg/IcebergUtils.java -------------------------------------------------------------------------------- /sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/partition/PartitionTransformSpec.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/partition/PartitionTransformSpec.java -------------------------------------------------------------------------------- /sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/partition/PartitionTransformSpecFactory.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/partition/PartitionTransformSpecFactory.java -------------------------------------------------------------------------------- /sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/schema/RecordSchemaCompatibilityTracker.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/schema/RecordSchemaCompatibilityTracker.java -------------------------------------------------------------------------------- /sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/spi/IcebergWriter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/spi/IcebergWriter.java -------------------------------------------------------------------------------- /sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/spi/IcebergWriterFactory.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/spi/IcebergWriterFactory.java -------------------------------------------------------------------------------- /sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/util/Version.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/util/Version.java -------------------------------------------------------------------------------- /sink/src/main/resources/kafka-connect-iceberg-sink.properties: -------------------------------------------------------------------------------- 1 | version=@VERSION@ 2 | -------------------------------------------------------------------------------- /sink/src/main/resources/manifest.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/main/resources/manifest.json -------------------------------------------------------------------------------- /sink/src/test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/IcebergSinkConnectorConfigTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/IcebergSinkConnectorConfigTest.java -------------------------------------------------------------------------------- /sink/src/test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/converter/RecordConverterTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/converter/RecordConverterTest.java -------------------------------------------------------------------------------- /sink/src/test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/converter/SchemaConverterTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/converter/SchemaConverterTest.java -------------------------------------------------------------------------------- /sink/src/test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/iceberg/IcebergUtilsTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/iceberg/IcebergUtilsTest.java -------------------------------------------------------------------------------- /sink/src/test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/partition/PartitionTransformSpecFactoryTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/partition/PartitionTransformSpecFactoryTest.java -------------------------------------------------------------------------------- /sink/src/test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/partition/PartitionTransformSpecTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/partition/PartitionTransformSpecTest.java -------------------------------------------------------------------------------- /sink/src/test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/schema/RecordSchemaCompatibilityTrackerTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/10xfuturetechnologies/kafka-connect-iceberg/HEAD/sink/src/test/java/com/github/tenxfuturetechnologies/kafkaconnecticeberg/sink/schema/RecordSchemaCompatibilityTrackerTest.java --------------------------------------------------------------------------------