├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── app.py ├── assets ├── iceberg-data-level-01.png ├── iceberg-data-level-02.png ├── iceberg-data-level-03.png └── iceberg-table.png ├── cdk.json ├── cdk_stacks ├── __init__.py ├── glue_job_role.py ├── glue_stream_data_schema.py ├── glue_streaming_job.py ├── kds.py ├── lakeformation_permissions.py └── s3.py ├── glue-streaming-data-to-iceberg-table.svg ├── requirements-dev.txt ├── requirements.txt ├── source.bat └── src ├── main └── python │ ├── spark_iceberg_writes_with_dataframe.py │ ├── spark_iceberg_writes_with_sql_insert_overwrite.py │ └── spark_iceberg_writes_with_sql_merge_into.py └── utils └── gen_fake_kinesis_stream_data.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/.gitignore -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/README.md -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/app.py -------------------------------------------------------------------------------- /assets/iceberg-data-level-01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/assets/iceberg-data-level-01.png -------------------------------------------------------------------------------- /assets/iceberg-data-level-02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/assets/iceberg-data-level-02.png -------------------------------------------------------------------------------- /assets/iceberg-data-level-03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/assets/iceberg-data-level-03.png -------------------------------------------------------------------------------- /assets/iceberg-table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/assets/iceberg-table.png -------------------------------------------------------------------------------- /cdk.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/cdk.json -------------------------------------------------------------------------------- /cdk_stacks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/cdk_stacks/__init__.py -------------------------------------------------------------------------------- /cdk_stacks/glue_job_role.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/cdk_stacks/glue_job_role.py -------------------------------------------------------------------------------- /cdk_stacks/glue_stream_data_schema.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/cdk_stacks/glue_stream_data_schema.py -------------------------------------------------------------------------------- /cdk_stacks/glue_streaming_job.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/cdk_stacks/glue_streaming_job.py -------------------------------------------------------------------------------- /cdk_stacks/kds.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/cdk_stacks/kds.py -------------------------------------------------------------------------------- /cdk_stacks/lakeformation_permissions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/cdk_stacks/lakeformation_permissions.py -------------------------------------------------------------------------------- /cdk_stacks/s3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/cdk_stacks/s3.py -------------------------------------------------------------------------------- /glue-streaming-data-to-iceberg-table.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/glue-streaming-data-to-iceberg-table.svg -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | boto3>=1.24.41 2 | mimesis==6.0.0 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/requirements.txt -------------------------------------------------------------------------------- /source.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/source.bat -------------------------------------------------------------------------------- /src/main/python/spark_iceberg_writes_with_dataframe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/src/main/python/spark_iceberg_writes_with_dataframe.py -------------------------------------------------------------------------------- /src/main/python/spark_iceberg_writes_with_sql_insert_overwrite.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/src/main/python/spark_iceberg_writes_with_sql_insert_overwrite.py -------------------------------------------------------------------------------- /src/main/python/spark_iceberg_writes_with_sql_merge_into.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/src/main/python/spark_iceberg_writes_with_sql_merge_into.py -------------------------------------------------------------------------------- /src/utils/gen_fake_kinesis_stream_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-glue-streaming-etl-with-apache-iceberg/HEAD/src/utils/gen_fake_kinesis_stream_data.py --------------------------------------------------------------------------------