├── .astro └── config.yaml ├── .dockerignore ├── .gitignore ├── Dockerfile ├── README.md ├── airflow_settings.yaml ├── config └── airflow.cfg ├── dags ├── cloud-data-lake-pipeline.py └── sql │ ├── D_AIRPORT.sql │ ├── D_CITY_DEMO.sql │ ├── D_TIME.sql │ ├── D_WEATHER.sql │ └── F_IMMIGRATION_DATA.sql ├── img ├── airflow-ui.png ├── connection.png ├── data_pipeline.png ├── db_diagram.png └── dwh.png ├── packages.txt ├── requirements.txt ├── script └── entrypoint.sh └── src ├── setup_gcp_iac.py └── staging_schemas.py /.astro/config.yaml: -------------------------------------------------------------------------------- 1 | project: 2 | name: cloud-data-lake 3 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .astro 2 | .git 3 | .env 4 | airflow_settings.yaml 5 | logs/ -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuanchris/cloud-data-lake/HEAD/.gitignore -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM quay.io/astronomer/ap-airflow:1.10.12-buster-onbuild 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuanchris/cloud-data-lake/HEAD/README.md -------------------------------------------------------------------------------- /airflow_settings.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuanchris/cloud-data-lake/HEAD/airflow_settings.yaml -------------------------------------------------------------------------------- /config/airflow.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuanchris/cloud-data-lake/HEAD/config/airflow.cfg -------------------------------------------------------------------------------- /dags/cloud-data-lake-pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuanchris/cloud-data-lake/HEAD/dags/cloud-data-lake-pipeline.py -------------------------------------------------------------------------------- /dags/sql/D_AIRPORT.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuanchris/cloud-data-lake/HEAD/dags/sql/D_AIRPORT.sql -------------------------------------------------------------------------------- /dags/sql/D_CITY_DEMO.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuanchris/cloud-data-lake/HEAD/dags/sql/D_CITY_DEMO.sql -------------------------------------------------------------------------------- /dags/sql/D_TIME.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuanchris/cloud-data-lake/HEAD/dags/sql/D_TIME.sql -------------------------------------------------------------------------------- /dags/sql/D_WEATHER.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuanchris/cloud-data-lake/HEAD/dags/sql/D_WEATHER.sql -------------------------------------------------------------------------------- /dags/sql/F_IMMIGRATION_DATA.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuanchris/cloud-data-lake/HEAD/dags/sql/F_IMMIGRATION_DATA.sql -------------------------------------------------------------------------------- /img/airflow-ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuanchris/cloud-data-lake/HEAD/img/airflow-ui.png -------------------------------------------------------------------------------- /img/connection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuanchris/cloud-data-lake/HEAD/img/connection.png -------------------------------------------------------------------------------- /img/data_pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuanchris/cloud-data-lake/HEAD/img/data_pipeline.png -------------------------------------------------------------------------------- /img/db_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuanchris/cloud-data-lake/HEAD/img/db_diagram.png -------------------------------------------------------------------------------- /img/dwh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuanchris/cloud-data-lake/HEAD/img/dwh.png -------------------------------------------------------------------------------- /packages.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /script/entrypoint.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuanchris/cloud-data-lake/HEAD/script/entrypoint.sh -------------------------------------------------------------------------------- /src/setup_gcp_iac.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuanchris/cloud-data-lake/HEAD/src/setup_gcp_iac.py -------------------------------------------------------------------------------- /src/staging_schemas.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuanchris/cloud-data-lake/HEAD/src/staging_schemas.py --------------------------------------------------------------------------------