├── .gitignore ├── .idea ├── .gitignore ├── ELT-Data-Pipeline.iml ├── inspectionProfiles │ ├── Project_Default.xml │ └── profiles_settings.xml ├── misc.xml ├── modules.xml └── vcs.xml ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── README.md ├── airflow-docker-compose.yaml ├── airflow ├── README.md ├── config │ └── airflow.cfg ├── dags │ ├── .gitkeep │ ├── airquality_pipeline_dag.py │ ├── data_migration_dag.py │ └── utils │ │ ├── airquality_collector.py │ │ └── migrate_data.py └── plugins │ └── .gitkeep ├── data_validation ├── .gitkeep ├── README.md ├── gx │ ├── .gitignore │ ├── expectations │ │ ├── .ge_store_backend_id │ │ └── validate_openaq_measurement.json │ ├── great_expectations.yml │ ├── plugins │ │ └── custom_data_docs │ │ │ └── styles │ │ │ └── data_docs_custom_styles.css │ └── validation_definitions │ │ └── openaq_validation_definition.json └── openaq_data_quality.ipynb ├── dbt_openaq ├── .gitignore ├── .user.yml ├── README.md ├── analyses │ └── .gitkeep ├── dbt_project.yml ├── macros │ └── .gitkeep ├── models │ └── production │ │ ├── dim_date.sql │ │ ├── dim_instrument.sql │ │ ├── dim_location.sql │ │ ├── dim_owner.sql │ │ ├── dim_parameter.sql │ │ ├── dim_provider.sql │ │ ├── dim_sensor.sql │ │ ├── dim_time.sql │ │ ├── fact_air_quality_measurement.sql │ │ └── schema.yml ├── package-lock.yml ├── packages.yml ├── profiles.yml ├── seeds │ └── .gitkeep ├── snapshots │ └── .gitkeep └── tests │ └── .gitkeep ├── docker-compose.yaml ├── images ├── Airflow-Logo.png ├── DBT-Logo.png ├── PostgreSQLConnection.png ├── SystemArchitecture.png ├── WorkflowDiagram.png ├── dbt_debug.png ├── dbt_run.png ├── minio-webui.png ├── trino-dbeaver.png └── trino-webui.png ├── requirements.txt ├── scripts ├── __init__.py ├── create_schema.py ├── create_table.py └── postgresql_client.py ├── storage-docker-compose.yaml └── trino-minio ├── conf └── metastore-site.xml └── etc ├── catalog └── minio.properties ├── config.properties ├── jvm.config ├── log.properties └── node.properties /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/.gitignore -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | -------------------------------------------------------------------------------- /.idea/ELT-Data-Pipeline.iml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/.idea/ELT-Data-Pipeline.iml -------------------------------------------------------------------------------- /.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/.idea/inspectionProfiles/Project_Default.xml -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/.idea/inspectionProfiles/profiles_settings.xml -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/.idea/misc.xml -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/.idea/modules.xml -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/.idea/vcs.xml -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/CHANGELOG.md -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/Dockerfile -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/README.md -------------------------------------------------------------------------------- /airflow-docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/airflow-docker-compose.yaml -------------------------------------------------------------------------------- /airflow/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/airflow/README.md -------------------------------------------------------------------------------- /airflow/config/airflow.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/airflow/config/airflow.cfg -------------------------------------------------------------------------------- /airflow/dags/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /airflow/dags/airquality_pipeline_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/airflow/dags/airquality_pipeline_dag.py -------------------------------------------------------------------------------- /airflow/dags/data_migration_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/airflow/dags/data_migration_dag.py -------------------------------------------------------------------------------- /airflow/dags/utils/airquality_collector.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/airflow/dags/utils/airquality_collector.py -------------------------------------------------------------------------------- /airflow/dags/utils/migrate_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/airflow/dags/utils/migrate_data.py -------------------------------------------------------------------------------- /airflow/plugins/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data_validation/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data_validation/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/data_validation/README.md -------------------------------------------------------------------------------- /data_validation/gx/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | uncommitted/ -------------------------------------------------------------------------------- /data_validation/gx/expectations/.ge_store_backend_id: -------------------------------------------------------------------------------- 1 | store_backend_id = d27e374d-865c-4512-9f92-3a4a38cb9c6b 2 | -------------------------------------------------------------------------------- /data_validation/gx/expectations/validate_openaq_measurement.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/data_validation/gx/expectations/validate_openaq_measurement.json -------------------------------------------------------------------------------- /data_validation/gx/great_expectations.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/data_validation/gx/great_expectations.yml -------------------------------------------------------------------------------- /data_validation/gx/plugins/custom_data_docs/styles/data_docs_custom_styles.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/data_validation/gx/plugins/custom_data_docs/styles/data_docs_custom_styles.css -------------------------------------------------------------------------------- /data_validation/gx/validation_definitions/openaq_validation_definition.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/data_validation/gx/validation_definitions/openaq_validation_definition.json -------------------------------------------------------------------------------- /data_validation/openaq_data_quality.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/data_validation/openaq_data_quality.ipynb -------------------------------------------------------------------------------- /dbt_openaq/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | -------------------------------------------------------------------------------- /dbt_openaq/.user.yml: -------------------------------------------------------------------------------- 1 | id: df971031-ed00-4783-bb43-278ea9eab9b7 2 | -------------------------------------------------------------------------------- /dbt_openaq/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/dbt_openaq/README.md -------------------------------------------------------------------------------- /dbt_openaq/analyses/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt_openaq/dbt_project.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/dbt_openaq/dbt_project.yml -------------------------------------------------------------------------------- /dbt_openaq/macros/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt_openaq/models/production/dim_date.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/dbt_openaq/models/production/dim_date.sql -------------------------------------------------------------------------------- /dbt_openaq/models/production/dim_instrument.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/dbt_openaq/models/production/dim_instrument.sql -------------------------------------------------------------------------------- /dbt_openaq/models/production/dim_location.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/dbt_openaq/models/production/dim_location.sql -------------------------------------------------------------------------------- /dbt_openaq/models/production/dim_owner.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/dbt_openaq/models/production/dim_owner.sql -------------------------------------------------------------------------------- /dbt_openaq/models/production/dim_parameter.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/dbt_openaq/models/production/dim_parameter.sql -------------------------------------------------------------------------------- /dbt_openaq/models/production/dim_provider.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/dbt_openaq/models/production/dim_provider.sql -------------------------------------------------------------------------------- /dbt_openaq/models/production/dim_sensor.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/dbt_openaq/models/production/dim_sensor.sql -------------------------------------------------------------------------------- /dbt_openaq/models/production/dim_time.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/dbt_openaq/models/production/dim_time.sql -------------------------------------------------------------------------------- /dbt_openaq/models/production/fact_air_quality_measurement.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/dbt_openaq/models/production/fact_air_quality_measurement.sql -------------------------------------------------------------------------------- /dbt_openaq/models/production/schema.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/dbt_openaq/models/production/schema.yml -------------------------------------------------------------------------------- /dbt_openaq/package-lock.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/dbt_openaq/package-lock.yml -------------------------------------------------------------------------------- /dbt_openaq/packages.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/dbt_openaq/packages.yml -------------------------------------------------------------------------------- /dbt_openaq/profiles.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/dbt_openaq/profiles.yml -------------------------------------------------------------------------------- /dbt_openaq/seeds/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt_openaq/snapshots/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt_openaq/tests/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/docker-compose.yaml -------------------------------------------------------------------------------- /images/Airflow-Logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/images/Airflow-Logo.png -------------------------------------------------------------------------------- /images/DBT-Logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/images/DBT-Logo.png -------------------------------------------------------------------------------- /images/PostgreSQLConnection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/images/PostgreSQLConnection.png -------------------------------------------------------------------------------- /images/SystemArchitecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/images/SystemArchitecture.png -------------------------------------------------------------------------------- /images/WorkflowDiagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/images/WorkflowDiagram.png -------------------------------------------------------------------------------- /images/dbt_debug.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/images/dbt_debug.png -------------------------------------------------------------------------------- /images/dbt_run.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/images/dbt_run.png -------------------------------------------------------------------------------- /images/minio-webui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/images/minio-webui.png -------------------------------------------------------------------------------- /images/trino-dbeaver.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/images/trino-dbeaver.png -------------------------------------------------------------------------------- /images/trino-webui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/images/trino-webui.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/__init__.py: -------------------------------------------------------------------------------- 1 | # Make scripts directory a proper Python package -------------------------------------------------------------------------------- /scripts/create_schema.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/scripts/create_schema.py -------------------------------------------------------------------------------- /scripts/create_table.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/scripts/create_table.py -------------------------------------------------------------------------------- /scripts/postgresql_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/scripts/postgresql_client.py -------------------------------------------------------------------------------- /storage-docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/storage-docker-compose.yaml -------------------------------------------------------------------------------- /trino-minio/conf/metastore-site.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/trino-minio/conf/metastore-site.xml -------------------------------------------------------------------------------- /trino-minio/etc/catalog/minio.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/trino-minio/etc/catalog/minio.properties -------------------------------------------------------------------------------- /trino-minio/etc/config.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/trino-minio/etc/config.properties -------------------------------------------------------------------------------- /trino-minio/etc/jvm.config: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/trino-minio/etc/jvm.config -------------------------------------------------------------------------------- /trino-minio/etc/log.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/trino-minio/etc/log.properties -------------------------------------------------------------------------------- /trino-minio/etc/node.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dain55788/ELT-Data-Pipeline/HEAD/trino-minio/etc/node.properties --------------------------------------------------------------------------------