├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── airflow ├── dags │ └── project_dags.py └── tasks │ ├── DataScraper │ ├── __init__.py │ ├── main.py │ ├── scraper_men_products.py │ ├── scraper_women_products.py │ └── upload_azure.py │ ├── DataValidator │ ├── __init__.py │ ├── configuration_env.py │ ├── data_cleaner.py │ ├── list_files.py │ ├── main.py │ ├── model_pydantic.py │ └── upload_function.py │ └── LoadDataSQL │ ├── __init__.py │ ├── config_parameters.py │ ├── list_files.py │ ├── load_to_postgres.py │ ├── load_to_synapse.py │ ├── main.py │ ├── postgres_create_tables.py │ └── synapse_create_tables.py ├── az_login.sh ├── docker-compose.yaml ├── images ├── Sunglasseshutetl_dashboard.png ├── Sunglasseshutetl_dashboard_cheap.png └── azure_etl.png └── terraform ├── outputs.tf ├── postgres_flexible.tf ├── provider.tf ├── resource_group.tf ├── storage_account.tf ├── synapse_workspace.tf └── variables.tf /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/.gitignore -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/Dockerfile -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/LICENSE -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/README.md -------------------------------------------------------------------------------- /airflow/dags/project_dags.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/airflow/dags/project_dags.py -------------------------------------------------------------------------------- /airflow/tasks/DataScraper/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /airflow/tasks/DataScraper/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/airflow/tasks/DataScraper/main.py -------------------------------------------------------------------------------- /airflow/tasks/DataScraper/scraper_men_products.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/airflow/tasks/DataScraper/scraper_men_products.py -------------------------------------------------------------------------------- /airflow/tasks/DataScraper/scraper_women_products.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/airflow/tasks/DataScraper/scraper_women_products.py -------------------------------------------------------------------------------- /airflow/tasks/DataScraper/upload_azure.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/airflow/tasks/DataScraper/upload_azure.py -------------------------------------------------------------------------------- /airflow/tasks/DataValidator/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /airflow/tasks/DataValidator/configuration_env.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/airflow/tasks/DataValidator/configuration_env.py -------------------------------------------------------------------------------- /airflow/tasks/DataValidator/data_cleaner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/airflow/tasks/DataValidator/data_cleaner.py -------------------------------------------------------------------------------- /airflow/tasks/DataValidator/list_files.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/airflow/tasks/DataValidator/list_files.py -------------------------------------------------------------------------------- /airflow/tasks/DataValidator/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/airflow/tasks/DataValidator/main.py -------------------------------------------------------------------------------- /airflow/tasks/DataValidator/model_pydantic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/airflow/tasks/DataValidator/model_pydantic.py -------------------------------------------------------------------------------- /airflow/tasks/DataValidator/upload_function.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/airflow/tasks/DataValidator/upload_function.py -------------------------------------------------------------------------------- /airflow/tasks/LoadDataSQL/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /airflow/tasks/LoadDataSQL/config_parameters.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/airflow/tasks/LoadDataSQL/config_parameters.py -------------------------------------------------------------------------------- /airflow/tasks/LoadDataSQL/list_files.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/airflow/tasks/LoadDataSQL/list_files.py -------------------------------------------------------------------------------- /airflow/tasks/LoadDataSQL/load_to_postgres.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/airflow/tasks/LoadDataSQL/load_to_postgres.py -------------------------------------------------------------------------------- /airflow/tasks/LoadDataSQL/load_to_synapse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/airflow/tasks/LoadDataSQL/load_to_synapse.py -------------------------------------------------------------------------------- /airflow/tasks/LoadDataSQL/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/airflow/tasks/LoadDataSQL/main.py -------------------------------------------------------------------------------- /airflow/tasks/LoadDataSQL/postgres_create_tables.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/airflow/tasks/LoadDataSQL/postgres_create_tables.py -------------------------------------------------------------------------------- /airflow/tasks/LoadDataSQL/synapse_create_tables.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/airflow/tasks/LoadDataSQL/synapse_create_tables.py -------------------------------------------------------------------------------- /az_login.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/az_login.sh -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/docker-compose.yaml -------------------------------------------------------------------------------- /images/Sunglasseshutetl_dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/images/Sunglasseshutetl_dashboard.png -------------------------------------------------------------------------------- /images/Sunglasseshutetl_dashboard_cheap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/images/Sunglasseshutetl_dashboard_cheap.png -------------------------------------------------------------------------------- /images/azure_etl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/images/azure_etl.png -------------------------------------------------------------------------------- /terraform/outputs.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/terraform/outputs.tf -------------------------------------------------------------------------------- /terraform/postgres_flexible.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/terraform/postgres_flexible.tf -------------------------------------------------------------------------------- /terraform/provider.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/terraform/provider.tf -------------------------------------------------------------------------------- /terraform/resource_group.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/terraform/resource_group.tf -------------------------------------------------------------------------------- /terraform/storage_account.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/terraform/storage_account.tf -------------------------------------------------------------------------------- /terraform/synapse_workspace.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/terraform/synapse_workspace.tf -------------------------------------------------------------------------------- /terraform/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SebasMBK/SunglassesHubETL/HEAD/terraform/variables.tf --------------------------------------------------------------------------------