├── .gitignore ├── README.md ├── airflow ├── Dockerfile ├── dags │ ├── daily_DAG.py │ ├── historical_DAG.py │ └── spark_job.py ├── docker-compose.yaml ├── requirements.txt └── scripts │ ├── entrypoint.sh │ └── spark_job.py ├── scripts ├── airflow_clear.sh ├── airflow_startup.sh ├── airflow_stop.sh ├── setup_config.sh └── vm_setup.sh ├── setup ├── gcp_account.md ├── gcp_vm.md └── terraform_vm.md └── terraform ├── .gitignore ├── main.tf └── variables.tf /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/README.md -------------------------------------------------------------------------------- /airflow/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/airflow/Dockerfile -------------------------------------------------------------------------------- /airflow/dags/daily_DAG.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/airflow/dags/daily_DAG.py -------------------------------------------------------------------------------- /airflow/dags/historical_DAG.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/airflow/dags/historical_DAG.py -------------------------------------------------------------------------------- /airflow/dags/spark_job.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/airflow/dags/spark_job.py -------------------------------------------------------------------------------- /airflow/docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/airflow/docker-compose.yaml -------------------------------------------------------------------------------- /airflow/requirements.txt: -------------------------------------------------------------------------------- 1 | apache-airflow-providers-google 2 | pyspark -------------------------------------------------------------------------------- /airflow/scripts/entrypoint.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/airflow/scripts/entrypoint.sh -------------------------------------------------------------------------------- /airflow/scripts/spark_job.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/airflow/scripts/spark_job.py -------------------------------------------------------------------------------- /scripts/airflow_clear.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/scripts/airflow_clear.sh -------------------------------------------------------------------------------- /scripts/airflow_startup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/scripts/airflow_startup.sh -------------------------------------------------------------------------------- /scripts/airflow_stop.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/scripts/airflow_stop.sh -------------------------------------------------------------------------------- /scripts/setup_config.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/scripts/setup_config.sh -------------------------------------------------------------------------------- /scripts/vm_setup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/scripts/vm_setup.sh -------------------------------------------------------------------------------- /setup/gcp_account.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/setup/gcp_account.md -------------------------------------------------------------------------------- /setup/gcp_vm.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/setup/gcp_vm.md -------------------------------------------------------------------------------- /setup/terraform_vm.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/setup/terraform_vm.md -------------------------------------------------------------------------------- /terraform/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/terraform/.gitignore -------------------------------------------------------------------------------- /terraform/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/terraform/main.tf -------------------------------------------------------------------------------- /terraform/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AliaHa3/data-engineering-zoomcamp-project/HEAD/terraform/variables.tf --------------------------------------------------------------------------------