├── .github └── workflows │ └── GHA.yml ├── .gitignore ├── LICENSE ├── README.md ├── data └── covid-data.parquet ├── env ├── images ├── covid19_architecture.jpg ├── covid19_dashboard.png ├── covid19_dashboard_7days.png ├── covid19_monitoring_architecture.png ├── covid19_video.gif ├── dashboard_example.png ├── dbt_environment.png ├── dbt_jobs.png ├── dbt_jobs2.png ├── dbt_jobs3.png ├── prefect_automation.png └── prefect_deployment.png ├── infrastucture ├── main.tf ├── terraform.tfvars └── variables.tf ├── prerequisites_readme.md ├── requirements.txt ├── scripts ├── create_service_account.sh ├── make_dbt_block.py ├── make_gcp_block.py └── make_gh_block.py ├── src ├── .prefectignore ├── __init__.py ├── dbt │ ├── analyses │ │ └── .gitkeep │ ├── dbt_project.yml │ ├── macros │ │ └── .gitkeep │ ├── models │ │ ├── production │ │ │ ├── prod_covid.sql │ │ │ ├── prod_covid_lastdays.sql │ │ │ └── schema.yaml │ │ └── staging │ │ │ ├── schema.yml │ │ │ └── stg_covid.sql │ ├── profiles.yml │ ├── seeds │ │ └── .gitkeep │ └── snapshots │ │ └── .gitkeep ├── gcs_to_bq-deployment.yaml ├── gcs_to_bq.py ├── run-dbt.py ├── run_dbt_job_flow-deployment.yaml ├── trigger_dbt.py ├── web_to_gcs-deployment.yaml └── web_to_gcs.py ├── tests ├── .gitkeep ├── __init__.py ├── test_gcs_to_bq_functions.py └── test_web_to_gcs_functions.py └── visualizations_readme.md /.github/workflows/GHA.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/.github/workflows/GHA.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/README.md -------------------------------------------------------------------------------- /data/covid-data.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/data/covid-data.parquet -------------------------------------------------------------------------------- /env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/env -------------------------------------------------------------------------------- /images/covid19_architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/images/covid19_architecture.jpg -------------------------------------------------------------------------------- /images/covid19_dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/images/covid19_dashboard.png -------------------------------------------------------------------------------- /images/covid19_dashboard_7days.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/images/covid19_dashboard_7days.png -------------------------------------------------------------------------------- /images/covid19_monitoring_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/images/covid19_monitoring_architecture.png -------------------------------------------------------------------------------- /images/covid19_video.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/images/covid19_video.gif -------------------------------------------------------------------------------- /images/dashboard_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/images/dashboard_example.png -------------------------------------------------------------------------------- /images/dbt_environment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/images/dbt_environment.png -------------------------------------------------------------------------------- /images/dbt_jobs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/images/dbt_jobs.png -------------------------------------------------------------------------------- /images/dbt_jobs2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/images/dbt_jobs2.png -------------------------------------------------------------------------------- /images/dbt_jobs3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/images/dbt_jobs3.png -------------------------------------------------------------------------------- /images/prefect_automation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/images/prefect_automation.png -------------------------------------------------------------------------------- /images/prefect_deployment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/images/prefect_deployment.png -------------------------------------------------------------------------------- /infrastucture/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/infrastucture/main.tf -------------------------------------------------------------------------------- /infrastucture/terraform.tfvars: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/infrastucture/terraform.tfvars -------------------------------------------------------------------------------- /infrastucture/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/infrastucture/variables.tf -------------------------------------------------------------------------------- /prerequisites_readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/prerequisites_readme.md -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/create_service_account.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/scripts/create_service_account.sh -------------------------------------------------------------------------------- /scripts/make_dbt_block.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/scripts/make_dbt_block.py -------------------------------------------------------------------------------- /scripts/make_gcp_block.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/scripts/make_gcp_block.py -------------------------------------------------------------------------------- /scripts/make_gh_block.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/scripts/make_gh_block.py -------------------------------------------------------------------------------- /src/.prefectignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/src/.prefectignore -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /src/dbt/analyses/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/dbt/dbt_project.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/src/dbt/dbt_project.yml -------------------------------------------------------------------------------- /src/dbt/macros/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/dbt/models/production/prod_covid.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/src/dbt/models/production/prod_covid.sql -------------------------------------------------------------------------------- /src/dbt/models/production/prod_covid_lastdays.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/src/dbt/models/production/prod_covid_lastdays.sql -------------------------------------------------------------------------------- /src/dbt/models/production/schema.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/src/dbt/models/production/schema.yaml -------------------------------------------------------------------------------- /src/dbt/models/staging/schema.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/src/dbt/models/staging/schema.yml -------------------------------------------------------------------------------- /src/dbt/models/staging/stg_covid.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/src/dbt/models/staging/stg_covid.sql -------------------------------------------------------------------------------- /src/dbt/profiles.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/src/dbt/profiles.yml -------------------------------------------------------------------------------- /src/dbt/seeds/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/dbt/snapshots/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/gcs_to_bq-deployment.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/src/gcs_to_bq-deployment.yaml -------------------------------------------------------------------------------- /src/gcs_to_bq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/src/gcs_to_bq.py -------------------------------------------------------------------------------- /src/run-dbt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/src/run-dbt.py -------------------------------------------------------------------------------- /src/run_dbt_job_flow-deployment.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/src/run_dbt_job_flow-deployment.yaml -------------------------------------------------------------------------------- /src/trigger_dbt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/src/trigger_dbt.py -------------------------------------------------------------------------------- /src/web_to_gcs-deployment.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/src/web_to_gcs-deployment.yaml -------------------------------------------------------------------------------- /src/web_to_gcs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/src/web_to_gcs.py -------------------------------------------------------------------------------- /tests/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /tests/test_gcs_to_bq_functions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/tests/test_gcs_to_bq_functions.py -------------------------------------------------------------------------------- /tests/test_web_to_gcs_functions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/tests/test_web_to_gcs_functions.py -------------------------------------------------------------------------------- /visualizations_readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MikhailKuklin/data-pipeline-COVID19-monitoring/HEAD/visualizations_readme.md --------------------------------------------------------------------------------