├── README.md ├── basics_setup └── docker-compose.yml ├── create_connection_vm.md ├── create_gcp_project.md ├── create_service_account.md ├── create_vm_instance.md ├── dbt ├── .gitignore ├── README.md ├── analyses │ └── .gitkeep ├── dbt_project.yml ├── models │ ├── core │ │ ├── dim_zones_dep.sql │ │ ├── dim_zones_district.sql │ │ ├── fact_sien_an.sql │ │ └── fact_sien_en.sql │ └── staging │ │ ├── children_Anemia_6-35m_x_DEP_2022.sql │ │ ├── children_Anemia_6-59m_x_DEP_2022.sql │ │ ├── children_EN_0-35m_x_DEP_2022.sql │ │ ├── children_EN_0-59m_x_DEP_2022.sql │ │ ├── foreign_children_Anemia_6-35m_x_DEP_2022.sql │ │ ├── foreign_children_Anemia_6-59m_x_DEP_2022.sql │ │ ├── foreign_children_EN_0-35m_x_DEP_2022.sql │ │ ├── foreign_children_EN_0-59m_x_DEP_2022.sql │ │ ├── pregnant_women_Anemia_x_DEP_2022.sql │ │ └── schema.yml ├── seeds │ ├── .gitkeep │ ├── lon_lat.csv │ └── ubigeous_lookup.csv ├── snapshots │ └── .gitkeep └── tests │ └── .gitkeep ├── images ├── IAM_API.png ├── IAM_API_2.png ├── IAM_sien_project.png ├── SIEN.png ├── apply_agent.png ├── bashrc_spark.png ├── boot_disk.png ├── cat_gcp.png ├── compute_engine_api.png ├── compute_engine_vm.png ├── connect_to_host.png ├── connection_pgadmin.png ├── core_dbt.png ├── create_project_gcp.png ├── create_service_account.png ├── create_vm_instance.png ├── dashboard.png ├── data_architecture.png ├── data_architecture_chosen.png ├── deployment_apply.png ├── deployment_build.png ├── deployment_build_cmd.png ├── docker_block_configuration.png ├── docker_block_prefect.png ├── docker_build_prefect.png ├── docker_compose_added.png ├── docker_compose_path.png ├── docker_compose_ps.png ├── docker_compose_structure.png ├── docker_deploy_py.png ├── docker_push_prefect.png ├── docker_run_hw.png ├── e2_instance.png ├── edit_permissions_terraform_sien.png ├── environments_dbt.png ├── establishment_level.png ├── export_google_credentials.png ├── files_permissions.png ├── finally_create_vm.png ├── gc_file_json.png ├── gcp.png ├── gcs_bucket_block.png ├── generate_key_service_account.png ├── google_compute_engine.png ├── indicator_sien.png ├── informes_sien.png ├── java_path.png ├── lineage_dbt.png ├── metadata_compute_engine.png ├── mkdir_bin.png ├── more_permissions_terraform.png ├── name_server.png ├── name_vm.png ├── new_project_gcp.png ├── orion_prefect.png ├── pg_admin_ingest_data.png ├── pgadmin_browser.png ├── port_forwarding.png ├── prefect_ingest_data_2.png ├── prefect_orion_block.png ├── prefect_orion_dashboard.png ├── production_dbt.png ├── public_key_added.png ├── python_ingest_data.png ├── pythonpath.png ├── quick_run.png ├── remote-ssh-extension.png ├── remote_ssh.png ├── seed_dbt.png ├── service_basic_viewer.png ├── sftp_sien_project.png ├── sh_anaconda.png ├── source_bash.png ├── spark_shell.png ├── ssh_de_zoomcamp.png ├── ssh_i.png ├── ssh_key.png ├── ssh_key_added.png ├── ssh_ls.png ├── ssh_name_instance.png ├── staging_dbt.png ├── tar_spark.png ├── tar_xzvf.png ├── terraform-sien.png ├── terraform_apply.png ├── terraform_init.png ├── terraform_logo.png ├── terraform_plan.png ├── terraform_version.png ├── touch_config.png ├── unzip_terraform.png ├── vm_instance_name.png ├── vsc_remote_ssh.png ├── vsc_remote_ssh_connection.png ├── wget_docker_compose.png ├── which_java.png └── which_pyspark.png ├── notebooks ├── Dockerfile ├── ingest_data.py └── read_parameters_2.py ├── power_point └── data architecture.pptx ├── prefect ├── cloud │ ├── .prefectignore │ ├── Dockerfile │ ├── __pycache__ │ │ ├── ingest_data_6.cpython-310.pyc │ │ ├── ingest_data_6.cpython-311.pyc │ │ ├── read_parameters.cpython-310.pyc │ │ ├── read_parameters.cpython-311.pyc │ │ └── read_parameters_2.cpython-311.pyc │ ├── data_excel │ │ └── 2022 │ │ │ ├── VRAEM_children_2022_01_03.xlsx │ │ │ ├── VRAEM_children_2022_01_06.xlsx │ │ │ ├── VRAEM_children_2022_01_09.xlsx │ │ │ ├── children_2022_01_03.xlsx │ │ │ ├── children_2022_01_06.xlsx │ │ │ ├── children_2022_01_09.xlsx │ │ │ ├── foreign_children_2022_01_03.xlsx │ │ │ ├── foreign_children_2022_01_06.xlsx │ │ │ ├── foreign_children_2022_01_09.xlsx │ │ │ ├── pregnant_women_2022_01_06.xlsx │ │ │ └── pregnant_women_2022_01_09.xlsx │ ├── docker-requirements.txt │ ├── docker_deploy.py │ ├── ingest_data_3.py │ ├── ingest_data_4.py │ ├── ingest_data_5.py │ ├── ingest_data_6.py │ ├── main_flow-deployment.yaml │ ├── read_parameters.py │ └── requiremenets.txt └── local │ ├── __pycache__ │ └── read_parameters_2.cpython-311.pyc │ ├── data │ └── 2022 │ │ └── pregnant_women_2022_01_09.xlsx │ ├── ingest_data_2.py │ ├── read_parameters_2.py │ └── requirements.txt ├── remote_ssh.md ├── setup_gcp.md ├── setup_terraform.md ├── setup_vm.md ├── terraform ├── .terraform-version ├── main.tf └── variables.tf └── vm_configuration.md /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/README.md -------------------------------------------------------------------------------- /basics_setup/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/basics_setup/docker-compose.yml -------------------------------------------------------------------------------- /create_connection_vm.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/create_connection_vm.md -------------------------------------------------------------------------------- /create_gcp_project.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/create_gcp_project.md -------------------------------------------------------------------------------- /create_service_account.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/create_service_account.md -------------------------------------------------------------------------------- /create_vm_instance.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/create_vm_instance.md -------------------------------------------------------------------------------- /dbt/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | -------------------------------------------------------------------------------- /dbt/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/dbt/README.md -------------------------------------------------------------------------------- /dbt/analyses/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt/dbt_project.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/dbt/dbt_project.yml -------------------------------------------------------------------------------- /dbt/models/core/dim_zones_dep.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/dbt/models/core/dim_zones_dep.sql -------------------------------------------------------------------------------- /dbt/models/core/dim_zones_district.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/dbt/models/core/dim_zones_district.sql -------------------------------------------------------------------------------- /dbt/models/core/fact_sien_an.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/dbt/models/core/fact_sien_an.sql -------------------------------------------------------------------------------- /dbt/models/core/fact_sien_en.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/dbt/models/core/fact_sien_en.sql -------------------------------------------------------------------------------- /dbt/models/staging/children_Anemia_6-35m_x_DEP_2022.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/dbt/models/staging/children_Anemia_6-35m_x_DEP_2022.sql -------------------------------------------------------------------------------- /dbt/models/staging/children_Anemia_6-59m_x_DEP_2022.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/dbt/models/staging/children_Anemia_6-59m_x_DEP_2022.sql -------------------------------------------------------------------------------- /dbt/models/staging/children_EN_0-35m_x_DEP_2022.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/dbt/models/staging/children_EN_0-35m_x_DEP_2022.sql -------------------------------------------------------------------------------- /dbt/models/staging/children_EN_0-59m_x_DEP_2022.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/dbt/models/staging/children_EN_0-59m_x_DEP_2022.sql -------------------------------------------------------------------------------- /dbt/models/staging/foreign_children_Anemia_6-35m_x_DEP_2022.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/dbt/models/staging/foreign_children_Anemia_6-35m_x_DEP_2022.sql -------------------------------------------------------------------------------- /dbt/models/staging/foreign_children_Anemia_6-59m_x_DEP_2022.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/dbt/models/staging/foreign_children_Anemia_6-59m_x_DEP_2022.sql -------------------------------------------------------------------------------- /dbt/models/staging/foreign_children_EN_0-35m_x_DEP_2022.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/dbt/models/staging/foreign_children_EN_0-35m_x_DEP_2022.sql -------------------------------------------------------------------------------- /dbt/models/staging/foreign_children_EN_0-59m_x_DEP_2022.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/dbt/models/staging/foreign_children_EN_0-59m_x_DEP_2022.sql -------------------------------------------------------------------------------- /dbt/models/staging/pregnant_women_Anemia_x_DEP_2022.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/dbt/models/staging/pregnant_women_Anemia_x_DEP_2022.sql -------------------------------------------------------------------------------- /dbt/models/staging/schema.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/dbt/models/staging/schema.yml -------------------------------------------------------------------------------- /dbt/seeds/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt/seeds/lon_lat.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/dbt/seeds/lon_lat.csv -------------------------------------------------------------------------------- /dbt/seeds/ubigeous_lookup.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/dbt/seeds/ubigeous_lookup.csv -------------------------------------------------------------------------------- /dbt/snapshots/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt/tests/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/IAM_API.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/IAM_API.png -------------------------------------------------------------------------------- /images/IAM_API_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/IAM_API_2.png -------------------------------------------------------------------------------- /images/IAM_sien_project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/IAM_sien_project.png -------------------------------------------------------------------------------- /images/SIEN.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/SIEN.png -------------------------------------------------------------------------------- /images/apply_agent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/apply_agent.png -------------------------------------------------------------------------------- /images/bashrc_spark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/bashrc_spark.png -------------------------------------------------------------------------------- /images/boot_disk.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/boot_disk.png -------------------------------------------------------------------------------- /images/cat_gcp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/cat_gcp.png -------------------------------------------------------------------------------- /images/compute_engine_api.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/compute_engine_api.png -------------------------------------------------------------------------------- /images/compute_engine_vm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/compute_engine_vm.png -------------------------------------------------------------------------------- /images/connect_to_host.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/connect_to_host.png -------------------------------------------------------------------------------- /images/connection_pgadmin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/connection_pgadmin.png -------------------------------------------------------------------------------- /images/core_dbt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/core_dbt.png -------------------------------------------------------------------------------- /images/create_project_gcp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/create_project_gcp.png -------------------------------------------------------------------------------- /images/create_service_account.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/create_service_account.png -------------------------------------------------------------------------------- /images/create_vm_instance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/create_vm_instance.png -------------------------------------------------------------------------------- /images/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/dashboard.png -------------------------------------------------------------------------------- /images/data_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/data_architecture.png -------------------------------------------------------------------------------- /images/data_architecture_chosen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/data_architecture_chosen.png -------------------------------------------------------------------------------- /images/deployment_apply.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/deployment_apply.png -------------------------------------------------------------------------------- /images/deployment_build.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/deployment_build.png -------------------------------------------------------------------------------- /images/deployment_build_cmd.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/deployment_build_cmd.png -------------------------------------------------------------------------------- /images/docker_block_configuration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/docker_block_configuration.png -------------------------------------------------------------------------------- /images/docker_block_prefect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/docker_block_prefect.png -------------------------------------------------------------------------------- /images/docker_build_prefect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/docker_build_prefect.png -------------------------------------------------------------------------------- /images/docker_compose_added.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/docker_compose_added.png -------------------------------------------------------------------------------- /images/docker_compose_path.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/docker_compose_path.png -------------------------------------------------------------------------------- /images/docker_compose_ps.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/docker_compose_ps.png -------------------------------------------------------------------------------- /images/docker_compose_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/docker_compose_structure.png -------------------------------------------------------------------------------- /images/docker_deploy_py.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/docker_deploy_py.png -------------------------------------------------------------------------------- /images/docker_push_prefect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/docker_push_prefect.png -------------------------------------------------------------------------------- /images/docker_run_hw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/docker_run_hw.png -------------------------------------------------------------------------------- /images/e2_instance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/e2_instance.png -------------------------------------------------------------------------------- /images/edit_permissions_terraform_sien.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/edit_permissions_terraform_sien.png -------------------------------------------------------------------------------- /images/environments_dbt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/environments_dbt.png -------------------------------------------------------------------------------- /images/establishment_level.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/establishment_level.png -------------------------------------------------------------------------------- /images/export_google_credentials.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/export_google_credentials.png -------------------------------------------------------------------------------- /images/files_permissions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/files_permissions.png -------------------------------------------------------------------------------- /images/finally_create_vm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/finally_create_vm.png -------------------------------------------------------------------------------- /images/gc_file_json.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/gc_file_json.png -------------------------------------------------------------------------------- /images/gcp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/gcp.png -------------------------------------------------------------------------------- /images/gcs_bucket_block.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/gcs_bucket_block.png -------------------------------------------------------------------------------- /images/generate_key_service_account.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/generate_key_service_account.png -------------------------------------------------------------------------------- /images/google_compute_engine.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/google_compute_engine.png -------------------------------------------------------------------------------- /images/indicator_sien.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/indicator_sien.png -------------------------------------------------------------------------------- /images/informes_sien.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/informes_sien.png -------------------------------------------------------------------------------- /images/java_path.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/java_path.png -------------------------------------------------------------------------------- /images/lineage_dbt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/lineage_dbt.png -------------------------------------------------------------------------------- /images/metadata_compute_engine.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/metadata_compute_engine.png -------------------------------------------------------------------------------- /images/mkdir_bin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/mkdir_bin.png -------------------------------------------------------------------------------- /images/more_permissions_terraform.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/more_permissions_terraform.png -------------------------------------------------------------------------------- /images/name_server.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/name_server.png -------------------------------------------------------------------------------- /images/name_vm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/name_vm.png -------------------------------------------------------------------------------- /images/new_project_gcp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/new_project_gcp.png -------------------------------------------------------------------------------- /images/orion_prefect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/orion_prefect.png -------------------------------------------------------------------------------- /images/pg_admin_ingest_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/pg_admin_ingest_data.png -------------------------------------------------------------------------------- /images/pgadmin_browser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/pgadmin_browser.png -------------------------------------------------------------------------------- /images/port_forwarding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/port_forwarding.png -------------------------------------------------------------------------------- /images/prefect_ingest_data_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/prefect_ingest_data_2.png -------------------------------------------------------------------------------- /images/prefect_orion_block.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/prefect_orion_block.png -------------------------------------------------------------------------------- /images/prefect_orion_dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/prefect_orion_dashboard.png -------------------------------------------------------------------------------- /images/production_dbt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/production_dbt.png -------------------------------------------------------------------------------- /images/public_key_added.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/public_key_added.png -------------------------------------------------------------------------------- /images/python_ingest_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/python_ingest_data.png -------------------------------------------------------------------------------- /images/pythonpath.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/pythonpath.png -------------------------------------------------------------------------------- /images/quick_run.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/quick_run.png -------------------------------------------------------------------------------- /images/remote-ssh-extension.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/remote-ssh-extension.png -------------------------------------------------------------------------------- /images/remote_ssh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/remote_ssh.png -------------------------------------------------------------------------------- /images/seed_dbt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/seed_dbt.png -------------------------------------------------------------------------------- /images/service_basic_viewer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/service_basic_viewer.png -------------------------------------------------------------------------------- /images/sftp_sien_project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/sftp_sien_project.png -------------------------------------------------------------------------------- /images/sh_anaconda.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/sh_anaconda.png -------------------------------------------------------------------------------- /images/source_bash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/source_bash.png -------------------------------------------------------------------------------- /images/spark_shell.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/spark_shell.png -------------------------------------------------------------------------------- /images/ssh_de_zoomcamp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/ssh_de_zoomcamp.png -------------------------------------------------------------------------------- /images/ssh_i.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/ssh_i.png -------------------------------------------------------------------------------- /images/ssh_key.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/ssh_key.png -------------------------------------------------------------------------------- /images/ssh_key_added.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/ssh_key_added.png -------------------------------------------------------------------------------- /images/ssh_ls.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/ssh_ls.png -------------------------------------------------------------------------------- /images/ssh_name_instance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/ssh_name_instance.png -------------------------------------------------------------------------------- /images/staging_dbt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/staging_dbt.png -------------------------------------------------------------------------------- /images/tar_spark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/tar_spark.png -------------------------------------------------------------------------------- /images/tar_xzvf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/tar_xzvf.png -------------------------------------------------------------------------------- /images/terraform-sien.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/terraform-sien.png -------------------------------------------------------------------------------- /images/terraform_apply.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/terraform_apply.png -------------------------------------------------------------------------------- /images/terraform_init.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/terraform_init.png -------------------------------------------------------------------------------- /images/terraform_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/terraform_logo.png -------------------------------------------------------------------------------- /images/terraform_plan.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/terraform_plan.png -------------------------------------------------------------------------------- /images/terraform_version.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/terraform_version.png -------------------------------------------------------------------------------- /images/touch_config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/touch_config.png -------------------------------------------------------------------------------- /images/unzip_terraform.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/unzip_terraform.png -------------------------------------------------------------------------------- /images/vm_instance_name.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/vm_instance_name.png -------------------------------------------------------------------------------- /images/vsc_remote_ssh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/vsc_remote_ssh.png -------------------------------------------------------------------------------- /images/vsc_remote_ssh_connection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/vsc_remote_ssh_connection.png -------------------------------------------------------------------------------- /images/wget_docker_compose.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/wget_docker_compose.png -------------------------------------------------------------------------------- /images/which_java.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/which_java.png -------------------------------------------------------------------------------- /images/which_pyspark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/images/which_pyspark.png -------------------------------------------------------------------------------- /notebooks/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/notebooks/Dockerfile -------------------------------------------------------------------------------- /notebooks/ingest_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/notebooks/ingest_data.py -------------------------------------------------------------------------------- /notebooks/read_parameters_2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/notebooks/read_parameters_2.py -------------------------------------------------------------------------------- /power_point/data architecture.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/power_point/data architecture.pptx -------------------------------------------------------------------------------- /prefect/cloud/.prefectignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/.prefectignore -------------------------------------------------------------------------------- /prefect/cloud/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/Dockerfile -------------------------------------------------------------------------------- /prefect/cloud/__pycache__/ingest_data_6.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/__pycache__/ingest_data_6.cpython-310.pyc -------------------------------------------------------------------------------- /prefect/cloud/__pycache__/ingest_data_6.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/__pycache__/ingest_data_6.cpython-311.pyc -------------------------------------------------------------------------------- /prefect/cloud/__pycache__/read_parameters.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/__pycache__/read_parameters.cpython-310.pyc -------------------------------------------------------------------------------- /prefect/cloud/__pycache__/read_parameters.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/__pycache__/read_parameters.cpython-311.pyc -------------------------------------------------------------------------------- /prefect/cloud/__pycache__/read_parameters_2.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/__pycache__/read_parameters_2.cpython-311.pyc -------------------------------------------------------------------------------- /prefect/cloud/data_excel/2022/VRAEM_children_2022_01_03.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/data_excel/2022/VRAEM_children_2022_01_03.xlsx -------------------------------------------------------------------------------- /prefect/cloud/data_excel/2022/VRAEM_children_2022_01_06.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/data_excel/2022/VRAEM_children_2022_01_06.xlsx -------------------------------------------------------------------------------- /prefect/cloud/data_excel/2022/VRAEM_children_2022_01_09.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/data_excel/2022/VRAEM_children_2022_01_09.xlsx -------------------------------------------------------------------------------- /prefect/cloud/data_excel/2022/children_2022_01_03.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/data_excel/2022/children_2022_01_03.xlsx -------------------------------------------------------------------------------- /prefect/cloud/data_excel/2022/children_2022_01_06.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/data_excel/2022/children_2022_01_06.xlsx -------------------------------------------------------------------------------- /prefect/cloud/data_excel/2022/children_2022_01_09.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/data_excel/2022/children_2022_01_09.xlsx -------------------------------------------------------------------------------- /prefect/cloud/data_excel/2022/foreign_children_2022_01_03.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/data_excel/2022/foreign_children_2022_01_03.xlsx -------------------------------------------------------------------------------- /prefect/cloud/data_excel/2022/foreign_children_2022_01_06.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/data_excel/2022/foreign_children_2022_01_06.xlsx -------------------------------------------------------------------------------- /prefect/cloud/data_excel/2022/foreign_children_2022_01_09.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/data_excel/2022/foreign_children_2022_01_09.xlsx -------------------------------------------------------------------------------- /prefect/cloud/data_excel/2022/pregnant_women_2022_01_06.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/data_excel/2022/pregnant_women_2022_01_06.xlsx -------------------------------------------------------------------------------- /prefect/cloud/data_excel/2022/pregnant_women_2022_01_09.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/data_excel/2022/pregnant_women_2022_01_09.xlsx -------------------------------------------------------------------------------- /prefect/cloud/docker-requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/docker-requirements.txt -------------------------------------------------------------------------------- /prefect/cloud/docker_deploy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/docker_deploy.py -------------------------------------------------------------------------------- /prefect/cloud/ingest_data_3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/ingest_data_3.py -------------------------------------------------------------------------------- /prefect/cloud/ingest_data_4.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/ingest_data_4.py -------------------------------------------------------------------------------- /prefect/cloud/ingest_data_5.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/ingest_data_5.py -------------------------------------------------------------------------------- /prefect/cloud/ingest_data_6.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/ingest_data_6.py -------------------------------------------------------------------------------- /prefect/cloud/main_flow-deployment.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/main_flow-deployment.yaml -------------------------------------------------------------------------------- /prefect/cloud/read_parameters.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/read_parameters.py -------------------------------------------------------------------------------- /prefect/cloud/requiremenets.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/cloud/requiremenets.txt -------------------------------------------------------------------------------- /prefect/local/__pycache__/read_parameters_2.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/local/__pycache__/read_parameters_2.cpython-311.pyc -------------------------------------------------------------------------------- /prefect/local/data/2022/pregnant_women_2022_01_09.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/local/data/2022/pregnant_women_2022_01_09.xlsx -------------------------------------------------------------------------------- /prefect/local/ingest_data_2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/local/ingest_data_2.py -------------------------------------------------------------------------------- /prefect/local/read_parameters_2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/local/read_parameters_2.py -------------------------------------------------------------------------------- /prefect/local/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/prefect/local/requirements.txt -------------------------------------------------------------------------------- /remote_ssh.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/remote_ssh.md -------------------------------------------------------------------------------- /setup_gcp.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/setup_gcp.md -------------------------------------------------------------------------------- /setup_terraform.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/setup_terraform.md -------------------------------------------------------------------------------- /setup_vm.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/setup_vm.md -------------------------------------------------------------------------------- /terraform/.terraform-version: -------------------------------------------------------------------------------- 1 | 1.0.2 2 | -------------------------------------------------------------------------------- /terraform/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/terraform/main.tf -------------------------------------------------------------------------------- /terraform/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/terraform/variables.tf -------------------------------------------------------------------------------- /vm_configuration.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JesusAcuna/data-engineering-project/HEAD/vm_configuration.md --------------------------------------------------------------------------------