├── LICENSE ├── README.md ├── week1 ├── .gitignore ├── .ipynb_checkpoints │ └── postgres_connection-checkpoint.ipynb ├── Dockerfile ├── README.md ├── docker-compose.yaml ├── ingest_data.py ├── postgres_connection.ipynb └── terraform │ ├── .terraform-version │ ├── main.tf │ └── variables.tf ├── week2 ├── .gitignore ├── README.md └── airflow │ ├── Dockerfile │ ├── dags │ └── dag_ingestion_gcs.py │ ├── docker-compose.yaml │ └── requirements.txt ├── week3 ├── .gitignore ├── README.md └── airflow │ ├── dags │ ├── __pycache__ │ │ └── gcp_to_bq_dag.cpython-37.pyc │ └── gcp_to_bq_dag.py │ └── docker-compose.yaml ├── week4 ├── README.md ├── data_to_gcs │ ├── .gitignore │ └── upload_to_gcs.py └── dbt │ ├── .gitignore │ ├── analyses │ └── .gitkeep │ ├── data │ └── taxi_zone.csv │ ├── dbt_project.yml │ ├── macros │ ├── .gitkeep │ └── get_payment_type_description.sql │ ├── models │ ├── core │ │ ├── dim_zones.sql │ │ ├── dm_monthly_zone_revenue.sql │ │ ├── fact_trips.sql │ │ └── schema.yml │ └── staging │ │ ├── schema.yml │ │ ├── stg_green_tripdata.sql │ │ └── stg_yellow_tripdata.sql │ ├── packages.yml │ ├── profiles.yml │ ├── seeds │ └── .gitkeep │ ├── snapshots │ └── .gitkeep │ └── tests │ └── .gitkeep └── week6 ├── README.md ├── avro_example ├── consumer.py ├── data │ └── rides_new.csv ├── producer.py ├── taxi_ride_key.avsc └── taxi_ride_value.avsc ├── consumer.py ├── docker-compose.yml ├── producer.py ├── requirements.txt └── streams ├── __pycache__ └── taxi_rides.cpython-37.pyc ├── branch_price.py ├── producer_taxi_json.py ├── stream.py └── taxi_rides.py /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/README.md -------------------------------------------------------------------------------- /week1/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week1/.gitignore -------------------------------------------------------------------------------- /week1/.ipynb_checkpoints/postgres_connection-checkpoint.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week1/.ipynb_checkpoints/postgres_connection-checkpoint.ipynb -------------------------------------------------------------------------------- /week1/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week1/Dockerfile -------------------------------------------------------------------------------- /week1/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week1/README.md -------------------------------------------------------------------------------- /week1/docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week1/docker-compose.yaml -------------------------------------------------------------------------------- /week1/ingest_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week1/ingest_data.py -------------------------------------------------------------------------------- /week1/postgres_connection.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week1/postgres_connection.ipynb -------------------------------------------------------------------------------- /week1/terraform/.terraform-version: -------------------------------------------------------------------------------- 1 | 1.2.0 -------------------------------------------------------------------------------- /week1/terraform/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week1/terraform/main.tf -------------------------------------------------------------------------------- /week1/terraform/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week1/terraform/variables.tf -------------------------------------------------------------------------------- /week2/.gitignore: -------------------------------------------------------------------------------- 1 | google/ 2 | .env 3 | logs/ 4 | plugins/ 5 | **__pycache__/ -------------------------------------------------------------------------------- /week2/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week2/README.md -------------------------------------------------------------------------------- /week2/airflow/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week2/airflow/Dockerfile -------------------------------------------------------------------------------- /week2/airflow/dags/dag_ingestion_gcs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week2/airflow/dags/dag_ingestion_gcs.py -------------------------------------------------------------------------------- /week2/airflow/docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week2/airflow/docker-compose.yaml -------------------------------------------------------------------------------- /week2/airflow/requirements.txt: -------------------------------------------------------------------------------- 1 | apache-airflow-providers-google 2 | pyarrow 3 | -------------------------------------------------------------------------------- /week3/.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | logs/ -------------------------------------------------------------------------------- /week3/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week3/README.md -------------------------------------------------------------------------------- /week3/airflow/dags/__pycache__/gcp_to_bq_dag.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week3/airflow/dags/__pycache__/gcp_to_bq_dag.cpython-37.pyc -------------------------------------------------------------------------------- /week3/airflow/dags/gcp_to_bq_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week3/airflow/dags/gcp_to_bq_dag.py -------------------------------------------------------------------------------- /week3/airflow/docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week3/airflow/docker-compose.yaml -------------------------------------------------------------------------------- /week4/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week4/README.md -------------------------------------------------------------------------------- /week4/data_to_gcs/.gitignore: -------------------------------------------------------------------------------- 1 | google/ -------------------------------------------------------------------------------- /week4/data_to_gcs/upload_to_gcs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week4/data_to_gcs/upload_to_gcs.py -------------------------------------------------------------------------------- /week4/dbt/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | -------------------------------------------------------------------------------- /week4/dbt/analyses/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /week4/dbt/data/taxi_zone.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week4/dbt/data/taxi_zone.csv -------------------------------------------------------------------------------- /week4/dbt/dbt_project.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week4/dbt/dbt_project.yml -------------------------------------------------------------------------------- /week4/dbt/macros/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /week4/dbt/macros/get_payment_type_description.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week4/dbt/macros/get_payment_type_description.sql -------------------------------------------------------------------------------- /week4/dbt/models/core/dim_zones.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week4/dbt/models/core/dim_zones.sql -------------------------------------------------------------------------------- /week4/dbt/models/core/dm_monthly_zone_revenue.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week4/dbt/models/core/dm_monthly_zone_revenue.sql -------------------------------------------------------------------------------- /week4/dbt/models/core/fact_trips.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week4/dbt/models/core/fact_trips.sql -------------------------------------------------------------------------------- /week4/dbt/models/core/schema.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week4/dbt/models/core/schema.yml -------------------------------------------------------------------------------- /week4/dbt/models/staging/schema.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week4/dbt/models/staging/schema.yml -------------------------------------------------------------------------------- /week4/dbt/models/staging/stg_green_tripdata.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week4/dbt/models/staging/stg_green_tripdata.sql -------------------------------------------------------------------------------- /week4/dbt/models/staging/stg_yellow_tripdata.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week4/dbt/models/staging/stg_yellow_tripdata.sql -------------------------------------------------------------------------------- /week4/dbt/packages.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week4/dbt/packages.yml -------------------------------------------------------------------------------- /week4/dbt/profiles.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week4/dbt/profiles.yml -------------------------------------------------------------------------------- /week4/dbt/seeds/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /week4/dbt/snapshots/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /week4/dbt/tests/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /week6/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week6/README.md -------------------------------------------------------------------------------- /week6/avro_example/consumer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week6/avro_example/consumer.py -------------------------------------------------------------------------------- /week6/avro_example/data/rides_new.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week6/avro_example/data/rides_new.csv -------------------------------------------------------------------------------- /week6/avro_example/producer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week6/avro_example/producer.py -------------------------------------------------------------------------------- /week6/avro_example/taxi_ride_key.avsc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week6/avro_example/taxi_ride_key.avsc -------------------------------------------------------------------------------- /week6/avro_example/taxi_ride_value.avsc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week6/avro_example/taxi_ride_value.avsc -------------------------------------------------------------------------------- /week6/consumer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week6/consumer.py -------------------------------------------------------------------------------- /week6/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week6/docker-compose.yml -------------------------------------------------------------------------------- /week6/producer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week6/producer.py -------------------------------------------------------------------------------- /week6/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week6/requirements.txt -------------------------------------------------------------------------------- /week6/streams/__pycache__/taxi_rides.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week6/streams/__pycache__/taxi_rides.cpython-37.pyc -------------------------------------------------------------------------------- /week6/streams/branch_price.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week6/streams/branch_price.py -------------------------------------------------------------------------------- /week6/streams/producer_taxi_json.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week6/streams/producer_taxi_json.py -------------------------------------------------------------------------------- /week6/streams/stream.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week6/streams/stream.py -------------------------------------------------------------------------------- /week6/streams/taxi_rides.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcrespoo/data-engineering-bootcamp/HEAD/week6/streams/taxi_rides.py --------------------------------------------------------------------------------