├── .coveragerc ├── .github └── workflows │ ├── cd-workflow.yml │ └── ci-workflow.yml ├── .gitignore ├── README.md ├── citibike_etl ├── dlt │ ├── 01_bronze │ │ └── 01_bronze_citibike.ipynb │ ├── 02_silver │ │ └── 02_silver_citibike.ipynb │ └── 03_gold │ │ ├── 03_gold_citibike_daily_ride_summary.ipynb │ │ └── 03_gold_citibike_daily_station_performance.ipynb ├── notebooks │ ├── 01_bronze │ │ └── 01_bronze_citibike.ipynb │ ├── 02_silver │ │ └── 02_silver_citibike.ipynb │ └── 03_gold │ │ ├── 03_gold_citibike_daily_ride_summary.ipynb │ │ └── 03_gold_citibike_daily_station_performance.ipynb └── scripts │ ├── 01_bronze │ └── 01_bronze_citibike.py │ ├── 02_silver │ └── 02_silver_citibike.py │ └── 03_gold │ ├── 03_gold_citibike_daily_ride_summary.py │ └── 03_gold_citibike_daily_station_performance.py ├── databricks.yml ├── docs ├── cicd-workflow.drawio ├── citibike_project_diagram.drawio └── citibike_source_data_structure.png ├── requirements-dbc.txt ├── requirements-pyspark.txt ├── resources ├── citibike_etl_pipeline.dlt.yml ├── citibike_etl_pipeline_nb.job.yml ├── citibike_etl_pipeline_py.job.yml └── clusters.yml ├── setup.py ├── src ├── __init__.py ├── citibike │ ├── __init__.py │ └── citibike_utils.py ├── dab_project │ ├── __init__.py │ └── main.py └── utils │ ├── __init__.py │ └── datetime_utils.py └── tests ├── conftest.py ├── test_citibike_utils.py └── test_datetime_utils.py /.coveragerc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/.coveragerc -------------------------------------------------------------------------------- /.github/workflows/cd-workflow.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/.github/workflows/cd-workflow.yml -------------------------------------------------------------------------------- /.github/workflows/ci-workflow.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/.github/workflows/ci-workflow.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/README.md -------------------------------------------------------------------------------- /citibike_etl/dlt/01_bronze/01_bronze_citibike.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/citibike_etl/dlt/01_bronze/01_bronze_citibike.ipynb -------------------------------------------------------------------------------- /citibike_etl/dlt/02_silver/02_silver_citibike.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/citibike_etl/dlt/02_silver/02_silver_citibike.ipynb -------------------------------------------------------------------------------- /citibike_etl/dlt/03_gold/03_gold_citibike_daily_ride_summary.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/citibike_etl/dlt/03_gold/03_gold_citibike_daily_ride_summary.ipynb -------------------------------------------------------------------------------- /citibike_etl/dlt/03_gold/03_gold_citibike_daily_station_performance.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/citibike_etl/dlt/03_gold/03_gold_citibike_daily_station_performance.ipynb -------------------------------------------------------------------------------- /citibike_etl/notebooks/01_bronze/01_bronze_citibike.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/citibike_etl/notebooks/01_bronze/01_bronze_citibike.ipynb -------------------------------------------------------------------------------- /citibike_etl/notebooks/02_silver/02_silver_citibike.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/citibike_etl/notebooks/02_silver/02_silver_citibike.ipynb -------------------------------------------------------------------------------- /citibike_etl/notebooks/03_gold/03_gold_citibike_daily_ride_summary.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/citibike_etl/notebooks/03_gold/03_gold_citibike_daily_ride_summary.ipynb -------------------------------------------------------------------------------- /citibike_etl/notebooks/03_gold/03_gold_citibike_daily_station_performance.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/citibike_etl/notebooks/03_gold/03_gold_citibike_daily_station_performance.ipynb -------------------------------------------------------------------------------- /citibike_etl/scripts/01_bronze/01_bronze_citibike.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/citibike_etl/scripts/01_bronze/01_bronze_citibike.py -------------------------------------------------------------------------------- /citibike_etl/scripts/02_silver/02_silver_citibike.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/citibike_etl/scripts/02_silver/02_silver_citibike.py -------------------------------------------------------------------------------- /citibike_etl/scripts/03_gold/03_gold_citibike_daily_ride_summary.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/citibike_etl/scripts/03_gold/03_gold_citibike_daily_ride_summary.py -------------------------------------------------------------------------------- /citibike_etl/scripts/03_gold/03_gold_citibike_daily_station_performance.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/citibike_etl/scripts/03_gold/03_gold_citibike_daily_station_performance.py -------------------------------------------------------------------------------- /databricks.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/databricks.yml -------------------------------------------------------------------------------- /docs/cicd-workflow.drawio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/docs/cicd-workflow.drawio -------------------------------------------------------------------------------- /docs/citibike_project_diagram.drawio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/docs/citibike_project_diagram.drawio -------------------------------------------------------------------------------- /docs/citibike_source_data_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/docs/citibike_source_data_structure.png -------------------------------------------------------------------------------- /requirements-dbc.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/requirements-dbc.txt -------------------------------------------------------------------------------- /requirements-pyspark.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/requirements-pyspark.txt -------------------------------------------------------------------------------- /resources/citibike_etl_pipeline.dlt.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/resources/citibike_etl_pipeline.dlt.yml -------------------------------------------------------------------------------- /resources/citibike_etl_pipeline_nb.job.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/resources/citibike_etl_pipeline_nb.job.yml -------------------------------------------------------------------------------- /resources/citibike_etl_pipeline_py.job.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/resources/citibike_etl_pipeline_py.job.yml -------------------------------------------------------------------------------- /resources/clusters.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/resources/clusters.yml -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/setup.py -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/citibike/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/citibike/citibike_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/src/citibike/citibike_utils.py -------------------------------------------------------------------------------- /src/dab_project/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/dab_project/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/src/dab_project/main.py -------------------------------------------------------------------------------- /src/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/utils/datetime_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/src/utils/datetime_utils.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/tests/conftest.py -------------------------------------------------------------------------------- /tests/test_citibike_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/tests/test_citibike_utils.py -------------------------------------------------------------------------------- /tests/test_datetime_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pathfinder-analytics-uk/dab_project/HEAD/tests/test_datetime_utils.py --------------------------------------------------------------------------------