├── .gitignore ├── 00_data_source_exploration ├── energy_demand_api.ipynb ├── environment.yml ├── readme.md ├── weather_forecast_data.ipynb └── weather_station.ipynb ├── 01_terraform ├── .terraform-version ├── main.tf ├── readme.md ├── variables.tf └── vm_init.sh ├── 02_airflow ├── Dockerfile ├── dags │ ├── batch_predict.py │ ├── batch_predict_dag.py │ ├── fix_owm_schema_dag.py │ ├── gcloud_helpers.py │ ├── ingest_historical_weather_data_dag.py │ ├── ingest_live_hourly_weather_dag.py │ ├── ingest_raw_electricity_data_dag.py │ └── ingest_weather_forecast_dag.py ├── docker-compose.yml ├── readme.md └── requirements.txt ├── 03_dbt ├── .gitignore ├── README.md ├── analysis │ └── .gitkeep ├── dbt_project.yml ├── macros │ └── .gitkeep ├── models │ ├── core │ │ ├── fact_eia_demand_forecast.sql │ │ ├── fact_eia_demand_historical.sql │ │ ├── ml_model_metrics.sql │ │ ├── recorded_temperature.sql │ │ └── schema.yml │ ├── mart │ │ └── joined_temp_and_demand.sql │ └── staging │ │ ├── cast_isd_weather.sql │ │ ├── cast_owm_weather.sql │ │ └── union_weather_station.sql ├── seeds │ ├── .gitkeep │ ├── isd_stations.csv │ └── properties.yml ├── snapshots │ └── .gitkeep └── tests │ └── .gitkeep ├── 04_dashboard ├── .streamlit │ └── config.toml ├── app.py ├── info.py ├── readme.md └── requirements.txt ├── 05_model_training ├── 01_EDA.ipynb ├── 02_simple_linear_model.ipynb ├── 03_mlflow.ipynb ├── Untitled.ipynb ├── mlflow_docker │ ├── Dockerfile │ ├── docker-compose.yml │ └── requirements.txt ├── readme.md ├── requirements.txt └── ts_diagnostics.py ├── 06_deployment ├── batch_predict.py └── readme.md ├── 07_monitoring ├── 00_EDA.ipynb ├── app.py ├── readme.md └── requirements.txt ├── img ├── Architecture.PNG ├── batch_predict_dag.PNG ├── dashboard.PNG ├── dashboard1.PNG ├── dashboard_mockup.png ├── dbt_demand.PNG ├── dbt_demand_forecast.PNG ├── dbt_monitoring.PNG ├── dbt_temp.PNG ├── de_architecture.PNG ├── eia_dag.PNG ├── mlflow1.PNG ├── mlflow2.PNG ├── mlops_architecture.PNG ├── monitoring_dashboard_1.PNG ├── monitoring_dashboard_2.PNG ├── noaa_dag.PNG ├── owm_dag.PNG ├── pipeline.png └── weather_forecast_dag.PNG ├── proposal.md ├── readme.md └── steps_to_recreate_project ├── images ├── 01_service_account.PNG ├── 02_service_account_key.PNG └── 03_vm.PNG └── readme.md /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/.gitignore -------------------------------------------------------------------------------- /00_data_source_exploration/energy_demand_api.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/00_data_source_exploration/energy_demand_api.ipynb -------------------------------------------------------------------------------- /00_data_source_exploration/environment.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/00_data_source_exploration/environment.yml -------------------------------------------------------------------------------- /00_data_source_exploration/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/00_data_source_exploration/readme.md -------------------------------------------------------------------------------- /00_data_source_exploration/weather_forecast_data.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/00_data_source_exploration/weather_forecast_data.ipynb -------------------------------------------------------------------------------- /00_data_source_exploration/weather_station.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/00_data_source_exploration/weather_station.ipynb -------------------------------------------------------------------------------- /01_terraform/.terraform-version: -------------------------------------------------------------------------------- 1 | 1.0.2 -------------------------------------------------------------------------------- /01_terraform/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/01_terraform/main.tf -------------------------------------------------------------------------------- /01_terraform/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/01_terraform/readme.md -------------------------------------------------------------------------------- /01_terraform/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/01_terraform/variables.tf -------------------------------------------------------------------------------- /01_terraform/vm_init.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/01_terraform/vm_init.sh -------------------------------------------------------------------------------- /02_airflow/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/02_airflow/Dockerfile -------------------------------------------------------------------------------- /02_airflow/dags/batch_predict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/02_airflow/dags/batch_predict.py -------------------------------------------------------------------------------- /02_airflow/dags/batch_predict_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/02_airflow/dags/batch_predict_dag.py -------------------------------------------------------------------------------- /02_airflow/dags/fix_owm_schema_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/02_airflow/dags/fix_owm_schema_dag.py -------------------------------------------------------------------------------- /02_airflow/dags/gcloud_helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/02_airflow/dags/gcloud_helpers.py -------------------------------------------------------------------------------- /02_airflow/dags/ingest_historical_weather_data_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/02_airflow/dags/ingest_historical_weather_data_dag.py -------------------------------------------------------------------------------- /02_airflow/dags/ingest_live_hourly_weather_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/02_airflow/dags/ingest_live_hourly_weather_dag.py -------------------------------------------------------------------------------- /02_airflow/dags/ingest_raw_electricity_data_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/02_airflow/dags/ingest_raw_electricity_data_dag.py -------------------------------------------------------------------------------- /02_airflow/dags/ingest_weather_forecast_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/02_airflow/dags/ingest_weather_forecast_dag.py -------------------------------------------------------------------------------- /02_airflow/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/02_airflow/docker-compose.yml -------------------------------------------------------------------------------- /02_airflow/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/02_airflow/readme.md -------------------------------------------------------------------------------- /02_airflow/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/02_airflow/requirements.txt -------------------------------------------------------------------------------- /03_dbt/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | -------------------------------------------------------------------------------- /03_dbt/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/03_dbt/README.md -------------------------------------------------------------------------------- /03_dbt/analysis/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /03_dbt/dbt_project.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/03_dbt/dbt_project.yml -------------------------------------------------------------------------------- /03_dbt/macros/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /03_dbt/models/core/fact_eia_demand_forecast.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/03_dbt/models/core/fact_eia_demand_forecast.sql -------------------------------------------------------------------------------- /03_dbt/models/core/fact_eia_demand_historical.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/03_dbt/models/core/fact_eia_demand_historical.sql -------------------------------------------------------------------------------- /03_dbt/models/core/ml_model_metrics.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/03_dbt/models/core/ml_model_metrics.sql -------------------------------------------------------------------------------- /03_dbt/models/core/recorded_temperature.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/03_dbt/models/core/recorded_temperature.sql -------------------------------------------------------------------------------- /03_dbt/models/core/schema.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/03_dbt/models/core/schema.yml -------------------------------------------------------------------------------- /03_dbt/models/mart/joined_temp_and_demand.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/03_dbt/models/mart/joined_temp_and_demand.sql -------------------------------------------------------------------------------- /03_dbt/models/staging/cast_isd_weather.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/03_dbt/models/staging/cast_isd_weather.sql -------------------------------------------------------------------------------- /03_dbt/models/staging/cast_owm_weather.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/03_dbt/models/staging/cast_owm_weather.sql -------------------------------------------------------------------------------- /03_dbt/models/staging/union_weather_station.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/03_dbt/models/staging/union_weather_station.sql -------------------------------------------------------------------------------- /03_dbt/seeds/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /03_dbt/seeds/isd_stations.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/03_dbt/seeds/isd_stations.csv -------------------------------------------------------------------------------- /03_dbt/seeds/properties.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/03_dbt/seeds/properties.yml -------------------------------------------------------------------------------- /03_dbt/snapshots/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /03_dbt/tests/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /04_dashboard/.streamlit/config.toml: -------------------------------------------------------------------------------- 1 | [theme] 2 | base="dark" -------------------------------------------------------------------------------- /04_dashboard/app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/04_dashboard/app.py -------------------------------------------------------------------------------- /04_dashboard/info.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/04_dashboard/info.py -------------------------------------------------------------------------------- /04_dashboard/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/04_dashboard/readme.md -------------------------------------------------------------------------------- /04_dashboard/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/04_dashboard/requirements.txt -------------------------------------------------------------------------------- /05_model_training/01_EDA.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/05_model_training/01_EDA.ipynb -------------------------------------------------------------------------------- /05_model_training/02_simple_linear_model.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/05_model_training/02_simple_linear_model.ipynb -------------------------------------------------------------------------------- /05_model_training/03_mlflow.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/05_model_training/03_mlflow.ipynb -------------------------------------------------------------------------------- /05_model_training/Untitled.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/05_model_training/Untitled.ipynb -------------------------------------------------------------------------------- /05_model_training/mlflow_docker/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/05_model_training/mlflow_docker/Dockerfile -------------------------------------------------------------------------------- /05_model_training/mlflow_docker/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/05_model_training/mlflow_docker/docker-compose.yml -------------------------------------------------------------------------------- /05_model_training/mlflow_docker/requirements.txt: -------------------------------------------------------------------------------- 1 | mlflow 2 | google-cloud-storage 3 | psycopg2-binary 4 | 5 | -------------------------------------------------------------------------------- /05_model_training/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/05_model_training/readme.md -------------------------------------------------------------------------------- /05_model_training/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/05_model_training/requirements.txt -------------------------------------------------------------------------------- /05_model_training/ts_diagnostics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/05_model_training/ts_diagnostics.py -------------------------------------------------------------------------------- /06_deployment/batch_predict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/06_deployment/batch_predict.py -------------------------------------------------------------------------------- /06_deployment/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/06_deployment/readme.md -------------------------------------------------------------------------------- /07_monitoring/00_EDA.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/07_monitoring/00_EDA.ipynb -------------------------------------------------------------------------------- /07_monitoring/app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/07_monitoring/app.py -------------------------------------------------------------------------------- /07_monitoring/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/07_monitoring/readme.md -------------------------------------------------------------------------------- /07_monitoring/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/07_monitoring/requirements.txt -------------------------------------------------------------------------------- /img/Architecture.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/Architecture.PNG -------------------------------------------------------------------------------- /img/batch_predict_dag.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/batch_predict_dag.PNG -------------------------------------------------------------------------------- /img/dashboard.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/dashboard.PNG -------------------------------------------------------------------------------- /img/dashboard1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/dashboard1.PNG -------------------------------------------------------------------------------- /img/dashboard_mockup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/dashboard_mockup.png -------------------------------------------------------------------------------- /img/dbt_demand.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/dbt_demand.PNG -------------------------------------------------------------------------------- /img/dbt_demand_forecast.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/dbt_demand_forecast.PNG -------------------------------------------------------------------------------- /img/dbt_monitoring.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/dbt_monitoring.PNG -------------------------------------------------------------------------------- /img/dbt_temp.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/dbt_temp.PNG -------------------------------------------------------------------------------- /img/de_architecture.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/de_architecture.PNG -------------------------------------------------------------------------------- /img/eia_dag.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/eia_dag.PNG -------------------------------------------------------------------------------- /img/mlflow1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/mlflow1.PNG -------------------------------------------------------------------------------- /img/mlflow2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/mlflow2.PNG -------------------------------------------------------------------------------- /img/mlops_architecture.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/mlops_architecture.PNG -------------------------------------------------------------------------------- /img/monitoring_dashboard_1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/monitoring_dashboard_1.PNG -------------------------------------------------------------------------------- /img/monitoring_dashboard_2.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/monitoring_dashboard_2.PNG -------------------------------------------------------------------------------- /img/noaa_dag.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/noaa_dag.PNG -------------------------------------------------------------------------------- /img/owm_dag.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/owm_dag.PNG -------------------------------------------------------------------------------- /img/pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/pipeline.png -------------------------------------------------------------------------------- /img/weather_forecast_dag.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/img/weather_forecast_dag.PNG -------------------------------------------------------------------------------- /proposal.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/proposal.md -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/readme.md -------------------------------------------------------------------------------- /steps_to_recreate_project/images/01_service_account.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/steps_to_recreate_project/images/01_service_account.PNG -------------------------------------------------------------------------------- /steps_to_recreate_project/images/02_service_account_key.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/steps_to_recreate_project/images/02_service_account_key.PNG -------------------------------------------------------------------------------- /steps_to_recreate_project/images/03_vm.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/steps_to_recreate_project/images/03_vm.PNG -------------------------------------------------------------------------------- /steps_to_recreate_project/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mharty3/energy_data_capstone/HEAD/steps_to_recreate_project/readme.md --------------------------------------------------------------------------------