├── README.md ├── data ├── 2014_2018_df_cleaned_keep_bad_apr23.pkl ├── 2014_2019.pkl ├── 2014_2019_df_cleaned_no_remove_bad_values_apr5_2020.pkl ├── 2014_2019_df_cleaned_remove_bad_mar26_2020.pkl ├── 2014_2019_df_cleaned_remove_bad_oct18.pkl ├── 2014_2019_df_cleaned_remove_bad_values_apr26_2020.pkl ├── 2014_2019_df_cleaned_remove_bad_values_apr5_2020.pkl ├── 2014_2019_df_cleaned_remove_bad_values_may16_2020.pkl ├── 2014_2019_df_keep_bad_directions_apr5_2020.pkl ├── 2014_2019_upto_june.pkl ├── AB_NYC_2019_remove_bad_values_jun21_2020.pkl ├── Streetcar Jan 2014.csv ├── bounding_box_df_july15 ├── iris.csv ├── iris_dataframe.pkl ├── iriscaps.csv ├── model_plotapr11b_2020.png ├── model_plotapr11c_2020.png ├── routedirection.csv ├── routedirection_badvalues.csv ├── tensorboard_log │ ├── 20200412-144606 │ │ ├── train │ │ │ ├── events.out.tfevents.1586717173.DESKTOP-TLSI8GI.13296.6750.v2 │ │ │ ├── events.out.tfevents.1586717175.DESKTOP-TLSI8GI.profile-empty │ │ │ └── plugins │ │ │ │ └── profile │ │ │ │ └── 2020-04-12_14-46-15 │ │ │ │ └── local.trace │ │ └── validation │ │ │ └── events.out.tfevents.1586717189.DESKTOP-TLSI8GI.13296.37127.v2 │ ├── 20200412-193604 │ │ ├── train │ │ │ ├── events.out.tfevents.1586734577.DESKTOP-TLSI8GI.13296.879061.v2 │ │ │ ├── events.out.tfevents.1586734579.DESKTOP-TLSI8GI.profile-empty │ │ │ └── plugins │ │ │ │ └── profile │ │ │ │ └── 2020-04-12_19-36-19 │ │ │ │ └── local.trace │ │ └── validation │ │ │ └── events.out.tfevents.1586734594.DESKTOP-TLSI8GI.13296.909438.v2 │ ├── 20200415-215015 │ │ ├── train │ │ │ ├── events.out.tfevents.1587001852.DESKTOP-TLSI8GI.13296.3234273.v2 │ │ │ ├── events.out.tfevents.1587001865.DESKTOP-TLSI8GI.profile-empty │ │ │ └── plugins │ │ │ │ └── profile │ │ │ │ └── 2020-04-15_21-51-05 │ │ │ │ └── local.trace │ │ └── validation │ │ │ └── events.out.tfevents.1587001881.DESKTOP-TLSI8GI.13296.3264650.v2 │ ├── 20200417-210021 │ │ ├── train │ │ │ ├── events.out.tfevents.1587171629.DESKTOP-TLSI8GI.41100.6750.v2 │ │ │ ├── events.out.tfevents.1587171630.DESKTOP-TLSI8GI.profile-empty │ │ │ └── plugins │ │ │ │ └── profile │ │ │ │ └── 2020-04-17_21-00-30 │ │ │ │ └── local.trace │ │ └── validation │ │ │ └── events.out.tfevents.1587171644.DESKTOP-TLSI8GI.41100.37127.v2 │ ├── 20200417-210902 │ │ ├── train │ │ │ ├── events.out.tfevents.1587172153.DESKTOP-TLSI8GI.30044.6750.v2 │ │ │ ├── events.out.tfevents.1587172155.DESKTOP-TLSI8GI.profile-empty │ │ │ └── plugins │ │ │ │ └── profile │ │ │ │ └── 2020-04-17_21-09-15 │ │ │ │ └── local.trace │ │ └── validation │ │ │ └── events.out.tfevents.1587172174.DESKTOP-TLSI8GI.30044.37127.v2 │ ├── 20200417-211818 │ │ ├── train │ │ │ ├── events.out.tfevents.1587172714.DESKTOP-TLSI8GI.30044.879225.v2 │ │ │ ├── events.out.tfevents.1587172718.DESKTOP-TLSI8GI.profile-empty │ │ │ └── plugins │ │ │ │ └── profile │ │ │ │ └── 2020-04-17_21-18-38 │ │ │ │ └── local.trace │ │ └── validation │ │ │ └── events.out.tfevents.1587172736.DESKTOP-TLSI8GI.30044.909602.v2 │ └── 20200417-213403 │ │ ├── train │ │ ├── events.out.tfevents.1587173665.DESKTOP-TLSI8GI.30044.1751700.v2 │ │ ├── events.out.tfevents.1587173670.DESKTOP-TLSI8GI.profile-empty │ │ └── plugins │ │ │ └── profile │ │ │ └── 2020-04-17_21-34-30 │ │ │ └── local.trace │ │ └── validation │ │ └── events.out.tfevents.1587173686.DESKTOP-TLSI8GI.30044.1782077.v2 └── ttc-streetcar-delay-data-2014.xlsx ├── deep_learning_with_structured_data_nov22_2021_final.pdf ├── deploy ├── __init__.py ├── __pycache__ │ ├── actions.cpython-37.pyc │ └── custom_classes.cpython-37.pyc ├── actions.py ├── config.yml ├── credentials.yml ├── custom_classes.py ├── data │ ├── nlu.md │ └── stories.md ├── deploy_config.yml ├── domain.yml ├── endpoints.yml ├── input_sample_data │ ├── prepped_dec27.csv │ └── raw_dec27.xlsx ├── keras_models │ └── scmodeldec27b_5.h5 ├── models │ ├── 20191207-201544.tar.gz │ ├── 20191210-220835.tar.gz │ ├── 20191228-145744.tar.gz │ ├── 20191228-192944.tar.gz │ ├── 20191228-193317.tar.gz │ ├── 20191228-202857.tar.gz │ ├── 20191228-204246.tar.gz │ ├── 20191228-212651.tar.gz │ ├── 20191229-081543.tar.gz │ ├── 20191229-085443.tar.gz │ └── 20191229-142939.tar.gz ├── one_off_scoring.py ├── pipelines │ ├── sc_delay_pipeline_dec27b.pkl │ ├── sc_delay_pipeline_keras_prep_dec27b.pkl │ └── sc_delay_pipelinedec27.pkl └── test_results │ └── python_log_dec28_29.txt ├── deploy_web ├── __pycache__ │ └── custom_classes.cpython-37.pyc ├── custom_classes.py ├── deploy_web_config.yml ├── flask_server.py ├── static │ └── css │ │ ├── main.css │ │ └── main2.css └── templates │ ├── home.html │ └── show-prediction.html ├── models ├── modelapr16_2020.json ├── modelapr17_2020.json ├── modelapr18_2020.json ├── modelapr19_2020.json ├── sc_xgbmodelapr12020_5.txt ├── sc_xgbmodelapr22020_5.txt ├── sc_xgbmodelapr2d2020_5.txt ├── sc_xgbmodelapr52020_5.txt ├── sc_xgbmodelmar302020_5.txt ├── sc_xgbmodelxgb_apr9d_2020_5.txt ├── scmodelapr11_2020_5.h5 ├── scmodelapr11c_2020_5.h5 ├── scmodelapr12020_5.h5 ├── scmodelapr12_2020_5.h5 ├── scmodelapr12b_2020_5.h5 ├── scmodelapr16_2020.h5 ├── scmodelapr17_2020.h5 ├── scmodelapr17_2020_9.h5 ├── scmodelapr18_2020.h5 ├── scmodelapr18_2020_9.h5 ├── scmodelapr19_2020.h5 ├── scmodelapr19_2020_9.h5 ├── scmodelapr22020_5.h5 ├── scmodelapr26_2020_5.h5 ├── scmodelapr2b2020_5.h5 ├── scmodelapr2d2020_5.h5 ├── scmodelapr52020_5.h5 ├── scmodelapr5_2020_5.h5 ├── scmodelapr5_bv_2020_5.h5 ├── scmodelapr6_2020_5.h5 ├── scmodelapr9_2020_5.h5 ├── scmodelapr9b_2020_5.h5 ├── scmodelapr9c_2020_5.h5 ├── scmodeldec1.h5 ├── scmodeldec21_4.h5 ├── scmodeldec21_5.h5 ├── scmodeldec27b_5.h5 ├── scmodeldec5.h5 ├── scmodeldec8_50_epoch_val_loss.h5 ├── scmodelmar262020_5.h5 ├── scmodelmar26b2020_5.h5 ├── scmodelmar272020_5.h5 ├── scmodelmar302020_5.h5 ├── scmodelmar62020_5.h5 ├── scmodelmay16_2020_5.h5 ├── scweightsapr16_2020.h5 ├── scweightsapr17_2020.h5 ├── scweightsapr18_2020.h5 └── scweightsapr19_2020.h5 ├── notebooks ├── .ipynb_checkpoints │ ├── chapter2-checkpoint.ipynb │ ├── chapter5-checkpoint.ipynb │ ├── exercises_streetcar_model_training-checkpoint.ipynb │ ├── streetcar_data-geocode-get-boundaries-checkpoint.ipynb │ ├── streetcar_data_exploration-checkpoint.ipynb │ ├── streetcar_data_preparation-checkpoint.ipynb │ ├── streetcar_data_preparation-geocode-checkpoint.ipynb │ ├── streetcar_model_training-checkpoint.ipynb │ ├── streetcar_model_training_xgb-checkpoint.ipynb │ └── streetcar_time_series-checkpoint.ipynb ├── __pycache__ │ └── custom_classes.cpython-37.pyc ├── airbnb_data_preparation.ipynb ├── airbnb_data_preparation_config.yml ├── airbnb_model_training.ipynb ├── airbnb_model_training_config.yml ├── chapter2.ipynb ├── chapter5.ipynb ├── custom_classes.py ├── exercises_streetcar_model_training.ipynb ├── keras_functional_api_mnist.py ├── keras_sequential_api_mnist.py ├── streetcar_data-geocode-get-boundaries.ipynb ├── streetcar_data_exploration.ipynb ├── streetcar_data_preparation-geocode.ipynb ├── streetcar_data_preparation.ipynb ├── streetcar_data_preparation_config.yml ├── streetcar_model_training.ipynb ├── streetcar_model_training_config.yml ├── streetcar_model_training_xgb.ipynb └── streetcar_time_series.ipynb ├── pipelines ├── sc_delay_pipeline_dec27b.pkl ├── sc_delay_pipeline_keras_prep_dec27b.pkl ├── sc_delay_pipelinedec27.pkl ├── sc_delay_pipleline_keras_prepapr11_2020.pkl ├── sc_delay_pipleline_keras_prepapr11b_2020.pkl ├── sc_delay_pipleline_keras_prepapr11c_2020.pkl ├── sc_delay_pipleline_keras_prepapr12020.pkl ├── sc_delay_pipleline_keras_prepapr12_2020.pkl ├── sc_delay_pipleline_keras_prepapr12b_2020.pkl ├── sc_delay_pipleline_keras_prepapr16_2020.pkl ├── sc_delay_pipleline_keras_prepapr17_2020.pkl ├── sc_delay_pipleline_keras_prepapr18_2020.pkl ├── sc_delay_pipleline_keras_prepapr19_2020.pkl ├── sc_delay_pipleline_keras_prepapr22020.pkl ├── sc_delay_pipleline_keras_prepapr26_2020.pkl ├── sc_delay_pipleline_keras_prepapr2b2020.pkl ├── sc_delay_pipleline_keras_prepapr2d2020.pkl ├── sc_delay_pipleline_keras_prepapr52020.pkl ├── sc_delay_pipleline_keras_prepapr5_2020.pkl ├── sc_delay_pipleline_keras_prepapr5_bv_2020.pkl ├── sc_delay_pipleline_keras_prepapr5_keep_bad_directions_2020.pkl ├── sc_delay_pipleline_keras_prepapr6_2020.pkl ├── sc_delay_pipleline_keras_prepapr9_2020.pkl ├── sc_delay_pipleline_keras_prepapr9b_2020.pkl ├── sc_delay_pipleline_keras_prepapr9c_2020.pkl ├── sc_delay_pipleline_keras_prepmar26b2020.pkl ├── sc_delay_pipleline_keras_prepmar272020.pkl ├── sc_delay_pipleline_keras_prepmar302020.pkl ├── sc_delay_pipleline_keras_prepmay16_2020.pkl ├── sc_delay_pipleline_keras_prepxgb_apr9d_2020.pkl ├── sc_delay_piplelineapr11_2020.pkl ├── sc_delay_piplelineapr11b_2020.pkl ├── sc_delay_piplelineapr11c_2020.pkl ├── sc_delay_piplelineapr12020.pkl ├── sc_delay_piplelineapr12_2020.pkl ├── sc_delay_piplelineapr12b_2020.pkl ├── sc_delay_piplelineapr16_2020.pkl ├── sc_delay_piplelineapr17_2020.pkl ├── sc_delay_piplelineapr18_2020.pkl ├── sc_delay_piplelineapr19_2020.pkl ├── sc_delay_piplelineapr22020.pkl ├── sc_delay_piplelineapr26_2020.pkl ├── sc_delay_piplelineapr2b2020.pkl ├── sc_delay_piplelineapr2d2020.pkl ├── sc_delay_piplelineapr52020.pkl ├── sc_delay_piplelineapr5_2020.pkl ├── sc_delay_piplelineapr5_bv_2020.pkl ├── sc_delay_piplelineapr5_keep_bad_directions_2020.pkl ├── sc_delay_piplelineapr6_2020.pkl ├── sc_delay_piplelineapr9_2020.pkl ├── sc_delay_piplelineapr9b_2020.pkl ├── sc_delay_piplelineapr9c_2020.pkl ├── sc_delay_piplelinemar26b2020.pkl ├── sc_delay_piplelinemar272020.pkl ├── sc_delay_piplelinemar302020.pkl ├── sc_delay_piplelinemay16_2020.pkl └── sc_delay_piplelinexgb_apr9d_2020.pkl └── sql └── streetcarjan2014.sql /README.md: -------------------------------------------------------------------------------- 1 | # deep_learning_for_structured_data 2 | revised repo for Manning book **Deep Learning with Structured Data** https://www.manning.com/books/deep-learning-with-structured-data 3 | 4 | ## Note 5 | This repo is a rework of the original repo for the book at https://github.com/ryanmark1867/manning (which is being kept in place for the convenience of MEAP users who have already started to use it). Improvements in this new repo include: 6 | - rationalized file names 7 | - simplified directory structure 8 | - notebooks tested on Python 3.6 and Python 3.7 in free and for-fee environments 9 | - config files used to remove hard-coded parameters 10 | - code largely refactored to make it easier to follow and simpler to run 11 | - incorporate TensorFlow 2.0 for model training and deployment 12 | 13 | ## Directory structure 14 | - **data** - processed datasets and pickle files for intermediate datasets 15 | - **deploy** - code for deploying the trained model using Rasa and Facebook Messenger, as described in chapter 8 of the book 16 | - **deploy_web** - code for deploying the trained model via a simple web page served by Flask. Description coming in chapter 8. 17 | - **models** - saved trained models 18 | - **notebooks** - Jupyter notebooks **streetcar_data_preparation.ipynb** (data preparation), **streetcar_model_training.ipynb** (deep learning model training) and **streetcar_model_training_xgb.ipynb** (XGBoost model training) along with associated class and config files 19 | - **pipelines** - pickled pipeline files generated by **streetcar_model_training.ipynb**. Together with the trained models these pipeline files are used in the deployment. 20 | - **sql** - SQL used to generate a table that can be used for the simple SQL examples in chapter 2 21 | 22 | ## To exercise the code 23 | **prepare data** - steps to clean up input data to prepare it to train a deep learning model. Output is a pickled dataframe containing the cleaned up dataset. 24 | 1. update **notebooks/streetcar_data_preparation_config.yml** to specify name of input data (pickled_input_dataframe if load_from_scratch = False; if load_from_scratch = True then copy xls files from https://open.toronto.ca/dataset/ttc-streetcar-delay-data/ to **data** directory) and output dataframe 25 | 2. run **notebooks/streetcar_data_preparation.ipynb** 26 | 27 | **train deep learning model** - steps to train a deep learning model on the cleaned up dataset from the previous step. Output is a trained model (h5 file) and two pickle files for the pipeline 28 | 1. update **notebooks/streetcar_model_training_config.yml** to specify input dataframe (pickled_dataframe). Set this to the filename of the dataframe output in the **prepare data** step 29 | 2. run **notebooks/streetcar_model_training.ipynb** in an env. with TensorFlow 2.0 installed 30 | 31 | **train XGBoost model** - steps to train a deep learning model on the cleaned up dataset from the previous step. Output is a trained model (h5 file) and two pickle files for the pipeline 32 | 1. update **notebooks/streetcar_model_training_config.yml** to specify input dataframe (pickled_dataframe). Set this to the filename of the dataframe output in the **prepare data** step 33 | 2. run **streetcar_model_training_xgb.ipynb** - note that the saved XGBoost models are saved in the models directory along with the saved deep learning models 34 | 35 | **web deployment of model** - steps to set up a simple web page for exercising the trained model. Uses the trained model and pipeline files from the previous step. 36 | 1. update **deploy_web/deploy_web_config.yml**: set pipeline1_filename, pipeline2_filename and model_filename to the names of the pipeline files and trained model file generated in the training step. Alternately, you can run the training notebook using the config file as-is to use prepared pipeline and model files already included in the repo. 37 | 2. start the Flask server **deploy_web/flask_server.py** by running: python flask_server.py 38 | 3. open localhost:5000 in a browser, select the details for your trip, and click on **Get prediction** 39 | 40 | ## Background 41 | 42 | - **https://open.toronto.ca/dataset/ttc-streetcar-delay-data/** original dataset 43 | - **https://www.kaggle.com/knowledgegrappler/a-simple-nn-solution-with-keras-0-48611-pl** Kaggle submission that was used as input to creation of the Keras model used in this example 44 | -------------------------------------------------------------------------------- /data/2014_2018_df_cleaned_keep_bad_apr23.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/2014_2018_df_cleaned_keep_bad_apr23.pkl -------------------------------------------------------------------------------- /data/2014_2019.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/2014_2019.pkl -------------------------------------------------------------------------------- /data/2014_2019_df_cleaned_no_remove_bad_values_apr5_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/2014_2019_df_cleaned_no_remove_bad_values_apr5_2020.pkl -------------------------------------------------------------------------------- /data/2014_2019_df_cleaned_remove_bad_mar26_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/2014_2019_df_cleaned_remove_bad_mar26_2020.pkl -------------------------------------------------------------------------------- /data/2014_2019_df_cleaned_remove_bad_oct18.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/2014_2019_df_cleaned_remove_bad_oct18.pkl -------------------------------------------------------------------------------- /data/2014_2019_df_cleaned_remove_bad_values_apr26_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/2014_2019_df_cleaned_remove_bad_values_apr26_2020.pkl -------------------------------------------------------------------------------- /data/2014_2019_df_cleaned_remove_bad_values_apr5_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/2014_2019_df_cleaned_remove_bad_values_apr5_2020.pkl -------------------------------------------------------------------------------- /data/2014_2019_df_cleaned_remove_bad_values_may16_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/2014_2019_df_cleaned_remove_bad_values_may16_2020.pkl -------------------------------------------------------------------------------- /data/2014_2019_df_keep_bad_directions_apr5_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/2014_2019_df_keep_bad_directions_apr5_2020.pkl -------------------------------------------------------------------------------- /data/2014_2019_upto_june.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/2014_2019_upto_june.pkl -------------------------------------------------------------------------------- /data/AB_NYC_2019_remove_bad_values_jun21_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/AB_NYC_2019_remove_bad_values_jun21_2020.pkl -------------------------------------------------------------------------------- /data/bounding_box_df_july15: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/bounding_box_df_july15 -------------------------------------------------------------------------------- /data/iris.csv: -------------------------------------------------------------------------------- 1 | sepal_length,sepal_width,petal_length,petal_width,species 2 | 5.1,3.5,1.4,0.2,setosa 3 | 4.9,3.0,1.4,0.2,setosa 4 | 4.7,3.2,1.3,0.2,setosa 5 | 4.6,3.1,1.5,0.2,setosa 6 | 5.0,3.6,1.4,0.2,setosa 7 | 5.4,3.9,1.7,0.4,setosa 8 | 4.6,3.4,1.4,0.3,setosa 9 | 5.0,3.4,1.5,0.2,setosa 10 | 4.4,2.9,1.4,0.2,setosa 11 | 4.9,3.1,1.5,0.1,setosa 12 | 5.4,3.7,1.5,0.2,setosa 13 | 4.8,3.4,1.6,0.2,setosa 14 | 4.8,3.0,1.4,0.1,setosa 15 | 4.3,3.0,1.1,0.1,setosa 16 | 5.8,4.0,1.2,0.2,setosa 17 | 5.7,4.4,1.5,0.4,setosa 18 | 5.4,3.9,1.3,0.4,setosa 19 | 5.1,3.5,1.4,0.3,setosa 20 | 5.7,3.8,1.7,0.3,setosa 21 | 5.1,3.8,1.5,0.3,setosa 22 | 5.4,3.4,1.7,0.2,setosa 23 | 5.1,3.7,1.5,0.4,setosa 24 | 4.6,3.6,1.0,0.2,setosa 25 | 5.1,3.3,1.7,0.5,setosa 26 | 4.8,3.4,1.9,0.2,setosa 27 | 5.0,3.0,1.6,0.2,setosa 28 | 5.0,3.4,1.6,0.4,setosa 29 | 5.2,3.5,1.5,0.2,setosa 30 | 5.2,3.4,1.4,0.2,setosa 31 | 4.7,3.2,1.6,0.2,setosa 32 | 4.8,3.1,1.6,0.2,setosa 33 | 5.4,3.4,1.5,0.4,setosa 34 | 5.2,4.1,1.5,0.1,setosa 35 | 5.5,4.2,1.4,0.2,setosa 36 | 4.9,3.1,1.5,0.1,setosa 37 | 5.0,3.2,1.2,0.2,setosa 38 | 5.5,3.5,1.3,0.2,setosa 39 | 4.9,3.1,1.5,0.1,setosa 40 | 4.4,3.0,1.3,0.2,setosa 41 | 5.1,3.4,1.5,0.2,setosa 42 | 5.0,3.5,1.3,0.3,setosa 43 | 4.5,2.3,1.3,0.3,setosa 44 | 4.4,3.2,1.3,0.2,setosa 45 | 5.0,3.5,1.6,0.6,setosa 46 | 5.1,3.8,1.9,0.4,setosa 47 | 4.8,3.0,1.4,0.3,setosa 48 | 5.1,3.8,1.6,0.2,setosa 49 | 4.6,3.2,1.4,0.2,setosa 50 | 5.3,3.7,1.5,0.2,setosa 51 | 5.0,3.3,1.4,0.2,setosa 52 | 7.0,3.2,4.7,1.4,versicolor 53 | 6.4,3.2,4.5,1.5,versicolor 54 | 6.9,3.1,4.9,1.5,versicolor 55 | 5.5,2.3,4.0,1.3,versicolor 56 | 6.5,2.8,4.6,1.5,versicolor 57 | 5.7,2.8,4.5,1.3,versicolor 58 | 6.3,3.3,4.7,1.6,versicolor 59 | 4.9,2.4,3.3,1.0,versicolor 60 | 6.6,2.9,4.6,1.3,versicolor 61 | 5.2,2.7,3.9,1.4,versicolor 62 | 5.0,2.0,3.5,1.0,versicolor 63 | 5.9,3.0,4.2,1.5,versicolor 64 | 6.0,2.2,4.0,1.0,versicolor 65 | 6.1,2.9,4.7,1.4,versicolor 66 | 5.6,2.9,3.6,1.3,versicolor 67 | 6.7,3.1,4.4,1.4,versicolor 68 | 5.6,3.0,4.5,1.5,versicolor 69 | 5.8,2.7,4.1,1.0,versicolor 70 | 6.2,2.2,4.5,1.5,versicolor 71 | 5.6,2.5,3.9,1.1,versicolor 72 | 5.9,3.2,4.8,1.8,versicolor 73 | 6.1,2.8,4.0,1.3,versicolor 74 | 6.3,2.5,4.9,1.5,versicolor 75 | 6.1,2.8,4.7,1.2,versicolor 76 | 6.4,2.9,4.3,1.3,versicolor 77 | 6.6,3.0,4.4,1.4,versicolor 78 | 6.8,2.8,4.8,1.4,versicolor 79 | 6.7,3.0,5.0,1.7,versicolor 80 | 6.0,2.9,4.5,1.5,versicolor 81 | 5.7,2.6,3.5,1.0,versicolor 82 | 5.5,2.4,3.8,1.1,versicolor 83 | 5.5,2.4,3.7,1.0,versicolor 84 | 5.8,2.7,3.9,1.2,versicolor 85 | 6.0,2.7,5.1,1.6,versicolor 86 | 5.4,3.0,4.5,1.5,versicolor 87 | 6.0,3.4,4.5,1.6,versicolor 88 | 6.7,3.1,4.7,1.5,versicolor 89 | 6.3,2.3,4.4,1.3,versicolor 90 | 5.6,3.0,4.1,1.3,versicolor 91 | 5.5,2.5,4.0,1.3,versicolor 92 | 5.5,2.6,4.4,1.2,versicolor 93 | 6.1,3.0,4.6,1.4,versicolor 94 | 5.8,2.6,4.0,1.2,versicolor 95 | 5.0,2.3,3.3,1.0,versicolor 96 | 5.6,2.7,4.2,1.3,versicolor 97 | 5.7,3.0,4.2,1.2,versicolor 98 | 5.7,2.9,4.2,1.3,versicolor 99 | 6.2,2.9,4.3,1.3,versicolor 100 | 5.1,2.5,3.0,1.1,versicolor 101 | 5.7,2.8,4.1,1.3,versicolor 102 | 6.3,3.3,6.0,2.5,virginica 103 | 5.8,2.7,5.1,1.9,virginica 104 | 7.1,3.0,5.9,2.1,virginica 105 | 6.3,2.9,5.6,1.8,virginica 106 | 6.5,3.0,5.8,2.2,virginica 107 | 7.6,3.0,6.6,2.1,virginica 108 | 4.9,2.5,4.5,1.7,virginica 109 | 7.3,2.9,6.3,1.8,virginica 110 | 6.7,2.5,5.8,1.8,virginica 111 | 7.2,3.6,6.1,2.5,virginica 112 | 6.5,3.2,5.1,2.0,virginica 113 | 6.4,2.7,5.3,1.9,virginica 114 | 6.8,3.0,5.5,2.1,virginica 115 | 5.7,2.5,5.0,2.0,virginica 116 | 5.8,2.8,5.1,2.4,virginica 117 | 6.4,3.2,5.3,2.3,virginica 118 | 6.5,3.0,5.5,1.8,virginica 119 | 7.7,3.8,6.7,2.2,virginica 120 | 7.7,2.6,6.9,2.3,virginica 121 | 6.0,2.2,5.0,1.5,virginica 122 | 6.9,3.2,5.7,2.3,virginica 123 | 5.6,2.8,4.9,2.0,virginica 124 | 7.7,2.8,6.7,2.0,virginica 125 | 6.3,2.7,4.9,1.8,virginica 126 | 6.7,3.3,5.7,2.1,virginica 127 | 7.2,3.2,6.0,1.8,virginica 128 | 6.2,2.8,4.8,1.8,virginica 129 | 6.1,3.0,4.9,1.8,virginica 130 | 6.4,2.8,5.6,2.1,virginica 131 | 7.2,3.0,5.8,1.6,virginica 132 | 7.4,2.8,6.1,1.9,virginica 133 | 7.9,3.8,6.4,2.0,virginica 134 | 6.4,2.8,5.6,2.2,virginica 135 | 6.3,2.8,5.1,1.5,virginica 136 | 6.1,2.6,5.6,1.4,virginica 137 | 7.7,3.0,6.1,2.3,virginica 138 | 6.3,3.4,5.6,2.4,virginica 139 | 6.4,3.1,5.5,1.8,virginica 140 | 6.0,3.0,4.8,1.8,virginica 141 | 6.9,3.1,5.4,2.1,virginica 142 | 6.7,3.1,5.6,2.4,virginica 143 | 6.9,3.1,5.1,2.3,virginica 144 | 5.8,2.7,5.1,1.9,virginica 145 | 6.8,3.2,5.9,2.3,virginica 146 | 6.7,3.3,5.7,2.5,virginica 147 | 6.7,3.0,5.2,2.3,virginica 148 | 6.3,2.5,5.0,1.9,virginica 149 | 6.5,3.0,5.2,2.0,virginica 150 | 6.2,3.4,5.4,2.3,virginica 151 | 5.9,3.0,5.1,1.8,virginica -------------------------------------------------------------------------------- /data/iris_dataframe.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/iris_dataframe.pkl -------------------------------------------------------------------------------- /data/iriscaps.csv: -------------------------------------------------------------------------------- 1 | sepal_length,sepal_width,petal_length,petal_width,species 2 | 5.1,3.5,1.4,0.2,Setosa 3 | 4.9,3,1.4,0.2,Setosa 4 | 4.7,3.2,1.3,0.2,Setosa 5 | 4.6,3.1,1.5,0.2,Setosa 6 | 5,3.6,1.4,0.2,Setosa 7 | 5.4,3.9,1.7,0.4,Setosa 8 | 4.6,3.4,1.4,0.3,Setosa 9 | 5,3.4,1.5,0.2,Setosa 10 | 4.4,2.9,1.4,0.2,Setosa 11 | 4.9,3.1,1.5,0.1,Setosa 12 | 5.4,3.7,1.5,0.2,Setosa 13 | 4.8,3.4,1.6,0.2,Setosa 14 | 4.8,3,1.4,0.1,Setosa 15 | 4.3,3,1.1,0.1,Setosa 16 | 5.8,4,1.2,0.2,Setosa 17 | 5.7,4.4,1.5,0.4,Setosa 18 | 5.4,3.9,1.3,0.4,Setosa 19 | 5.1,3.5,1.4,0.3,Setosa 20 | 5.7,3.8,1.7,0.3,Setosa 21 | 5.1,3.8,1.5,0.3,Setosa 22 | 5.4,3.4,1.7,0.2,Setosa 23 | 5.1,3.7,1.5,0.4,Setosa 24 | 4.6,3.6,1,0.2,Setosa 25 | 5.1,3.3,1.7,0.5,Setosa 26 | 4.8,3.4,1.9,0.2,Setosa 27 | 5,3,1.6,0.2,Setosa 28 | 5,3.4,1.6,0.4,Setosa 29 | 5.2,3.5,1.5,0.2,Setosa 30 | 5.2,3.4,1.4,0.2,Setosa 31 | 4.7,3.2,1.6,0.2,Setosa 32 | 4.8,3.1,1.6,0.2,Setosa 33 | 5.4,3.4,1.5,0.4,Setosa 34 | 5.2,4.1,1.5,0.1,Setosa 35 | 5.5,4.2,1.4,0.2,Setosa 36 | 4.9,3.1,1.5,0.1,Setosa 37 | 5,3.2,1.2,0.2,Setosa 38 | 5.5,3.5,1.3,0.2,Setosa 39 | 4.9,3.1,1.5,0.1,Setosa 40 | 4.4,3,1.3,0.2,Setosa 41 | 5.1,3.4,1.5,0.2,Setosa 42 | 5,3.5,1.3,0.3,Setosa 43 | 4.5,2.3,1.3,0.3,Setosa 44 | 4.4,3.2,1.3,0.2,Setosa 45 | 5,3.5,1.6,0.6,Setosa 46 | 5.1,3.8,1.9,0.4,Setosa 47 | 4.8,3,1.4,0.3,Setosa 48 | 5.1,3.8,1.6,0.2,Setosa 49 | 4.6,3.2,1.4,0.2,Setosa 50 | 5.3,3.7,1.5,0.2,Setosa 51 | 5,3.3,1.4,0.2,Setosa 52 | 7,3.2,4.7,1.4,Versicolor 53 | 6.4,3.2,4.5,1.5,Versicolor 54 | 6.9,3.1,4.9,1.5,Versicolor 55 | 5.5,2.3,4,1.3,Versicolor 56 | 6.5,2.8,4.6,1.5,Versicolor 57 | 5.7,2.8,4.5,1.3,Versicolor 58 | 6.3,3.3,4.7,1.6,Versicolor 59 | 4.9,2.4,3.3,1,Versicolor 60 | 6.6,2.9,4.6,1.3,Versicolor 61 | 5.2,2.7,3.9,1.4,Versicolor 62 | 5,2,3.5,1,Versicolor 63 | 5.9,3,4.2,1.5,Versicolor 64 | 6,2.2,4,1,Versicolor 65 | 6.1,2.9,4.7,1.4,Versicolor 66 | 5.6,2.9,3.6,1.3,Versicolor 67 | 6.7,3.1,4.4,1.4,Versicolor 68 | 5.6,3,4.5,1.5,Versicolor 69 | 5.8,2.7,4.1,1,Versicolor 70 | 6.2,2.2,4.5,1.5,Versicolor 71 | 5.6,2.5,3.9,1.1,Versicolor 72 | 5.9,3.2,4.8,1.8,Versicolor 73 | 6.1,2.8,4,1.3,Versicolor 74 | 6.3,2.5,4.9,1.5,Versicolor 75 | 6.1,2.8,4.7,1.2,Versicolor 76 | 6.4,2.9,4.3,1.3,Versicolor 77 | 6.6,3,4.4,1.4,Versicolor 78 | 6.8,2.8,4.8,1.4,Versicolor 79 | 6.7,3,5,1.7,Versicolor 80 | 6,2.9,4.5,1.5,Versicolor 81 | 5.7,2.6,3.5,1,Versicolor 82 | 5.5,2.4,3.8,1.1,Versicolor 83 | 5.5,2.4,3.7,1,Versicolor 84 | 5.8,2.7,3.9,1.2,Versicolor 85 | 6,2.7,5.1,1.6,Versicolor 86 | 5.4,3,4.5,1.5,Versicolor 87 | 6,3.4,4.5,1.6,Versicolor 88 | 6.7,3.1,4.7,1.5,Versicolor 89 | 6.3,2.3,4.4,1.3,Versicolor 90 | 5.6,3,4.1,1.3,Versicolor 91 | 5.5,2.5,4,1.3,Versicolor 92 | 5.5,2.6,4.4,1.2,Versicolor 93 | 6.1,3,4.6,1.4,Versicolor 94 | 5.8,2.6,4,1.2,Versicolor 95 | 5,2.3,3.3,1,Versicolor 96 | 5.6,2.7,4.2,1.3,Versicolor 97 | 5.7,3,4.2,1.2,Versicolor 98 | 5.7,2.9,4.2,1.3,Versicolor 99 | 6.2,2.9,4.3,1.3,Versicolor 100 | 5.1,2.5,3,1.1,Versicolor 101 | 5.7,2.8,4.1,1.3,Versicolor 102 | 6.3,3.3,6,2.5,Virginica 103 | 5.8,2.7,5.1,1.9,Virginica 104 | 7.1,3,5.9,2.1,Virginica 105 | 6.3,2.9,5.6,1.8,Virginica 106 | 6.5,3,5.8,2.2,Virginica 107 | 7.6,3,6.6,2.1,Virginica 108 | 4.9,2.5,4.5,1.7,Virginica 109 | 7.3,2.9,6.3,1.8,Virginica 110 | 6.7,2.5,5.8,1.8,Virginica 111 | 7.2,3.6,6.1,2.5,Virginica 112 | 6.5,3.2,5.1,2,Virginica 113 | 6.4,2.7,5.3,1.9,Virginica 114 | 6.8,3,5.5,2.1,Virginica 115 | 5.7,2.5,5,2,Virginica 116 | 5.8,2.8,5.1,2.4,Virginica 117 | 6.4,3.2,5.3,2.3,Virginica 118 | 6.5,3,5.5,1.8,Virginica 119 | 7.7,3.8,6.7,2.2,Virginica 120 | 7.7,2.6,6.9,2.3,Virginica 121 | 6,2.2,5,1.5,Virginica 122 | 6.9,3.2,5.7,2.3,Virginica 123 | 5.6,2.8,4.9,2,Virginica 124 | 7.7,2.8,6.7,2,Virginica 125 | 6.3,2.7,4.9,1.8,Virginica 126 | 6.7,3.3,5.7,2.1,Virginica 127 | 7.2,3.2,6,1.8,Virginica 128 | 6.2,2.8,4.8,1.8,Virginica 129 | 6.1,3,4.9,1.8,Virginica 130 | 6.4,2.8,5.6,2.1,Virginica 131 | 7.2,3,5.8,1.6,Virginica 132 | 7.4,2.8,6.1,1.9,Virginica 133 | 7.9,3.8,6.4,2,Virginica 134 | 6.4,2.8,5.6,2.2,Virginica 135 | 6.3,2.8,5.1,1.5,Virginica 136 | 6.1,2.6,5.6,1.4,Virginica 137 | 7.7,3,6.1,2.3,Virginica 138 | 6.3,3.4,5.6,2.4,Virginica 139 | 6.4,3.1,5.5,1.8,Virginica 140 | 6,3,4.8,1.8,Virginica 141 | 6.9,3.1,5.4,2.1,Virginica 142 | 6.7,3.1,5.6,2.4,Virginica 143 | 6.9,3.1,5.1,2.3,Virginica 144 | 5.8,2.7,5.1,1.9,Virginica 145 | 6.8,3.2,5.9,2.3,Virginica 146 | 6.7,3.3,5.7,2.5,Virginica 147 | 6.7,3,5.2,2.3,Virginica 148 | 6.3,2.5,5,1.9,Virginica 149 | 6.5,3,5.2,2,Virginica 150 | 6.2,3.4,5.4,2.3,Virginica 151 | 5.9,3,5.1,1.8,Virginica 152 | -------------------------------------------------------------------------------- /data/model_plotapr11b_2020.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/model_plotapr11b_2020.png -------------------------------------------------------------------------------- /data/model_plotapr11c_2020.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/model_plotapr11c_2020.png -------------------------------------------------------------------------------- /data/routedirection.csv: -------------------------------------------------------------------------------- 1 | Route,Direction 2 | 301,e 3 | 301,w 4 | 301,b 5 | 304,e 6 | 304,e 7 | 304,n 8 | 304,s 9 | 304,b 10 | 306,e 11 | 306,w 12 | 306,b 13 | 310,n 14 | 310,s 15 | 310,b 16 | 501,e 17 | 501,w 18 | 501,b 19 | 502,e 20 | 502,w 21 | 502,b 22 | 503,e 23 | 503,w 24 | 503,b 25 | 504,e 26 | 504,w 27 | 504,n 28 | 504,s 29 | 504,b 30 | 505,e 31 | 505,w 32 | 505,n 33 | 505,s 34 | 505,b 35 | 506,e 36 | 506,w 37 | 506,n 38 | 506,s 39 | 506,b 40 | 509,e 41 | 509,w 42 | 509,n 43 | 509,s 44 | 509,b 45 | 510,e 46 | 510,w 47 | 510,n 48 | 510,s 49 | 510,b 50 | 511,e 51 | 511,w 52 | 511,n 53 | 511,s 54 | 511,b 55 | 512,e 56 | 512,w 57 | 512,b 58 | -------------------------------------------------------------------------------- /data/routedirection_badvalues.csv: -------------------------------------------------------------------------------- 1 | Route,Direction 2 | 301,e 3 | 301,w 4 | 301,b 5 | 304,e 6 | 304,e 7 | 304,n 8 | 304,s 9 | 304,b 10 | 306,e 11 | 306,w 12 | 306,b 13 | 310,n 14 | 310,s 15 | 310,b 16 | 501,e 17 | 501,w 18 | 501,b 19 | 502,e 20 | 502,w 21 | 502,b 22 | 503,e 23 | 503,w 24 | 503,b 25 | 504,e 26 | 504,w 27 | 504,n 28 | 504,s 29 | 504,b 30 | 505,e 31 | 505,w 32 | 505,n 33 | 505,s 34 | 505,b 35 | 506,e 36 | 506,w 37 | 506,n 38 | 506,s 39 | 506,b 40 | 509,e 41 | 509,w 42 | 509,n 43 | 509,s 44 | 509,b 45 | 510,e 46 | 510,w 47 | 510,n 48 | 510,s 49 | 510,b 50 | 511,e 51 | 511,w 52 | 511,n 53 | 511,s 54 | 511,b 55 | 512,e 56 | 512,w 57 | 512,b 58 | 301,bad direction 59 | 304,bad direction 60 | 306,bad direction 61 | 310,bad direction 62 | 501,bad direction 63 | 502,bad direction 64 | 503,bad direction 65 | 504,bad direction 66 | 505,bad direction 67 | 506,bad direction 68 | 509,bad direction 69 | 510,bad direction 70 | 511,bad direction 71 | 512,bad direction 72 | bad route,n 73 | bad route,s 74 | bad route,e 75 | bad route,w 76 | bad route,b 77 | -------------------------------------------------------------------------------- /data/tensorboard_log/20200412-144606/train/events.out.tfevents.1586717173.DESKTOP-TLSI8GI.13296.6750.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200412-144606/train/events.out.tfevents.1586717173.DESKTOP-TLSI8GI.13296.6750.v2 -------------------------------------------------------------------------------- /data/tensorboard_log/20200412-144606/train/events.out.tfevents.1586717175.DESKTOP-TLSI8GI.profile-empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200412-144606/train/events.out.tfevents.1586717175.DESKTOP-TLSI8GI.profile-empty -------------------------------------------------------------------------------- /data/tensorboard_log/20200412-144606/train/plugins/profile/2020-04-12_14-46-15/local.trace: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200412-144606/train/plugins/profile/2020-04-12_14-46-15/local.trace -------------------------------------------------------------------------------- /data/tensorboard_log/20200412-144606/validation/events.out.tfevents.1586717189.DESKTOP-TLSI8GI.13296.37127.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200412-144606/validation/events.out.tfevents.1586717189.DESKTOP-TLSI8GI.13296.37127.v2 -------------------------------------------------------------------------------- /data/tensorboard_log/20200412-193604/train/events.out.tfevents.1586734577.DESKTOP-TLSI8GI.13296.879061.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200412-193604/train/events.out.tfevents.1586734577.DESKTOP-TLSI8GI.13296.879061.v2 -------------------------------------------------------------------------------- /data/tensorboard_log/20200412-193604/train/events.out.tfevents.1586734579.DESKTOP-TLSI8GI.profile-empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200412-193604/train/events.out.tfevents.1586734579.DESKTOP-TLSI8GI.profile-empty -------------------------------------------------------------------------------- /data/tensorboard_log/20200412-193604/train/plugins/profile/2020-04-12_19-36-19/local.trace: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200412-193604/train/plugins/profile/2020-04-12_19-36-19/local.trace -------------------------------------------------------------------------------- /data/tensorboard_log/20200412-193604/validation/events.out.tfevents.1586734594.DESKTOP-TLSI8GI.13296.909438.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200412-193604/validation/events.out.tfevents.1586734594.DESKTOP-TLSI8GI.13296.909438.v2 -------------------------------------------------------------------------------- /data/tensorboard_log/20200415-215015/train/events.out.tfevents.1587001852.DESKTOP-TLSI8GI.13296.3234273.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200415-215015/train/events.out.tfevents.1587001852.DESKTOP-TLSI8GI.13296.3234273.v2 -------------------------------------------------------------------------------- /data/tensorboard_log/20200415-215015/train/events.out.tfevents.1587001865.DESKTOP-TLSI8GI.profile-empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200415-215015/train/events.out.tfevents.1587001865.DESKTOP-TLSI8GI.profile-empty -------------------------------------------------------------------------------- /data/tensorboard_log/20200415-215015/train/plugins/profile/2020-04-15_21-51-05/local.trace: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200415-215015/train/plugins/profile/2020-04-15_21-51-05/local.trace -------------------------------------------------------------------------------- /data/tensorboard_log/20200415-215015/validation/events.out.tfevents.1587001881.DESKTOP-TLSI8GI.13296.3264650.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200415-215015/validation/events.out.tfevents.1587001881.DESKTOP-TLSI8GI.13296.3264650.v2 -------------------------------------------------------------------------------- /data/tensorboard_log/20200417-210021/train/events.out.tfevents.1587171629.DESKTOP-TLSI8GI.41100.6750.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200417-210021/train/events.out.tfevents.1587171629.DESKTOP-TLSI8GI.41100.6750.v2 -------------------------------------------------------------------------------- /data/tensorboard_log/20200417-210021/train/events.out.tfevents.1587171630.DESKTOP-TLSI8GI.profile-empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200417-210021/train/events.out.tfevents.1587171630.DESKTOP-TLSI8GI.profile-empty -------------------------------------------------------------------------------- /data/tensorboard_log/20200417-210021/train/plugins/profile/2020-04-17_21-00-30/local.trace: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200417-210021/train/plugins/profile/2020-04-17_21-00-30/local.trace -------------------------------------------------------------------------------- /data/tensorboard_log/20200417-210021/validation/events.out.tfevents.1587171644.DESKTOP-TLSI8GI.41100.37127.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200417-210021/validation/events.out.tfevents.1587171644.DESKTOP-TLSI8GI.41100.37127.v2 -------------------------------------------------------------------------------- /data/tensorboard_log/20200417-210902/train/events.out.tfevents.1587172153.DESKTOP-TLSI8GI.30044.6750.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200417-210902/train/events.out.tfevents.1587172153.DESKTOP-TLSI8GI.30044.6750.v2 -------------------------------------------------------------------------------- /data/tensorboard_log/20200417-210902/train/events.out.tfevents.1587172155.DESKTOP-TLSI8GI.profile-empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200417-210902/train/events.out.tfevents.1587172155.DESKTOP-TLSI8GI.profile-empty -------------------------------------------------------------------------------- /data/tensorboard_log/20200417-210902/train/plugins/profile/2020-04-17_21-09-15/local.trace: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200417-210902/train/plugins/profile/2020-04-17_21-09-15/local.trace -------------------------------------------------------------------------------- /data/tensorboard_log/20200417-210902/validation/events.out.tfevents.1587172174.DESKTOP-TLSI8GI.30044.37127.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200417-210902/validation/events.out.tfevents.1587172174.DESKTOP-TLSI8GI.30044.37127.v2 -------------------------------------------------------------------------------- /data/tensorboard_log/20200417-211818/train/events.out.tfevents.1587172714.DESKTOP-TLSI8GI.30044.879225.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200417-211818/train/events.out.tfevents.1587172714.DESKTOP-TLSI8GI.30044.879225.v2 -------------------------------------------------------------------------------- /data/tensorboard_log/20200417-211818/train/events.out.tfevents.1587172718.DESKTOP-TLSI8GI.profile-empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200417-211818/train/events.out.tfevents.1587172718.DESKTOP-TLSI8GI.profile-empty -------------------------------------------------------------------------------- /data/tensorboard_log/20200417-211818/train/plugins/profile/2020-04-17_21-18-38/local.trace: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200417-211818/train/plugins/profile/2020-04-17_21-18-38/local.trace -------------------------------------------------------------------------------- /data/tensorboard_log/20200417-211818/validation/events.out.tfevents.1587172736.DESKTOP-TLSI8GI.30044.909602.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200417-211818/validation/events.out.tfevents.1587172736.DESKTOP-TLSI8GI.30044.909602.v2 -------------------------------------------------------------------------------- /data/tensorboard_log/20200417-213403/train/events.out.tfevents.1587173665.DESKTOP-TLSI8GI.30044.1751700.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200417-213403/train/events.out.tfevents.1587173665.DESKTOP-TLSI8GI.30044.1751700.v2 -------------------------------------------------------------------------------- /data/tensorboard_log/20200417-213403/train/events.out.tfevents.1587173670.DESKTOP-TLSI8GI.profile-empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200417-213403/train/events.out.tfevents.1587173670.DESKTOP-TLSI8GI.profile-empty -------------------------------------------------------------------------------- /data/tensorboard_log/20200417-213403/train/plugins/profile/2020-04-17_21-34-30/local.trace: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200417-213403/train/plugins/profile/2020-04-17_21-34-30/local.trace -------------------------------------------------------------------------------- /data/tensorboard_log/20200417-213403/validation/events.out.tfevents.1587173686.DESKTOP-TLSI8GI.30044.1782077.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/tensorboard_log/20200417-213403/validation/events.out.tfevents.1587173686.DESKTOP-TLSI8GI.30044.1782077.v2 -------------------------------------------------------------------------------- /data/ttc-streetcar-delay-data-2014.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/data/ttc-streetcar-delay-data-2014.xlsx -------------------------------------------------------------------------------- /deep_learning_with_structured_data_nov22_2021_final.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deep_learning_with_structured_data_nov22_2021_final.pdf -------------------------------------------------------------------------------- /deploy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/__init__.py -------------------------------------------------------------------------------- /deploy/__pycache__/actions.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/__pycache__/actions.cpython-37.pyc -------------------------------------------------------------------------------- /deploy/__pycache__/custom_classes.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/__pycache__/custom_classes.cpython-37.pyc -------------------------------------------------------------------------------- /deploy/actions.py: -------------------------------------------------------------------------------- 1 | # This files contains the custom actions related to the Rasa model for the streetcar delay prediction project 2 | 3 | # common imports 4 | from typing import Any, Text, Dict, List 5 | # 6 | from rasa_sdk import Action, Tracker 7 | from rasa_sdk.executor import CollectingDispatcher 8 | from rasa_sdk.events import SlotSet 9 | 10 | 11 | 12 | import zipfile 13 | import pandas as pd 14 | import numpy as np 15 | import time 16 | # import datetime, timedelta 17 | import datetime 18 | from datetime import datetime, timedelta 19 | from datetime import date 20 | from dateutil import relativedelta 21 | from io import StringIO 22 | import pandas as pd 23 | import pickle 24 | from sklearn.base import BaseEstimator 25 | from sklearn.base import TransformerMixin 26 | # DSX code to import uploaded documents 27 | from io import StringIO 28 | import requests 29 | import json 30 | from sklearn.preprocessing import LabelEncoder, MinMaxScaler, StandardScaler 31 | from sklearn.model_selection import train_test_split 32 | import matplotlib.pyplot as plt 33 | import os 34 | import math 35 | from subprocess import check_output 36 | #model libraries 37 | from tensorflow.keras.layers import Input, Dropout, Dense, BatchNormalization, Activation, concatenate, GRU, Embedding, Flatten, BatchNormalization 38 | from tensorflow.keras.models import Model 39 | from tensorflow.keras.callbacks import ModelCheckpoint, Callback, EarlyStopping 40 | from tensorflow.keras import regularizers 41 | from tensorflow.keras.layers import BatchNormalization 42 | from tensorflow.keras.regularizers import l2 43 | from tensorflow.keras.optimizers import Adam 44 | from tensorflow.keras.optimizers import SGD 45 | from tensorflow.keras import backend as K 46 | from tensorflow.keras.utils import plot_model 47 | from tensorflow.keras.preprocessing.text import Tokenizer 48 | from tensorflow.keras.preprocessing.sequence import pad_sequences 49 | from tensorflow.keras.models import load_model 50 | from sklearn import metrics 51 | from sklearn.datasets.samples_generator import make_blobs 52 | from sklearn.model_selection import train_test_split 53 | from sklearn.metrics import accuracy_score 54 | from pickle import load 55 | from custom_classes import encode_categorical 56 | from custom_classes import prep_for_keras_input 57 | from custom_classes import fill_empty 58 | from custom_classes import encode_text 59 | from datetime import date 60 | from datetime import datetime 61 | import logging 62 | import yaml 63 | 64 | # load config gile 65 | current_path = os.getcwd() 66 | print("current directory is: "+current_path) 67 | 68 | path_to_yaml = os.path.join(current_path, 'deploy_config.yml') 69 | print("path_to_yaml "+path_to_yaml) 70 | try: 71 | with open (path_to_yaml, 'r') as c_file: 72 | config = yaml.safe_load(c_file) 73 | except Exception as e: 74 | print('Error reading the config file') 75 | 76 | 77 | # paths for model and pipeline files 78 | pipeline1_filename = config['file_names']['pipeline1_filename'] 79 | pipeline2_filename = config['file_names']['pipeline2_filename'] 80 | model_filename = config['file_names']['model_filename'] 81 | 82 | # other parms 83 | debug_on = config['general']['debug_on'] 84 | logging_level = config['general']['logging_level'] 85 | BATCH_SIZE = config['general']['BATCH_SIZE'] 86 | 87 | # set logging level 88 | logging_level_set = logging.WARNING 89 | if logging_level == 'WARNING': 90 | logging_level_set = logging.WARNING 91 | if logging_level == 'ERROR': 92 | logging_level_set = logging.ERROR 93 | if logging_level == 'DEBUG': 94 | logging_level_set = logging.DEBUG 95 | if logging_level == 'INFO': 96 | logging_level_set = logging.INFO 97 | logging.getLogger().setLevel(logging_level_set) 98 | logging.warning("logging check - beginning of logging") 99 | 100 | def get_path(subpath): 101 | rawpath = os.getcwd() 102 | # data is in a directory called "data" that is a sibling to the directory containing the notebook 103 | path = os.path.abspath(os.path.join(rawpath, '..', subpath)) 104 | return(path) 105 | 106 | # get complete paths for pipelines and Keras models 107 | pipeline_path = get_path('pipelines') 108 | 109 | pipeline1_path = os.path.join(pipeline_path,pipeline1_filename) 110 | pipeline2_path = os.path.join(pipeline_path,pipeline2_filename) 111 | model_path = os.path.join(get_path('models'),model_filename) 112 | 113 | 114 | # load the Keras model 115 | loaded_model = load_model(model_path) 116 | loaded_model.summary() 117 | # moved pipeline definitions to custom action class 118 | #pipeline1 = load(open(pipeline1_path, 'rb')) 119 | #pipeline2 = load(open(pipeline2_path, 'rb')) 120 | 121 | 122 | # brute force a scoring sample, bagged from test set 123 | score_sample = {} 124 | score_sample['hour'] = np.array([18]) 125 | score_sample['Route'] = np.array([0]) 126 | score_sample['daym'] = np.array([21]) 127 | score_sample['month'] = np.array([0]) 128 | score_sample['year'] = np.array([5]) 129 | score_sample['Direction'] = np.array([1]) 130 | score_sample['day'] = np.array([1]) 131 | 132 | # 'day': array([1, 2, 2, ..., 0, 1, 1])} 133 | 134 | 135 | # dictionary of default values 136 | score_default = {} 137 | score_default['hour'] = 12 138 | score_default['Route'] = '501' 139 | score_default['daym'] = 1 140 | score_default['month'] = 1 141 | score_default['year'] = '2019' 142 | score_default['Direction'] = 'e' 143 | score_default['day'] = 2 144 | 145 | score_cols = ['Route','Direction','hour','year','month','daym','day'] 146 | logging.warning("score_cols after define is "+str(score_cols)) 147 | 148 | 149 | 150 | preds = loaded_model.predict(score_sample, batch_size=BATCH_SIZE) 151 | logging.warning("pred is "+str(preds)) 152 | 153 | logging.warning("preds[0] is "+str(preds[0])) 154 | logging.warning("preds[0][0] is "+str(preds[0][0])) 155 | 156 | # example using pipeline on prepped data 157 | # routedirection_frame = pd.read_csv(path+"routedirection.csv") 158 | 159 | 160 | class ActionPredictDelay(Action): 161 | 162 | def name(self) -> Text: 163 | return "action_predict_delay" 164 | 165 | def run(self, dispatcher: CollectingDispatcher, 166 | tracker: Tracker, 167 | domain: Dict[Text, Any]) -> List[Dict[Text, Any]]: 168 | dispatcher.utter_message("in action_predict_delay") 169 | if preds[0][0] >= 0.5: 170 | predict_string = "yes" 171 | else: 172 | predict_string = "no" 173 | dispatcher.utter_message("Delay prediction is:"+predict_string) 174 | 175 | return [] 176 | 177 | 178 | class ActionPredictDelayComplete(Action): 179 | ''' predict delay when the user has provided sufficient content to make a prediction''' 180 | def name(self) -> Text: 181 | return "action_predict_delay_complete" 182 | 183 | def run(self, dispatcher: CollectingDispatcher, 184 | tracker: Tracker, 185 | domain: Dict[Text, Any]) -> List[Dict[Text, Any]]: 186 | logging.warning("in action_predict_delay_complete") 187 | # TODO brute force way to ensure pipeline doesn't get clogged 188 | pipeline1 = load(open(pipeline1_path, 'rb')) 189 | pipeline2 = load(open(pipeline2_path, 'rb')) 190 | # init dictionary to hold scoring values 191 | score_current = {} 192 | now = datetime.now() 193 | score_default['daym'] = now.day 194 | score_default['month'] = now.month 195 | score_default['year'] = now.year 196 | # TODO - remove once the model is trained with post 2019 data 197 | if int(score_default['year']) > 2019: 198 | score_default['year'] = 2019 199 | score_default['hour'] = now.hour 200 | score_default['day'] = now.weekday() 201 | logging.warning("score_default is "+str(score_default)) 202 | logging.warning("score_cols is "+str(score_cols)) 203 | score_df = pd.DataFrame(columns=score_cols) 204 | 205 | logging.warning("score_df before load is "+str(score_df)) 206 | # load the score_df 207 | for col in score_cols: 208 | if tracker.get_slot(col) != None: 209 | # if the slot has been set in Rasa, use that value 210 | if tracker.get_slot(col) == "today": 211 | logging.warning("GOT a TOday") 212 | score_df.at[0,col] = score_default[col] 213 | score_df.at[0,col] = tracker.get_slot(col) 214 | else: 215 | # if the slot has not been set in Rasa, use the default value 216 | score_df.at[0,col] = score_default[col] 217 | logging.warning("score_df after load is "+str(score_df)) 218 | if debug_on: 219 | dispatcher.utter_message("input is: "+str(score_df)) 220 | prepped_xform1 = pipeline1.transform(score_df) 221 | prepped_xform2 = pipeline2.transform(prepped_xform1) 222 | print("prepped_xform2 is ",prepped_xform2) 223 | pred = loaded_model.predict(prepped_xform2, batch_size=BATCH_SIZE) 224 | logging.warning("pred is "+str(pred)) 225 | if pred[0][0] >= 0.5: 226 | predict_string = "yes, delay predicted" 227 | else: 228 | predict_string = "no delay predicted" 229 | 230 | dispatcher.utter_message("Delay prediction is: "+predict_string) 231 | 232 | return [SlotSet("hour",None),SlotSet("day",None),SlotSet("daym",None),SlotSet("month",None)] 233 | -------------------------------------------------------------------------------- /deploy/config.yml: -------------------------------------------------------------------------------- 1 | # Configuration for Rasa NLU. 2 | # https://rasa.com/docs/rasa/nlu/components/ 3 | language: en 4 | pipeline: supervised_embeddings 5 | 6 | # Configuration for Rasa Core. 7 | # https://rasa.com/docs/rasa/core/policies/ 8 | policies: 9 | - name: MemoizationPolicy 10 | - name: KerasPolicy 11 | - name: MappingPolicy 12 | -------------------------------------------------------------------------------- /deploy/credentials.yml: -------------------------------------------------------------------------------- 1 | # This file contains the credentials for the voice & chat platforms 2 | # which your bot is using. 3 | # https://rasa.com/docs/rasa/user-guide/messaging-and-voice-channels/ 4 | 5 | rest: 6 | # # you don't need to provide anything here - this channel doesn't 7 | # # require any credentials 8 | 9 | 10 | facebook: 11 | verify: "streetcar_delay" 12 | secret: "adc23d134213a9f30f611d97ad80a8e5" 13 | page-access-token: "EAAGJJZCYFfXkBAM6cHS63PYmxLK0fZASitS9qRxdF5VOVc8TSbe9z3ZBNt5aHnIKFVblFJWdoJNbfLAUUZBKxZBh6ZCWwmGoEpIH6WwDZAWZBu5y44ZAPU3wdT5xQgJI3ZCoUbvgKdIHVRJgSOecKx3aZBTyaeGzgZCHay2UaiaZBSqoomAZDZD" 14 | 15 | #slack: 16 | # slack_token: "" 17 | # slack_channel: "" 18 | 19 | #socketio: 20 | # user_message_evt: 21 | # bot_message_evt: 22 | # session_persistence: 23 | 24 | rasa: 25 | url: "http://localhost:5002/api" 26 | -------------------------------------------------------------------------------- /deploy/custom_classes.py: -------------------------------------------------------------------------------- 1 | from sklearn.model_selection import train_test_split 2 | from sklearn.metrics import accuracy_score 3 | from sklearn.metrics import average_precision_score 4 | from sklearn.pipeline import Pipeline 5 | from sklearn.base import TransformerMixin 6 | from sklearn.base import BaseEstimator 7 | from sklearn.preprocessing import LabelEncoder, MinMaxScaler, StandardScaler 8 | import numpy as np 9 | 10 | 11 | 12 | 13 | 14 | class encode_categorical(BaseEstimator, TransformerMixin): 15 | '''encode categorical columns''' 16 | def __init__(self): 17 | self.le = {} 18 | self.max_dict = {} 19 | return None 20 | 21 | def set_params(self, **kwargs): 22 | self.col_list = kwargs.get('col_list', None) 23 | return self 24 | 25 | 26 | 27 | def fit(self, X, y=None, **fit_params): 28 | for col in self.col_list: 29 | print("col is ",col) 30 | self.le[col] = LabelEncoder() 31 | self.le[col].fit(X[col].tolist()) 32 | return self 33 | 34 | 35 | def transform(self, X, y=None, **tranform_params): 36 | for col in self.col_list: 37 | print("transform col is ",col) 38 | X[col] = self.le[col].transform(X[col]) 39 | print("after transform col is ",col) 40 | self.max_dict[col] = X[col].max() +1 41 | # print(X.loc[[0]]) 42 | return X 43 | 44 | class prep_for_keras_input(BaseEstimator, TransformerMixin): 45 | '''prep columns for feeding Keras model''' 46 | def __init__(self): 47 | self.dictlist = [] 48 | return None 49 | 50 | def set_params(self, **kwargs): 51 | self.collist = kwargs.get('collist', None) 52 | self.continuouscols = kwargs.get('continuouscols', None) 53 | self.textcols = kwargs.get('textcols', None) 54 | return self 55 | 56 | 57 | 58 | def fit(self, X, y=None, **fit_params): 59 | return self 60 | 61 | 62 | def transform(self, X, y=None, **tranform_params): 63 | for col in self.collist: 64 | print("cat col is",col) 65 | self.dictlist.append(np.array(X[col])) 66 | for col in self.textcols: 67 | print("text col is",col) 68 | self.dictlist.append(pad_sequences(X[col], maxlen=max_dict[col])) 69 | for col in self.continuouscols: 70 | print("cont col is",col) 71 | self.dictlist.append(np.array(X[col])) 72 | # print(X.loc[[0]]) 73 | return self.dictlist 74 | 75 | class fill_empty(BaseEstimator, TransformerMixin): 76 | '''fill empty values with placeholders''' 77 | 78 | 79 | def set_params(self, **kwargs): 80 | self.collist = kwargs.get('collist', None) 81 | self.continuouscols = kwargs.get('continuouscols', None) 82 | self.textcols = kwargs.get('textcols', None) 83 | return self 84 | 85 | 86 | def transform(self, X, **tranform_params): 87 | print("fill empty xform") 88 | for col in self.collist: 89 | X[col].fillna(value="missing", inplace=True) 90 | for col in self.continuouscols: 91 | X[col].fillna(value=0.0,inplace=True) 92 | for col in self.textcols: 93 | X[col].fillna(value="missing", inplace=True) 94 | return X 95 | 96 | def fit(self, X, y=None, **fit_params): 97 | return self 98 | 99 | class encode_text(BaseEstimator, TransformerMixin): 100 | '''encode text columns''' 101 | def __init__(self): 102 | self.tok = {} 103 | return None 104 | 105 | def set_params(self, **kwargs): 106 | self.col_list = kwargs.get('col_list', None) 107 | return self 108 | 109 | 110 | 111 | def fit(self, X, y=None, **fit_params): 112 | for col in self.col_list: 113 | print("col is ",col) 114 | self.tok[col] = Tokenizer(num_words=maxwords,lower=True) 115 | self.tok[col].fit_on_texts(X[col]) 116 | return self 117 | 118 | 119 | def transform(self, X, y=None, **tranform_params): 120 | for col in self.col_list: 121 | print("transform col is ",col) 122 | X[col] = self.tok[col].texts_to_sequences(X[col]) 123 | print("after transform col is ",col) 124 | self.max_dict[col] = np.max(X[(X[col].map(len) != 0)][col].map(max)) 125 | if self.max_dict[cols] > textmax: 126 | textmax = self.max_dict[cols] 127 | # print(X.loc[[0]]) 128 | return X 129 | -------------------------------------------------------------------------------- /deploy/data/nlu.md: -------------------------------------------------------------------------------- 1 | ## intent:predict_delay_intro 2 | - what is the streetcar delay 3 | - predict delay 4 | - will my streetcar be delayed 5 | 6 | ## intent:predict_delay_complete 7 | - will route [501](Route) [eastbound](Direction:e) be delayed? 8 | - will [St Clair](Route:512) [eastbound](Direction:e) be delayed? 9 | - will [St Clair](Route:512) [westbound](Direction:w) be delayed? 10 | - will [Downtowner](Route:502) [eastbound](Direction:e) be delayed? 11 | - will [Downtowner](Route:502) [westbound](Direction:w) be delayed? 12 | - is [Spadina](Route:510) [south](Direction:s) going to be delayed? 13 | - will there be a delay on [Spadina](Route:510) [northbound](Direction:n)? 14 | - will there be a delay on [511](route) [southbound](Direction:s)? 15 | - will there be a delay on [502](route) [eastbound](Direction:e)? 16 | - will there be a delay on [502](route) [west](Direction:w)? 17 | - will [Bathurst](Route:511) streetcar be delayed? 18 | - will [Bathurst](Route:511) streetcar [north](Direction:n) be delayed? 19 | - will [Bathurst](Route:511) [north](Direction:n) streetcar be delayed? 20 | - will [Queen](Route:501) [east](Direction:e) be delayed? 21 | - will [Queen](Route:501) [west](Direction:w) be delayed? 22 | - will [Carlton](Route:506) [east](Direction:e) be delayed? 23 | - will [Carlton](Route:506) [west](Direction:w) be delayed? 24 | - will [Dundas](Route:505) [east](Direction:e) be delayed? 25 | - will [Dundas](Route:505) [west](Direction:w) be delayed? 26 | - will [King](Route:504) [east](Direction:e) be delayed? 27 | - will [King](Route:504) [west](Direction:w) be delayed? 28 | - will [Kingston](Route:503) [east](Direction:e) be delayed? 29 | - will [Kingston](Route:503) [west](Direction:w) be delayed? 30 | - will the [eastbound](Direction:e) [College](Route:506) be delayed? 31 | - will [westbound](Direction:w) [506](route) be delayed? 32 | - will [Dundas](Route:506) be delayed at [4:00 pm](hour:16) [today](day)? 33 | - will [508](route) [eastbound](Direction:e) be delayed at [noon](hour:12) [tomorrow](day)? 34 | - will [Harbourfront](Route:509) [north](Direction:n) be delayed? 35 | - will [Harbourfront](Route:509) [south](Direction:s) be delayed? 36 | - will [509](Route) [north](Direction:n) be delayed? 37 | - will [509](Route) [south](Direction:s) be delayed? 38 | - will [Lakeshore](Route:508) [west](Direction:w) be delayed in [3](delta) [hours](scale)? 39 | - will [Lakeshore](Route:508) [east](Direction:e) be delayed in [45](delta) [minutes](scale)? 40 | - will [Lakeshore](Route:508) [eastbound](Direction:e) be delayed this time [tomorrow](day)? 41 | 42 | ## intent:greet 43 | - hey 44 | - hello 45 | - hi 46 | - good morning 47 | - good evening 48 | - hey there 49 | 50 | ## intent:goodbye 51 | - bye 52 | - goodbye 53 | - see you around 54 | - see you later 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /deploy/data/stories.md: -------------------------------------------------------------------------------- 1 | ## happy path 2 | * greet 3 | - utter_greet 4 | * predict_delay 5 | - action_predict_delay 6 | 7 | ## New Story 8 | * predict_delay_complete{"Route":"501","Direction":"e"} 9 | - slot{"Route":"501"} 10 | - slot{"Direction":"e"} 11 | - action_predict_delay_complete 12 | 13 | ## New Story 14 | * predict_delay_complete{"Route":"506","hour":"16","day":"today"} 15 | - slot{"Route":"506"} 16 | - slot{"hour":"16"} 17 | - slot{"day":"today"} 18 | - action_predict_delay_complete 19 | 20 | 21 | ## New Story 22 | * predict_delay_complete{"Route":"508","delta":"3","scale":"hours"} 23 | - slot{"Route":"508"} 24 | - slot{"delta":"3"} 25 | - slot{"scale":"hours"} 26 | - action_predict_delay_complete 27 | 28 | ## say goodbye 29 | * goodbye 30 | - utter_goodbye 31 | -------------------------------------------------------------------------------- /deploy/deploy_config.yml: -------------------------------------------------------------------------------- 1 | general: 2 | debug_on: False 3 | logging_level: "WARNING" # switch to control logging - WARNING for full logging; ERROR to minimize logging 4 | BATCH_SIZE: 1000 5 | file_names: 6 | pipeline1_filename: sc_delay_piplelineapr26_2020.pkl 7 | pipeline2_filename: sc_delay_pipleline_keras_prepapr26_2020.pkl 8 | model_filename: scmodelapr26_2020_5.h5 9 | -------------------------------------------------------------------------------- /deploy/domain.yml: -------------------------------------------------------------------------------- 1 | intents: 2 | - greet 3 | - goodbye 4 | - predict_delay 5 | - predict_delay_complete 6 | 7 | entities: 8 | - Route 9 | - Direction 10 | - hour 11 | - day 12 | - delta 13 | - scale 14 | 15 | slots: 16 | Route: 17 | type: categorical 18 | values: 19 | - 501 20 | - 502 21 | - 503 22 | - 504 23 | - 505 24 | - 506 25 | - 508 26 | - 509 27 | - 510 28 | - 511 29 | - 512 30 | - 301 31 | - 304 32 | - 306 33 | - 310 34 | Direction: 35 | type: categorical 36 | initial_value: e 37 | values: 38 | - n 39 | - e 40 | - s 41 | - w 42 | - b 43 | hour: 44 | type: categorical 45 | values: 46 | - 0 47 | - 1 48 | - 2 49 | - 3 50 | - 4 51 | - 5 52 | - 6 53 | - 7 54 | - 8 55 | - 9 56 | - 10 57 | - 11 58 | - 12 59 | - 13 60 | - 14 61 | - 15 62 | - 16 63 | - 17 64 | - 18 65 | - 19 66 | - 20 67 | - 21 68 | - 22 69 | - 23 70 | month: 71 | type: categorical 72 | values: 73 | - 1 74 | - 2 75 | - 3 76 | - 4 77 | - 5 78 | - 6 79 | - 7 80 | - 8 81 | - 9 82 | - 10 83 | - 11 84 | - 12 85 | year: 86 | type: categorical 87 | initial_value: 2019 88 | values: 89 | - 2014 90 | - 2015 91 | - 2016 92 | - 2017 93 | - 2018 94 | - 2019 95 | - 2020 96 | - 2021 97 | - 2022 98 | - 2023 99 | day: 100 | type: text 101 | daym: 102 | type: text 103 | delta: 104 | type: text 105 | initial_value: 0 106 | scale: 107 | type: text 108 | 109 | 110 | actions: 111 | - utter_greet 112 | - utter_goodbye 113 | - action_predict_delay 114 | - action_predict_delay_complete 115 | 116 | templates: 117 | utter_greet: 118 | - text: "Hey! How are you?" 119 | 120 | utter_goodbye: 121 | - text: "Bye" 122 | -------------------------------------------------------------------------------- /deploy/endpoints.yml: -------------------------------------------------------------------------------- 1 | # This file contains the different endpoints your bot can use. 2 | 3 | # Server where the models are pulled from. 4 | # https://rasa.com/docs/rasa/user-guide/running-the-server/#fetching-models-from-a-server/ 5 | 6 | #models: 7 | # url: http://my-server.com/models/default_core@latest 8 | # wait_time_between_pulls: 10 # [optional](default: 100) 9 | 10 | # Server which runs your custom actions. 11 | # https://rasa.com/docs/rasa/core/actions/#custom-actions/ 12 | 13 | action_endpoint: 14 | url: "http://localhost:5055/webhook" 15 | 16 | # Tracker store which is used to store the conversations. 17 | # By default the conversations are stored in memory. 18 | # https://rasa.com/docs/rasa/api/tracker-stores/ 19 | 20 | #tracker_store: 21 | # type: redis 22 | # url: 23 | # port: 24 | # db: 25 | # password: 26 | 27 | #tracker_store: 28 | # type: mongod 29 | # url: 30 | # db: 31 | # username: 32 | # password: 33 | 34 | # Event broker which all conversation events should be streamed to. 35 | # https://rasa.com/docs/rasa/api/event-brokers/ 36 | 37 | #event_broker: 38 | # url: localhost 39 | # username: username 40 | # password: password 41 | # queue: queue 42 | -------------------------------------------------------------------------------- /deploy/input_sample_data/prepped_dec27.csv: -------------------------------------------------------------------------------- 1 | Route,Direction,hour,year,month,daym,day, 2 | 301,e,0,2018,1,1,2, 3 | 301,e,1,2017,1,1,2, 4 | -------------------------------------------------------------------------------- /deploy/input_sample_data/raw_dec27.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/input_sample_data/raw_dec27.xlsx -------------------------------------------------------------------------------- /deploy/keras_models/scmodeldec27b_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/keras_models/scmodeldec27b_5.h5 -------------------------------------------------------------------------------- /deploy/models/20191207-201544.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/models/20191207-201544.tar.gz -------------------------------------------------------------------------------- /deploy/models/20191210-220835.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/models/20191210-220835.tar.gz -------------------------------------------------------------------------------- /deploy/models/20191228-145744.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/models/20191228-145744.tar.gz -------------------------------------------------------------------------------- /deploy/models/20191228-192944.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/models/20191228-192944.tar.gz -------------------------------------------------------------------------------- /deploy/models/20191228-193317.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/models/20191228-193317.tar.gz -------------------------------------------------------------------------------- /deploy/models/20191228-202857.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/models/20191228-202857.tar.gz -------------------------------------------------------------------------------- /deploy/models/20191228-204246.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/models/20191228-204246.tar.gz -------------------------------------------------------------------------------- /deploy/models/20191228-212651.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/models/20191228-212651.tar.gz -------------------------------------------------------------------------------- /deploy/models/20191229-081543.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/models/20191229-081543.tar.gz -------------------------------------------------------------------------------- /deploy/models/20191229-085443.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/models/20191229-085443.tar.gz -------------------------------------------------------------------------------- /deploy/models/20191229-142939.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/models/20191229-142939.tar.gz -------------------------------------------------------------------------------- /deploy/one_off_scoring.py: -------------------------------------------------------------------------------- 1 | # stub to do one-off scoring using the trained streetcar delay model 2 | 3 | # imports 4 | 5 | import pandas as pd 6 | import numpy as np 7 | import time 8 | # import datetime, timedelta 9 | import datetime 10 | from datetime import datetime, timedelta 11 | from datetime import date 12 | from dateutil import relativedelta 13 | from io import StringIO 14 | import pandas as pd 15 | import pickle 16 | from sklearn.base import BaseEstimator 17 | from sklearn.base import TransformerMixin 18 | from io import StringIO 19 | import requests 20 | import json 21 | from sklearn.preprocessing import LabelEncoder, MinMaxScaler, StandardScaler 22 | from sklearn.model_selection import train_test_split 23 | import matplotlib.pyplot as plt 24 | import os 25 | import math 26 | from subprocess import check_output 27 | #model libraries 28 | from keras.layers import Input, Dropout, Dense, BatchNormalization, Activation, concatenate, GRU, Embedding, Flatten, BatchNormalization 29 | from keras.models import Model 30 | from keras.callbacks import ModelCheckpoint, Callback, EarlyStopping 31 | from keras import regularizers 32 | from keras.layers.normalization import BatchNormalization 33 | from keras.regularizers import l2 34 | from keras.optimizers import Adam 35 | from keras.optimizers import SGD 36 | from keras import backend as K 37 | from keras.utils.vis_utils import plot_model 38 | from keras.preprocessing.text import Tokenizer 39 | from keras.preprocessing.sequence import pad_sequences 40 | from keras.models import load_model 41 | from sklearn import metrics 42 | from sklearn.datasets.samples_generator import make_blobs 43 | from sklearn.model_selection import train_test_split 44 | from sklearn.metrics import accuracy_score 45 | from pickle import load 46 | from custom_classes import encode_categorical 47 | from custom_classes import prep_for_keras_input 48 | from custom_classes import fill_empty 49 | from custom_classes import encode_text 50 | from datetime import date 51 | from datetime import datetime 52 | import logging 53 | 54 | logging.getLogger().setLevel(logging.WARNING) 55 | logging.warning("logging check") 56 | 57 | # define key values: 58 | 59 | model_path = 'C:\personal\chatbot_july_2019\streetcar_1\keras_models\scmodeldec27b_5.h5' 60 | BATCH_SIZE = 1000 61 | 62 | ''' 63 | Output of X_test structure (used to test as part of the training process) to 64 | use to get a single record to exercise the trained model 65 | 66 | X_test {'hour': array([18, 4, 11, ..., 2, 23, 17]), 'Route': array([ 0, 12, 2, ..., 10, 12, 2]), 'daym': array([21, 16, 10, ..., 12, 26, 6]), 67 | 'month': array([0, 1, 0, ..., 6, 2, 1]), 'year': array([5, 2, 3, ..., 1, 4, 3]), 'Direction': array([1, 1, 4, ..., 2, 3, 0]), 68 | 'day': array([1, 2, 2, ..., 0, 1, 1])} 69 | 70 | 71 | ''' 72 | 73 | 74 | # define dictionary containing hand-crafted input for the trained model to score 75 | # take the first value from each of the numpy arrays in X_test above 76 | score_sample = {} 77 | score_sample['hour'] = np.array([18]) 78 | score_sample['Route'] = np.array([0]) 79 | score_sample['daym'] = np.array([21]) 80 | score_sample['month'] = np.array([0]) 81 | score_sample['year'] = np.array([5]) 82 | score_sample['Direction'] = np.array([1]) 83 | score_sample['day'] = np.array([1]) 84 | 85 | # load model from file, print summary and make prediction for the hand-crafted score-sample 86 | 87 | loaded_model = load_model(model_path) 88 | loaded_model.summary() 89 | 90 | preds = loaded_model.predict(score_sample, batch_size=BATCH_SIZE) 91 | 92 | print("prediction is "+str(preds[0][0])) 93 | 94 | -------------------------------------------------------------------------------- /deploy/pipelines/sc_delay_pipeline_dec27b.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/pipelines/sc_delay_pipeline_dec27b.pkl -------------------------------------------------------------------------------- /deploy/pipelines/sc_delay_pipeline_keras_prep_dec27b.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/pipelines/sc_delay_pipeline_keras_prep_dec27b.pkl -------------------------------------------------------------------------------- /deploy/pipelines/sc_delay_pipelinedec27.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy/pipelines/sc_delay_pipelinedec27.pkl -------------------------------------------------------------------------------- /deploy_web/__pycache__/custom_classes.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/deploy_web/__pycache__/custom_classes.cpython-37.pyc -------------------------------------------------------------------------------- /deploy_web/custom_classes.py: -------------------------------------------------------------------------------- 1 | from sklearn.model_selection import train_test_split 2 | from sklearn.metrics import accuracy_score 3 | from sklearn.metrics import average_precision_score 4 | from sklearn.pipeline import Pipeline 5 | from sklearn.base import TransformerMixin 6 | from sklearn.base import BaseEstimator 7 | from sklearn.preprocessing import LabelEncoder, MinMaxScaler, StandardScaler 8 | import numpy as np 9 | 10 | 11 | 12 | 13 | 14 | class encode_categorical(BaseEstimator, TransformerMixin): 15 | '''encode categorical columns''' 16 | def __init__(self): 17 | self.le = {} 18 | self.max_dict = {} 19 | return None 20 | 21 | def set_params(self, **kwargs): 22 | self.col_list = kwargs.get('col_list', None) 23 | return self 24 | 25 | 26 | 27 | def fit(self, X, y=None, **fit_params): 28 | for col in self.col_list: 29 | print("col is ",col) 30 | self.le[col] = LabelEncoder() 31 | self.le[col].fit(X[col].tolist()) 32 | return self 33 | 34 | 35 | def transform(self, X, y=None, **tranform_params): 36 | for col in self.col_list: 37 | print("transform col is ",col) 38 | X[col] = self.le[col].transform(X[col]) 39 | print("after transform col is ",col) 40 | self.max_dict[col] = X[col].max() +1 41 | # print(X.loc[[0]]) 42 | return X 43 | 44 | class prep_for_keras_input(BaseEstimator, TransformerMixin): 45 | '''prep columns for feeding Keras model''' 46 | def __init__(self): 47 | self.dictlist = [] 48 | return None 49 | 50 | def set_params(self, **kwargs): 51 | self.collist = kwargs.get('collist', None) 52 | self.continuouscols = kwargs.get('continuouscols', None) 53 | self.textcols = kwargs.get('textcols', None) 54 | return self 55 | 56 | 57 | 58 | def fit(self, X, y=None, **fit_params): 59 | return self 60 | 61 | 62 | def transform(self, X, y=None, **tranform_params): 63 | for col in self.collist: 64 | print("cat col is",col) 65 | self.dictlist.append(np.array(X[col])) 66 | for col in self.textcols: 67 | print("text col is",col) 68 | self.dictlist.append(pad_sequences(X[col], maxlen=max_dict[col])) 69 | for col in self.continuouscols: 70 | print("cont col is",col) 71 | self.dictlist.append(np.array(X[col])) 72 | # print(X.loc[[0]]) 73 | return self.dictlist 74 | 75 | class fill_empty(BaseEstimator, TransformerMixin): 76 | '''fill empty values with placeholders''' 77 | 78 | 79 | def set_params(self, **kwargs): 80 | self.collist = kwargs.get('collist', None) 81 | self.continuouscols = kwargs.get('continuouscols', None) 82 | self.textcols = kwargs.get('textcols', None) 83 | return self 84 | 85 | 86 | def transform(self, X, **tranform_params): 87 | print("fill empty xform") 88 | for col in self.collist: 89 | X[col].fillna(value="missing", inplace=True) 90 | for col in self.continuouscols: 91 | X[col].fillna(value=0.0,inplace=True) 92 | for col in self.textcols: 93 | X[col].fillna(value="missing", inplace=True) 94 | return X 95 | 96 | def fit(self, X, y=None, **fit_params): 97 | return self 98 | 99 | class encode_text(BaseEstimator, TransformerMixin): 100 | '''encode text columns''' 101 | def __init__(self): 102 | self.tok = {} 103 | return None 104 | 105 | def set_params(self, **kwargs): 106 | self.col_list = kwargs.get('col_list', None) 107 | return self 108 | 109 | 110 | 111 | def fit(self, X, y=None, **fit_params): 112 | for col in self.col_list: 113 | print("col is ",col) 114 | self.tok[col] = Tokenizer(num_words=maxwords,lower=True) 115 | self.tok[col].fit_on_texts(X[col]) 116 | return self 117 | 118 | 119 | def transform(self, X, y=None, **tranform_params): 120 | for col in self.col_list: 121 | print("transform col is ",col) 122 | X[col] = self.tok[col].texts_to_sequences(X[col]) 123 | print("after transform col is ",col) 124 | self.max_dict[col] = np.max(X[(X[col].map(len) != 0)][col].map(max)) 125 | if self.max_dict[cols] > textmax: 126 | textmax = self.max_dict[cols] 127 | # print(X.loc[[0]]) 128 | return X 129 | -------------------------------------------------------------------------------- /deploy_web/deploy_web_config.yml: -------------------------------------------------------------------------------- 1 | general: 2 | debug_on: False 3 | logging_level: "WARNING" # switch to control logging - WARNING for full logging; ERROR to minimize logging 4 | BATCH_SIZE: 1000 5 | file_names: 6 | pipeline1_filename: sc_delay_piplelineapr26_2020.pkl 7 | pipeline2_filename: sc_delay_pipleline_keras_prepapr26_2020.pkl 8 | model_filename: scmodelapr26_2020_5.h5 9 | -------------------------------------------------------------------------------- /deploy_web/flask_server.py: -------------------------------------------------------------------------------- 1 | # tester app to exercised Flask 2 | 3 | from flask import Flask, render_template, request 4 | from string import Template 5 | from OpenSSL import SSL 6 | import pickle 7 | import requests 8 | import json 9 | import pandas as pd 10 | import numpy as np 11 | import time 12 | #model libraries 13 | from tensorflow.keras.layers import Input, Dropout, Dense, BatchNormalization, Activation, concatenate, GRU, Embedding, Flatten, BatchNormalization 14 | from tensorflow.keras.models import Model 15 | from tensorflow.keras.callbacks import ModelCheckpoint, Callback, EarlyStopping 16 | from tensorflow.keras import regularizers 17 | from tensorflow.keras.layers import BatchNormalization 18 | from tensorflow.keras.regularizers import l2 19 | from tensorflow.keras.optimizers import Adam 20 | from tensorflow.keras.optimizers import SGD 21 | from tensorflow.keras import backend as K 22 | from tensorflow.keras.utils import plot_model 23 | from tensorflow.keras.preprocessing.text import Tokenizer 24 | from tensorflow.keras.preprocessing.sequence import pad_sequences 25 | from tensorflow.keras.models import load_model 26 | from sklearn import metrics 27 | from sklearn.datasets.samples_generator import make_blobs 28 | from sklearn.model_selection import train_test_split 29 | from sklearn.metrics import accuracy_score 30 | from sklearn.preprocessing import label 31 | from pickle import load 32 | from custom_classes import encode_categorical 33 | from custom_classes import prep_for_keras_input 34 | from custom_classes import fill_empty 35 | from custom_classes import encode_text 36 | from datetime import date 37 | from datetime import datetime 38 | import os 39 | import logging 40 | import yaml 41 | 42 | # load config gile 43 | current_path = os.getcwd() 44 | print("current directory is: "+current_path) 45 | path_to_yaml = os.path.join(current_path, 'deploy_web_config.yml') 46 | print("path_to_yaml "+path_to_yaml) 47 | try: 48 | with open (path_to_yaml, 'r') as c_file: 49 | config = yaml.safe_load(c_file) 50 | except Exception as e: 51 | print('Error reading the config file') 52 | 53 | # paths for model and pipeline files 54 | pipeline1_filename = config['file_names']['pipeline1_filename'] 55 | pipeline2_filename = config['file_names']['pipeline2_filename'] 56 | model_filename = config['file_names']['model_filename'] 57 | 58 | # other parms 59 | debug_on = config['general']['debug_on'] 60 | logging_level = config['general']['logging_level'] 61 | BATCH_SIZE = config['general']['BATCH_SIZE'] 62 | 63 | # set logging level 64 | logging_level_set = logging.WARNING 65 | if logging_level == 'WARNING': 66 | logging_level_set = logging.WARNING 67 | if logging_level == 'ERROR': 68 | logging_level_set = logging.ERROR 69 | if logging_level == 'DEBUG': 70 | logging_level_set = logging.DEBUG 71 | if logging_level == 'INFO': 72 | logging_level_set = logging.INFO 73 | logging.getLogger().setLevel(logging_level_set) 74 | logging.warning("logging check - beginning of logging") 75 | 76 | def get_path(subpath): 77 | rawpath = os.getcwd() 78 | # data is in a directory called "data" that is a sibling to the directory containing the notebook 79 | path = os.path.abspath(os.path.join(rawpath, '..', subpath)) 80 | return(path) 81 | 82 | # get complete paths for pipelines and Keras models 83 | pipeline_path = get_path('pipelines') 84 | 85 | pipeline1_path = os.path.join(pipeline_path,pipeline1_filename) 86 | pipeline2_path = os.path.join(pipeline_path,pipeline2_filename) 87 | model_path = os.path.join(get_path('models'),model_filename) 88 | 89 | ''' 90 | LOADING THE MODEL HERE CAUSES AN ERROR - LOADING IT WHEN THE PAGE IS LOADED IS FINE 91 | 92 | # load the Keras model 93 | loaded_model = load_model(model_path) 94 | loaded_model._make_predict_function() 95 | 96 | loaded_model.summary() 97 | ''' 98 | 99 | # brute force a scoring sample, bagged from test set 100 | score_sample = {} 101 | score_sample['hour'] = np.array([18]) 102 | score_sample['Route'] = np.array([0]) 103 | score_sample['daym'] = np.array([21]) 104 | score_sample['month'] = np.array([0]) 105 | score_sample['year'] = np.array([5]) 106 | score_sample['Direction'] = np.array([1]) 107 | score_sample['day'] = np.array([1]) 108 | 109 | # columns that are required for scoring 110 | score_cols = ['Route','Direction','hour','year','month','daym','day'] 111 | 112 | 113 | 114 | 115 | app = Flask(__name__) 116 | 117 | HTML_TEMPLATE = Template(""" 118 |

Hello ${file_name}!

119 | 120 | poster for ${file_name} 121 | 122 | """) 123 | 124 | 125 | 126 | @app.route('/') 127 | def home(): 128 | ''' render home page that is served at localhost and allows the user to enter details about their streetcar trip''' 129 | title_text = "Test title" 130 | title = {'titlename':title_text} 131 | return render_template('home.html',title=title) 132 | 133 | 134 | @app.route('/show-prediction/') 135 | def about(): 136 | ''' get the scoring parameters entered in home.html, assemble them into a dataframe, run that dataframe through pipelines 137 | apply the trained model to the output of the pipeline, and display the interpreted score in show-prediction.html 138 | ''' 139 | # the scoring parameters are sent to this page as parameters on the URL link from home.html 140 | # load the scoring parameter values into a dictionary indexed by the column names expected by the pipelines 141 | score_values_dict = {} 142 | score_values_dict['Route'] = request.args.get('route') 143 | score_values_dict['Direction'] = request.args.get('direction') 144 | score_values_dict['year'] = int(request.args.get('year')) 145 | score_values_dict['month'] = int(request.args.get('month')) 146 | score_values_dict['daym'] = int(request.args.get('daym')) 147 | score_values_dict['day'] = int(request.args.get('day')) 148 | score_values_dict['hour'] = int(request.args.get('hour')) 149 | # echo the parameter values 150 | for value in score_values_dict: 151 | logging.warning("value for "+value+" is: "+str(score_values_dict[value])) 152 | # load the trained model 153 | loaded_model = load_model(model_path) 154 | loaded_model._make_predict_function() 155 | logging.warning("DISPLAY PRED: model loaded") 156 | # load the pipelines 157 | pipeline1 = load(open(pipeline1_path, 'rb')) 158 | pipeline2 = load(open(pipeline2_path, 'rb')) 159 | logging.warning("DISPLAY PRED: pipelines loaded") 160 | # create and load scoring parameters dataframe (containing the scoring parameters)that will be fed into the pipelines 161 | score_df = pd.DataFrame(columns=score_cols) 162 | logging.warning("score_df before load is "+str(score_df)) 163 | for col in score_cols: 164 | score_df.at[0,col] = score_values_dict[col] 165 | # apply the pipelines to the scoring parameters dataframe 166 | prepped_xform1 = pipeline1.transform(score_df) 167 | prepped_xform2 = pipeline2.transform(prepped_xform1) 168 | logging.warning("prepped_xform2 is ",prepped_xform2) 169 | # apply the trained model to the output of the pipelines to get a score 170 | pred = loaded_model.predict(prepped_xform2, batch_size=BATCH_SIZE) 171 | logging.warning("pred is "+str(pred)) 172 | # get a result string from the value of the score 173 | if pred[0][0] >= 0.5: 174 | predict_string = "yes, delay predicted" 175 | else: 176 | predict_string = "no delay predicted" 177 | prediction = {'prediction_key':predict_string} 178 | # render the page that will show the prediction 179 | return(render_template('show-prediction.html',prediction=prediction)) 180 | 181 | 182 | 183 | 184 | if __name__ == '__main__': 185 | app.run(debug=True, host='0.0.0.0') 186 | 187 | -------------------------------------------------------------------------------- /deploy_web/static/css/main.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0.2; 3 | padding: 0; 4 | font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; 5 | color: #444; 6 | } 7 | /* 8 | * Formatting the header area 9 | */ 10 | header { 11 | background-color: #DFB887; 12 | height: 35px; 13 | width: 100%; 14 | opacity: .9; 15 | margin-bottom: 10px; 16 | } 17 | header h1.logo { 18 | margin: 0; 19 | font-size: 1.7em; 20 | color: #fff; 21 | text-transform: uppercase; 22 | float: left; 23 | } 24 | header h1.logo:hover { 25 | color: #fff; 26 | text-decoration: none; 27 | } 28 | /* 29 | * Centering the body content 30 | */ 31 | .container { 32 | width: 1200px; 33 | margin: 0 auto; 34 | } 35 | div.home { 36 | padding: 10px 0 30px 0; 37 | font-size: 1.2em; 38 | background-color: #E6E6FA; 39 | -webkit-border-radius: 6px; 40 | -moz-border-radius: 6px; 41 | border-radius: 6px; 42 | 43 | } 44 | 45 | div.about { 46 | padding: 10px 0 30px 0; 47 | background-color: #E6E6FA; 48 | -webkit-border-radius: 6px; 49 | -moz-border-radius: 6px; 50 | border-radius: 6px; 51 | } 52 | h2 { 53 | font-size: 3em; 54 | margin-top: 40px; 55 | text-align: center; 56 | letter-spacing: -2px; 57 | } 58 | h3 { 59 | font-size: 1.7em; 60 | font-weight: 100; 61 | margin-top: 30px; 62 | text-align: center; 63 | letter-spacing: -1px; 64 | color: #999; 65 | } 66 | .menu { 67 | float: right; 68 | margin-top: 8px; 69 | } 70 | .menu li { 71 | display: inline; 72 | } 73 | .menu li + li { 74 | margin-left: 35px; 75 | } 76 | .menu li a { 77 | color: #444; 78 | text-decoration: none; 79 | } 80 | -------------------------------------------------------------------------------- /deploy_web/static/css/main2.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0.2; 3 | padding: 0; 4 | font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; 5 | color: #444; 6 | } 7 | /* 8 | * Formatting the header area 9 | */ 10 | header { 11 | background-color: #DFB887; 12 | height: 35px; 13 | width: 100%; 14 | opacity: .9; 15 | margin-bottom: 10px; 16 | } 17 | header h1.logo { 18 | margin: 0; 19 | font-size: 1.7em; 20 | color: #fff; 21 | text-transform: uppercase; 22 | float: left; 23 | } 24 | header h1.logo:hover { 25 | color: #fff; 26 | text-decoration: none; 27 | } 28 | /* 29 | * Centering the body content 30 | */ 31 | .container { 32 | width: 1200px; 33 | margin: 0 auto; 34 | } 35 | div.home { 36 | padding: 10px 0 30px 0; 37 | font-size: 1.2em; 38 | background-color: #E6E6FA; 39 | -webkit-border-radius: 6px; 40 | -moz-border-radius: 6px; 41 | border-radius: 6px; 42 | 43 | } 44 | 45 | div.about { 46 | padding: 10px 0 30px 0; 47 | background-color: #E6E6FA; 48 | -webkit-border-radius: 6px; 49 | -moz-border-radius: 6px; 50 | border-radius: 6px; 51 | } 52 | h2 { 53 | font-size: 3em; 54 | margin-top: 40px; 55 | text-align: center; 56 | letter-spacing: -2px; 57 | } 58 | h3 { 59 | font-size: 1.7em; 60 | font-weight: 100; 61 | margin-top: 30px; 62 | text-align: center; 63 | letter-spacing: -1px; 64 | color: #999; 65 | } 66 | .menu { 67 | float: right; 68 | margin-top: 8px; 69 | } 70 | .menu li { 71 | display: inline; 72 | } 73 | .menu li + li { 74 | margin-left: 35px; 75 | } 76 | .menu li a { 77 | color: #444; 78 | text-decoration: none; 79 | } 80 | -------------------------------------------------------------------------------- /deploy_web/templates/home.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | How to get selected value in 5 | dropdown list using JavaScript? 6 | 7 | 8 | 9 | 10 | 11 |
12 |

13 | Please select the details for your streetcar trip 14 |

15 | 16 |

17 | Select the route of your trip: 18 | 33 |

34 |

35 | Select the direction of your trip: 36 | 42 |

43 |

44 | Select the year of your trip: 45 | 55 |

56 |

57 | Select the month of your trip: 58 | 72 |

73 |

74 | Select the day of the month of your trip: 75 | 108 |

109 |

110 | Select the day of your trip: 111 | 112 | 121 |

122 |

123 | Select the hour of your trip: 124 | 152 |

153 | 154 | 155 | 158 | 159 | 160 |

161 | The value of the option selected is: 162 | 163 |

164 | 165 | 166 | 167 | 193 | 203 |
204 | 205 | 206 | 207 | 208 | 209 | 210 | -------------------------------------------------------------------------------- /deploy_web/templates/show-prediction.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Page for showing prediction 5 | 6 | 7 | 8 | 9 | 10 | 11 |
12 |

13 | Here is the prediction for your streetcar trip: 14 |

15 |

16 | {{ prediction.prediction_key }} 17 |

18 | 19 | 20 | 21 |
22 | 23 |
24 | 25 |
26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /models/modelapr16_2020.json: -------------------------------------------------------------------------------- 1 | {"class_name": "Model", "config": {"name": "model_3", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "hour"}, "name": "hour", "inbound_nodes": []}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "day"}, "name": "day", "inbound_nodes": []}, {"class_name": "Embedding", "config": {"name": "embedding_21", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 24, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_21", "inbound_nodes": [[["hour", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_22", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 7, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_22", "inbound_nodes": [[["day", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "daym"}, "name": "daym", "inbound_nodes": []}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_21", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_21", "inbound_nodes": [[["embedding_21", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_22", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_22", "inbound_nodes": [[["embedding_22", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_23", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 31, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_23", "inbound_nodes": [[["daym", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "Direction"}, "name": "Direction", "inbound_nodes": []}, {"class_name": "Flatten", "config": {"name": "flatten_21", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_21", "inbound_nodes": [[["batch_normalization_21", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_22", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_22", "inbound_nodes": [[["batch_normalization_22", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_23", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_23", "inbound_nodes": [[["embedding_23", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_24", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 5, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_24", "inbound_nodes": [[["Direction", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "year"}, "name": "year", "inbound_nodes": []}, {"class_name": "Dropout", "config": {"name": "dropout_21", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_21", "inbound_nodes": [[["flatten_21", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_22", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_22", "inbound_nodes": [[["flatten_22", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_23", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_23", "inbound_nodes": [[["batch_normalization_23", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_24", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_24", "inbound_nodes": [[["embedding_24", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_25", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 6, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_25", "inbound_nodes": [[["year", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "Route"}, "name": "Route", "inbound_nodes": []}, {"class_name": "Concatenate", "config": {"name": "concatenate_18", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_18", "inbound_nodes": [[["dropout_21", 0, 0, {}], ["dropout_22", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_23", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_23", "inbound_nodes": [[["flatten_23", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_24", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_24", "inbound_nodes": [[["batch_normalization_24", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_25", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_25", "inbound_nodes": [[["embedding_25", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_26", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 14, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_26", "inbound_nodes": [[["Route", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "month"}, "name": "month", "inbound_nodes": []}, {"class_name": "Concatenate", "config": {"name": "concatenate_19", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_19", "inbound_nodes": [[["concatenate_18", 0, 0, {}], ["dropout_23", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_24", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_24", "inbound_nodes": [[["flatten_24", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_25", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_25", "inbound_nodes": [[["batch_normalization_25", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_26", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_26", "inbound_nodes": [[["embedding_26", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_27", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 12, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_27", "inbound_nodes": [[["month", 0, 0, {}]]]}, {"class_name": "Concatenate", "config": {"name": "concatenate_20", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_20", "inbound_nodes": [[["concatenate_19", 0, 0, {}], ["dropout_24", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_25", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_25", "inbound_nodes": [[["flatten_25", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_26", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_26", "inbound_nodes": [[["batch_normalization_26", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_27", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_27", "inbound_nodes": [[["embedding_27", 0, 0, {}]]]}, {"class_name": "Concatenate", "config": {"name": "concatenate_21", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_21", "inbound_nodes": [[["concatenate_20", 0, 0, {}], ["dropout_25", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_26", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_26", "inbound_nodes": [[["flatten_26", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_27", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_27", "inbound_nodes": [[["batch_normalization_27", 0, 0, {}]]]}, {"class_name": "Concatenate", "config": {"name": "concatenate_22", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_22", "inbound_nodes": [[["concatenate_21", 0, 0, {}], ["dropout_26", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_27", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_27", "inbound_nodes": [[["flatten_27", 0, 0, {}]]]}, {"class_name": "Concatenate", "config": {"name": "concatenate_23", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_23", "inbound_nodes": [[["concatenate_22", 0, 0, {}], ["dropout_27", 0, 0, {}]]]}, {"class_name": "Dense", "config": {"name": "dense_3", "trainable": true, "dtype": "float32", "units": 1, "activation": "hard_sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_3", "inbound_nodes": [[["concatenate_23", 0, 0, {}]]]}], "input_layers": [["hour", 0, 0], ["day", 0, 0], ["daym", 0, 0], ["Direction", 0, 0], ["year", 0, 0], ["Route", 0, 0], ["month", 0, 0]], "output_layers": [["dense_3", 0, 0]]}, "keras_version": "2.2.4-tf", "backend": "tensorflow"} -------------------------------------------------------------------------------- /models/modelapr17_2020.json: -------------------------------------------------------------------------------- 1 | {"class_name": "Model", "config": {"name": "model_1", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "Direction"}, "name": "Direction", "inbound_nodes": []}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "Route"}, "name": "Route", "inbound_nodes": []}, {"class_name": "Embedding", "config": {"name": "embedding_7", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 5, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_7", "inbound_nodes": [[["Direction", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_8", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 14, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_8", "inbound_nodes": [[["Route", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "daym"}, "name": "daym", "inbound_nodes": []}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_7", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_7", "inbound_nodes": [[["embedding_7", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_8", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_8", "inbound_nodes": [[["embedding_8", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_9", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 31, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_9", "inbound_nodes": [[["daym", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "month"}, "name": "month", "inbound_nodes": []}, {"class_name": "Flatten", "config": {"name": "flatten_7", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_7", "inbound_nodes": [[["batch_normalization_7", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_8", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_8", "inbound_nodes": [[["batch_normalization_8", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_9", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_9", "inbound_nodes": [[["embedding_9", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_10", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 12, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_10", "inbound_nodes": [[["month", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "day"}, "name": "day", "inbound_nodes": []}, {"class_name": "Dropout", "config": {"name": "dropout_7", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_7", "inbound_nodes": [[["flatten_7", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_8", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_8", "inbound_nodes": [[["flatten_8", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_9", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_9", "inbound_nodes": [[["batch_normalization_9", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_10", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_10", "inbound_nodes": [[["embedding_10", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_11", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 7, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_11", "inbound_nodes": [[["day", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "year"}, "name": "year", "inbound_nodes": []}, {"class_name": "Concatenate", "config": {"name": "concatenate_6", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_6", "inbound_nodes": [[["dropout_7", 0, 0, {}], ["dropout_8", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_9", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_9", "inbound_nodes": [[["flatten_9", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_10", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_10", "inbound_nodes": [[["batch_normalization_10", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_11", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_11", "inbound_nodes": [[["embedding_11", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_12", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 6, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_12", "inbound_nodes": [[["year", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "hour"}, "name": "hour", "inbound_nodes": []}, {"class_name": "Concatenate", "config": {"name": "concatenate_7", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_7", "inbound_nodes": [[["concatenate_6", 0, 0, {}], ["dropout_9", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_10", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_10", "inbound_nodes": [[["flatten_10", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_11", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_11", "inbound_nodes": [[["batch_normalization_11", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_12", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_12", "inbound_nodes": [[["embedding_12", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_13", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 24, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_13", "inbound_nodes": [[["hour", 0, 0, {}]]]}, {"class_name": "Concatenate", "config": {"name": "concatenate_8", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_8", "inbound_nodes": [[["concatenate_7", 0, 0, {}], ["dropout_10", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_11", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_11", "inbound_nodes": [[["flatten_11", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_12", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_12", "inbound_nodes": [[["batch_normalization_12", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_13", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_13", "inbound_nodes": [[["embedding_13", 0, 0, {}]]]}, {"class_name": "Concatenate", "config": {"name": "concatenate_9", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_9", "inbound_nodes": [[["concatenate_8", 0, 0, {}], ["dropout_11", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_12", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_12", "inbound_nodes": [[["flatten_12", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_13", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_13", "inbound_nodes": [[["batch_normalization_13", 0, 0, {}]]]}, {"class_name": "Concatenate", "config": {"name": "concatenate_10", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_10", "inbound_nodes": [[["concatenate_9", 0, 0, {}], ["dropout_12", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_13", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_13", "inbound_nodes": [[["flatten_13", 0, 0, {}]]]}, {"class_name": "Concatenate", "config": {"name": "concatenate_11", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_11", "inbound_nodes": [[["concatenate_10", 0, 0, {}], ["dropout_13", 0, 0, {}]]]}, {"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "units": 1, "activation": "hard_sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_1", "inbound_nodes": [[["concatenate_11", 0, 0, {}]]]}], "input_layers": [["Direction", 0, 0], ["Route", 0, 0], ["daym", 0, 0], ["month", 0, 0], ["day", 0, 0], ["year", 0, 0], ["hour", 0, 0]], "output_layers": [["dense_1", 0, 0]]}, "keras_version": "2.2.4-tf", "backend": "tensorflow"} -------------------------------------------------------------------------------- /models/modelapr18_2020.json: -------------------------------------------------------------------------------- 1 | {"class_name": "Model", "config": {"name": "model_11", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "year"}, "name": "year", "inbound_nodes": []}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "Direction"}, "name": "Direction", "inbound_nodes": []}, {"class_name": "Embedding", "config": {"name": "embedding_77", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 6, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_77", "inbound_nodes": [[["year", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_78", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 5, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_78", "inbound_nodes": [[["Direction", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "hour"}, "name": "hour", "inbound_nodes": []}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_77", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_77", "inbound_nodes": [[["embedding_77", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_78", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_78", "inbound_nodes": [[["embedding_78", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_79", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 24, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_79", "inbound_nodes": [[["hour", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "Route"}, "name": "Route", "inbound_nodes": []}, {"class_name": "Flatten", "config": {"name": "flatten_77", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_77", "inbound_nodes": [[["batch_normalization_77", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_78", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_78", "inbound_nodes": [[["batch_normalization_78", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_79", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_79", "inbound_nodes": [[["embedding_79", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_80", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 14, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_80", "inbound_nodes": [[["Route", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "month"}, "name": "month", "inbound_nodes": []}, {"class_name": "Dropout", "config": {"name": "dropout_77", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_77", "inbound_nodes": [[["flatten_77", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_78", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_78", "inbound_nodes": [[["flatten_78", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_79", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_79", "inbound_nodes": [[["batch_normalization_79", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_80", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_80", "inbound_nodes": [[["embedding_80", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_81", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 12, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_81", "inbound_nodes": [[["month", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "daym"}, "name": "daym", "inbound_nodes": []}, {"class_name": "Concatenate", "config": {"name": "concatenate_66", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_66", "inbound_nodes": [[["dropout_77", 0, 0, {}], ["dropout_78", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_79", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_79", "inbound_nodes": [[["flatten_79", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_80", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_80", "inbound_nodes": [[["batch_normalization_80", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_81", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_81", "inbound_nodes": [[["embedding_81", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_82", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 31, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_82", "inbound_nodes": [[["daym", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "day"}, "name": "day", "inbound_nodes": []}, {"class_name": "Concatenate", "config": {"name": "concatenate_67", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_67", "inbound_nodes": [[["concatenate_66", 0, 0, {}], ["dropout_79", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_80", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_80", "inbound_nodes": [[["flatten_80", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_81", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_81", "inbound_nodes": [[["batch_normalization_81", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_82", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_82", "inbound_nodes": [[["embedding_82", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_83", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 7, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_83", "inbound_nodes": [[["day", 0, 0, {}]]]}, {"class_name": "Concatenate", "config": {"name": "concatenate_68", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_68", "inbound_nodes": [[["concatenate_67", 0, 0, {}], ["dropout_80", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_81", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_81", "inbound_nodes": [[["flatten_81", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_82", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_82", "inbound_nodes": [[["batch_normalization_82", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_83", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_83", "inbound_nodes": [[["embedding_83", 0, 0, {}]]]}, {"class_name": "Concatenate", "config": {"name": "concatenate_69", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_69", "inbound_nodes": [[["concatenate_68", 0, 0, {}], ["dropout_81", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_82", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_82", "inbound_nodes": [[["flatten_82", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_83", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_83", "inbound_nodes": [[["batch_normalization_83", 0, 0, {}]]]}, {"class_name": "Concatenate", "config": {"name": "concatenate_70", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_70", "inbound_nodes": [[["concatenate_69", 0, 0, {}], ["dropout_82", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_83", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_83", "inbound_nodes": [[["flatten_83", 0, 0, {}]]]}, {"class_name": "Concatenate", "config": {"name": "concatenate_71", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_71", "inbound_nodes": [[["concatenate_70", 0, 0, {}], ["dropout_83", 0, 0, {}]]]}, {"class_name": "Dense", "config": {"name": "dense_11", "trainable": true, "dtype": "float32", "units": 1, "activation": "hard_sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_11", "inbound_nodes": [[["concatenate_71", 0, 0, {}]]]}], "input_layers": [["year", 0, 0], ["Direction", 0, 0], ["hour", 0, 0], ["Route", 0, 0], ["month", 0, 0], ["daym", 0, 0], ["day", 0, 0]], "output_layers": [["dense_11", 0, 0]]}, "keras_version": "2.2.4-tf", "backend": "tensorflow"} -------------------------------------------------------------------------------- /models/modelapr19_2020.json: -------------------------------------------------------------------------------- 1 | {"class_name": "Model", "config": {"name": "model_9", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "Route"}, "name": "Route", "inbound_nodes": []}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "Direction"}, "name": "Direction", "inbound_nodes": []}, {"class_name": "Embedding", "config": {"name": "embedding_63", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 14, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_63", "inbound_nodes": [[["Route", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_64", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 5, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_64", "inbound_nodes": [[["Direction", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "year"}, "name": "year", "inbound_nodes": []}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_63", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_63", "inbound_nodes": [[["embedding_63", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_64", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_64", "inbound_nodes": [[["embedding_64", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_65", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 6, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_65", "inbound_nodes": [[["year", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "day"}, "name": "day", "inbound_nodes": []}, {"class_name": "Flatten", "config": {"name": "flatten_63", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_63", "inbound_nodes": [[["batch_normalization_63", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_64", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_64", "inbound_nodes": [[["batch_normalization_64", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_65", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_65", "inbound_nodes": [[["embedding_65", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_66", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 7, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_66", "inbound_nodes": [[["day", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "month"}, "name": "month", "inbound_nodes": []}, {"class_name": "Dropout", "config": {"name": "dropout_63", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_63", "inbound_nodes": [[["flatten_63", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_64", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_64", "inbound_nodes": [[["flatten_64", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_65", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_65", "inbound_nodes": [[["batch_normalization_65", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_66", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_66", "inbound_nodes": [[["embedding_66", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_67", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 12, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_67", "inbound_nodes": [[["month", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "hour"}, "name": "hour", "inbound_nodes": []}, {"class_name": "Concatenate", "config": {"name": "concatenate_54", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_54", "inbound_nodes": [[["dropout_63", 0, 0, {}], ["dropout_64", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_65", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_65", "inbound_nodes": [[["flatten_65", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_66", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_66", "inbound_nodes": [[["batch_normalization_66", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_67", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_67", "inbound_nodes": [[["embedding_67", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_68", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 24, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_68", "inbound_nodes": [[["hour", 0, 0, {}]]]}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1], "dtype": "float32", "sparse": false, "name": "daym"}, "name": "daym", "inbound_nodes": []}, {"class_name": "Concatenate", "config": {"name": "concatenate_55", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_55", "inbound_nodes": [[["concatenate_54", 0, 0, {}], ["dropout_65", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_66", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_66", "inbound_nodes": [[["flatten_66", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_67", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_67", "inbound_nodes": [[["batch_normalization_67", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_68", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_68", "inbound_nodes": [[["embedding_68", 0, 0, {}]]]}, {"class_name": "Embedding", "config": {"name": "embedding_69", "trainable": true, "batch_input_shape": [null, null], "dtype": "float32", "input_dim": 31, "output_dim": 10, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -0.05, "maxval": 0.05, "seed": null}}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false, "input_length": null}, "name": "embedding_69", "inbound_nodes": [[["daym", 0, 0, {}]]]}, {"class_name": "Concatenate", "config": {"name": "concatenate_56", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_56", "inbound_nodes": [[["concatenate_55", 0, 0, {}], ["dropout_66", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_67", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_67", "inbound_nodes": [[["flatten_67", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_68", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_68", "inbound_nodes": [[["batch_normalization_68", 0, 0, {}]]]}, {"class_name": "BatchNormalization", "config": {"name": "batch_normalization_69", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"class_name": "Zeros", "config": {}}, "gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "name": "batch_normalization_69", "inbound_nodes": [[["embedding_69", 0, 0, {}]]]}, {"class_name": "Concatenate", "config": {"name": "concatenate_57", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_57", "inbound_nodes": [[["concatenate_56", 0, 0, {}], ["dropout_67", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_68", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_68", "inbound_nodes": [[["flatten_68", 0, 0, {}]]]}, {"class_name": "Flatten", "config": {"name": "flatten_69", "trainable": true, "dtype": "float32", "data_format": "channels_last"}, "name": "flatten_69", "inbound_nodes": [[["batch_normalization_69", 0, 0, {}]]]}, {"class_name": "Concatenate", "config": {"name": "concatenate_58", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_58", "inbound_nodes": [[["concatenate_57", 0, 0, {}], ["dropout_68", 0, 0, {}]]]}, {"class_name": "Dropout", "config": {"name": "dropout_69", "trainable": true, "dtype": "float32", "rate": 0.0003, "noise_shape": null, "seed": null}, "name": "dropout_69", "inbound_nodes": [[["flatten_69", 0, 0, {}]]]}, {"class_name": "Concatenate", "config": {"name": "concatenate_59", "trainable": true, "dtype": "float32", "axis": -1}, "name": "concatenate_59", "inbound_nodes": [[["concatenate_58", 0, 0, {}], ["dropout_69", 0, 0, {}]]]}, {"class_name": "Dense", "config": {"name": "dense_9", "trainable": true, "dtype": "float32", "units": 1, "activation": "hard_sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "name": "dense_9", "inbound_nodes": [[["concatenate_59", 0, 0, {}]]]}], "input_layers": [["Route", 0, 0], ["Direction", 0, 0], ["year", 0, 0], ["day", 0, 0], ["month", 0, 0], ["hour", 0, 0], ["daym", 0, 0]], "output_layers": [["dense_9", 0, 0]]}, "keras_version": "2.2.4-tf", "backend": "tensorflow"} -------------------------------------------------------------------------------- /models/sc_xgbmodelapr12020_5.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/sc_xgbmodelapr12020_5.txt -------------------------------------------------------------------------------- /models/sc_xgbmodelapr22020_5.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/sc_xgbmodelapr22020_5.txt -------------------------------------------------------------------------------- /models/sc_xgbmodelapr2d2020_5.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/sc_xgbmodelapr2d2020_5.txt -------------------------------------------------------------------------------- /models/sc_xgbmodelapr52020_5.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/sc_xgbmodelapr52020_5.txt -------------------------------------------------------------------------------- /models/sc_xgbmodelmar302020_5.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/sc_xgbmodelmar302020_5.txt -------------------------------------------------------------------------------- /models/sc_xgbmodelxgb_apr9d_2020_5.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/sc_xgbmodelxgb_apr9d_2020_5.txt -------------------------------------------------------------------------------- /models/scmodelapr11_2020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr11_2020_5.h5 -------------------------------------------------------------------------------- /models/scmodelapr11c_2020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr11c_2020_5.h5 -------------------------------------------------------------------------------- /models/scmodelapr12020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr12020_5.h5 -------------------------------------------------------------------------------- /models/scmodelapr12_2020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr12_2020_5.h5 -------------------------------------------------------------------------------- /models/scmodelapr12b_2020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr12b_2020_5.h5 -------------------------------------------------------------------------------- /models/scmodelapr16_2020.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr16_2020.h5 -------------------------------------------------------------------------------- /models/scmodelapr17_2020.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr17_2020.h5 -------------------------------------------------------------------------------- /models/scmodelapr17_2020_9.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr17_2020_9.h5 -------------------------------------------------------------------------------- /models/scmodelapr18_2020.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr18_2020.h5 -------------------------------------------------------------------------------- /models/scmodelapr18_2020_9.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr18_2020_9.h5 -------------------------------------------------------------------------------- /models/scmodelapr19_2020.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr19_2020.h5 -------------------------------------------------------------------------------- /models/scmodelapr19_2020_9.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr19_2020_9.h5 -------------------------------------------------------------------------------- /models/scmodelapr22020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr22020_5.h5 -------------------------------------------------------------------------------- /models/scmodelapr26_2020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr26_2020_5.h5 -------------------------------------------------------------------------------- /models/scmodelapr2b2020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr2b2020_5.h5 -------------------------------------------------------------------------------- /models/scmodelapr2d2020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr2d2020_5.h5 -------------------------------------------------------------------------------- /models/scmodelapr52020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr52020_5.h5 -------------------------------------------------------------------------------- /models/scmodelapr5_2020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr5_2020_5.h5 -------------------------------------------------------------------------------- /models/scmodelapr5_bv_2020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr5_bv_2020_5.h5 -------------------------------------------------------------------------------- /models/scmodelapr6_2020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr6_2020_5.h5 -------------------------------------------------------------------------------- /models/scmodelapr9_2020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr9_2020_5.h5 -------------------------------------------------------------------------------- /models/scmodelapr9b_2020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr9b_2020_5.h5 -------------------------------------------------------------------------------- /models/scmodelapr9c_2020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelapr9c_2020_5.h5 -------------------------------------------------------------------------------- /models/scmodeldec1.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodeldec1.h5 -------------------------------------------------------------------------------- /models/scmodeldec21_4.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodeldec21_4.h5 -------------------------------------------------------------------------------- /models/scmodeldec21_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodeldec21_5.h5 -------------------------------------------------------------------------------- /models/scmodeldec27b_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodeldec27b_5.h5 -------------------------------------------------------------------------------- /models/scmodeldec5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodeldec5.h5 -------------------------------------------------------------------------------- /models/scmodeldec8_50_epoch_val_loss.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodeldec8_50_epoch_val_loss.h5 -------------------------------------------------------------------------------- /models/scmodelmar262020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelmar262020_5.h5 -------------------------------------------------------------------------------- /models/scmodelmar26b2020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelmar26b2020_5.h5 -------------------------------------------------------------------------------- /models/scmodelmar272020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelmar272020_5.h5 -------------------------------------------------------------------------------- /models/scmodelmar302020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelmar302020_5.h5 -------------------------------------------------------------------------------- /models/scmodelmar62020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelmar62020_5.h5 -------------------------------------------------------------------------------- /models/scmodelmay16_2020_5.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scmodelmay16_2020_5.h5 -------------------------------------------------------------------------------- /models/scweightsapr16_2020.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scweightsapr16_2020.h5 -------------------------------------------------------------------------------- /models/scweightsapr17_2020.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scweightsapr17_2020.h5 -------------------------------------------------------------------------------- /models/scweightsapr18_2020.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scweightsapr18_2020.h5 -------------------------------------------------------------------------------- /models/scweightsapr19_2020.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/models/scweightsapr19_2020.h5 -------------------------------------------------------------------------------- /notebooks/.ipynb_checkpoints/chapter5-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Pandas equivalents for common SQL operations\n", 8 | "- this notebook shows the Pandas equivalents for some common SQL operations" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "metadata": {}, 14 | "source": [ 15 | "# Import libraries and ingest data\n", 16 | "- load one sheet from the streetcar delay dataset into a Pandas dataframe" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 26, 22 | "metadata": {}, 23 | "outputs": [], 24 | "source": [ 25 | "import numpy as np # linear algebra\n", 26 | "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", 27 | "import datetime\n", 28 | "import os\n" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 27, 34 | "metadata": {}, 35 | "outputs": [ 36 | { 37 | "name": "stdout", 38 | "output_type": "stream", 39 | "text": [ 40 | "raw path is C:\\personal\\manning\\deep_learning_for_structured_data\\notebooks\n" 41 | ] 42 | } 43 | ], 44 | "source": [ 45 | "# get the directory for that this notebook is in\n", 46 | "rawpath = os.getcwd()\n", 47 | "print(\"raw path is\",rawpath)" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 28, 53 | "metadata": {}, 54 | "outputs": [ 55 | { 56 | "name": "stdout", 57 | "output_type": "stream", 58 | "text": [ 59 | "path is C:\\personal\\manning\\deep_learning_for_structured_data\\data\n" 60 | ] 61 | } 62 | ], 63 | "source": [ 64 | "# data is in a directory called \"data\" that is a sibling to the directory containing the notebook\n", 65 | "# this code assumes you have copied to this directory all the XLS files from the source dataset: https://www.toronto.ca/city-government/data-research-maps/open-data/open-data-catalogue/#e8f359f0-2f47-3058-bf64-6ec488de52da\n", 66 | "path = os.path.abspath(os.path.join(rawpath, '..', 'data'))\n", 67 | "print(\"path is\", path)" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": 29, 73 | "metadata": {}, 74 | "outputs": [], 75 | "source": [ 76 | "# dataset into Pandas dataframe\n", 77 | "file = \"Streetcar Jan 2014.csv\"\n", 78 | "streetcarjan2014=pd.read_csv(os.path.join(path,file))" 79 | ] 80 | }, 81 | { 82 | "cell_type": "markdown", 83 | "metadata": {}, 84 | "source": [ 85 | "# Show SQL / Pandas equivalent statements\n", 86 | "- in the following cells, each Python statement is preceded by a comment that shows the SQL that would produce the same result as the Python statement" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 30, 92 | "metadata": {}, 93 | "outputs": [ 94 | { 95 | "data": { 96 | "text/html": [ 97 | "
\n", 98 | "\n", 111 | "\n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | "
Report DateRouteTimeDayLocationIncidentMin DelayMin GapDirectionVehicle
02014-01-025056:31:00 AMThursdayDundas and RoncesvallesLate Leaving Garage48.0E/B4018.0
12014-01-0250412:43:00 PMThursdayKing and ShawUtilized Off Route2022.0E/B4128.0
22014-01-025012:01:00 PMThursdayKingston road and BinghamHeld By1319.0W/B4016.0
\n", 169 | "
" 170 | ], 171 | "text/plain": [ 172 | " Report Date Route Time Day Location \\\n", 173 | "0 2014-01-02 505 6:31:00 AM Thursday Dundas and Roncesvalles \n", 174 | "1 2014-01-02 504 12:43:00 PM Thursday King and Shaw \n", 175 | "2 2014-01-02 501 2:01:00 PM Thursday Kingston road and Bingham \n", 176 | "\n", 177 | " Incident Min Delay Min Gap Direction Vehicle \n", 178 | "0 Late Leaving Garage 4 8.0 E/B 4018.0 \n", 179 | "1 Utilized Off Route 20 22.0 E/B 4128.0 \n", 180 | "2 Held By 13 19.0 W/B 4016.0 " 181 | ] 182 | }, 183 | "execution_count": 30, 184 | "metadata": {}, 185 | "output_type": "execute_result" 186 | } 187 | ], 188 | "source": [ 189 | "# select * from streetcarjan2014 limit 3\n", 190 | "streetcarjan2014.head(3)" 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": 31, 196 | "metadata": {}, 197 | "outputs": [ 198 | { 199 | "data": { 200 | "text/plain": [ 201 | "1 504\n", 202 | "Name: Route, dtype: int64" 203 | ] 204 | }, 205 | "execution_count": 31, 206 | "metadata": {}, 207 | "output_type": "execute_result" 208 | } 209 | ], 210 | "source": [ 211 | "# single condition on a Select\n", 212 | "# select \"Route\" from streetcarjan2014 where \"Location \" = 'King and Shaw'\n", 213 | "streetcarjan2014[streetcarjan2014.Location == \"King and Shaw\"].Route" 214 | ] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "execution_count": 32, 219 | "metadata": {}, 220 | "outputs": [ 221 | { 222 | "data": { 223 | "text/plain": [ 224 | "array(['Late Leaving Garage', 'Utilized Off Route', 'Held By',\n", 225 | " 'Investigation', 'Mechanical', 'General Delay',\n", 226 | " 'Emergency Services', 'Diversion'], dtype=object)" 227 | ] 228 | }, 229 | "execution_count": 32, 230 | "metadata": {}, 231 | "output_type": "execute_result" 232 | } 233 | ], 234 | "source": [ 235 | "# unique entries in a column\n", 236 | "# select distinct \"Incident\" from streetcarjan2014\n", 237 | "streetcarjan2014.Incident.unique()" 238 | ] 239 | }, 240 | { 241 | "cell_type": "code", 242 | "execution_count": 33, 243 | "metadata": {}, 244 | "outputs": [ 245 | { 246 | "data": { 247 | "text/html": [ 248 | "
\n", 249 | "\n", 262 | "\n", 263 | " \n", 264 | " \n", 265 | " \n", 266 | " \n", 267 | " \n", 268 | " \n", 269 | " \n", 270 | " \n", 271 | " \n", 272 | " \n", 273 | " \n", 274 | " \n", 275 | " \n", 276 | " \n", 277 | " \n", 278 | " \n", 279 | " \n", 280 | " \n", 281 | " \n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | " \n", 289 | " \n", 290 | " \n", 291 | " \n", 292 | " \n", 293 | " \n", 294 | " \n", 295 | " \n", 296 | " \n", 297 | " \n", 298 | " \n", 299 | " \n", 300 | " \n", 301 | " \n", 302 | " \n", 303 | " \n", 304 | " \n", 305 | " \n", 306 | "
Report DateRouteTimeDayLocationIncidentMin DelayMin GapDirectionVehicle
3052014-01-195048:33:00 AMSundayKing and QueenHeld By4050.0E/B4089.0
3112014-01-195117:17:00 PMSundayBathurst and FrontInvestigation3340.0S/B4179.0
\n", 307 | "
" 308 | ], 309 | "text/plain": [ 310 | " Report Date Route Time Day Location Incident \\\n", 311 | "305 2014-01-19 504 8:33:00 AM Sunday King and Queen Held By \n", 312 | "311 2014-01-19 511 7:17:00 PM Sunday Bathurst and Front Investigation \n", 313 | "\n", 314 | " Min Delay Min Gap Direction Vehicle \n", 315 | "305 40 50.0 E/B 4089.0 \n", 316 | "311 33 40.0 S/B 4179.0 " 317 | ] 318 | }, 319 | "execution_count": 33, 320 | "metadata": {}, 321 | "output_type": "execute_result" 322 | } 323 | ], 324 | "source": [ 325 | "# multiple conditions on a Select\n", 326 | "# select * from streetcarjan2014 where 'Min Delay' > 20 and Day = \"Sunday\"\n", 327 | "streetcarjan2014[(streetcarjan2014['Min Delay'] > 20) & (streetcarjan2014['Day'] == \"Sunday\")]\n" 328 | ] 329 | }, 330 | { 331 | "cell_type": "code", 332 | "execution_count": 34, 333 | "metadata": {}, 334 | "outputs": [ 335 | { 336 | "data": { 337 | "text/html": [ 338 | "
\n", 339 | "\n", 352 | "\n", 353 | " \n", 354 | " \n", 355 | " \n", 356 | " \n", 357 | " \n", 358 | " \n", 359 | " \n", 360 | " \n", 361 | " \n", 362 | " \n", 363 | " \n", 364 | " \n", 365 | " \n", 366 | " \n", 367 | " \n", 368 | " \n", 369 | " \n", 370 | " \n", 371 | " \n", 372 | " \n", 373 | " \n", 374 | " \n", 375 | " \n", 376 | " \n", 377 | " \n", 378 | " \n", 379 | " \n", 380 | " \n", 381 | " \n", 382 | " \n", 383 | " \n", 384 | " \n", 385 | " \n", 386 | " \n", 387 | " \n", 388 | " \n", 389 | " \n", 390 | " \n", 391 | " \n", 392 | " \n", 393 | " \n", 394 | " \n", 395 | " \n", 396 | " \n", 397 | " \n", 398 | " \n", 399 | " \n", 400 | " \n", 401 | " \n", 402 | " \n", 403 | " \n", 404 | " \n", 405 | " \n", 406 | " \n", 407 | " \n", 408 | " \n", 409 | " \n", 410 | " \n", 411 | " \n", 412 | " \n", 413 | " \n", 414 | " \n", 415 | " \n", 416 | " \n", 417 | " \n", 418 | " \n", 419 | " \n", 420 | " \n", 421 | " \n", 422 | " \n", 423 | " \n", 424 | " \n", 425 | " \n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | "
RouteMin Delay
14850122
27150122
27750623
29650625
29550425
29851127
29750428
29350533
30051133
31151133
30450435
30550440
28651242
29951166
29150674
23750477
23650193
\n", 448 | "
" 449 | ], 450 | "text/plain": [ 451 | " Route Min Delay\n", 452 | "148 501 22\n", 453 | "271 501 22\n", 454 | "277 506 23\n", 455 | "296 506 25\n", 456 | "295 504 25\n", 457 | "298 511 27\n", 458 | "297 504 28\n", 459 | "293 505 33\n", 460 | "300 511 33\n", 461 | "311 511 33\n", 462 | "304 504 35\n", 463 | "305 504 40\n", 464 | "286 512 42\n", 465 | "299 511 66\n", 466 | "291 506 74\n", 467 | "237 504 77\n", 468 | "236 501 93" 469 | ] 470 | }, 471 | "execution_count": 34, 472 | "metadata": {}, 473 | "output_type": "execute_result" 474 | } 475 | ], 476 | "source": [ 477 | "# ORDER BY\n", 478 | "# select \"Route\", \"Min Delay\" from streetcarjan2014 where \"Min Delay\" > 20 order by \"Min Delay\"\n", 479 | "streetcarjan2014[['Route','Min Delay']][(streetcarjan2014['Min Delay'] > 20)].sort_values('Min Delay')\n" 480 | ] 481 | }, 482 | { 483 | "cell_type": "code", 484 | "execution_count": null, 485 | "metadata": {}, 486 | "outputs": [], 487 | "source": [] 488 | } 489 | ], 490 | "metadata": { 491 | "kernelspec": { 492 | "display_name": "Python 3", 493 | "language": "python", 494 | "name": "python3" 495 | }, 496 | "language_info": { 497 | "codemirror_mode": { 498 | "name": "ipython", 499 | "version": 3 500 | }, 501 | "file_extension": ".py", 502 | "mimetype": "text/x-python", 503 | "name": "python", 504 | "nbconvert_exporter": "python", 505 | "pygments_lexer": "ipython3", 506 | "version": "3.7.4" 507 | } 508 | }, 509 | "nbformat": 4, 510 | "nbformat_minor": 2 511 | } 512 | -------------------------------------------------------------------------------- /notebooks/__pycache__/custom_classes.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/notebooks/__pycache__/custom_classes.cpython-37.pyc -------------------------------------------------------------------------------- /notebooks/airbnb_data_preparation_config.yml: -------------------------------------------------------------------------------- 1 | general: 2 | load_from_scratch: True 3 | save_transformed_dataframe: True 4 | remove_bad_values: True 5 | continuous_default: 0.0 6 | time_default: 2019-01-01 7 | categorical_default: missing 8 | text_default: missing 9 | categorical: 10 | - 'neighbourhood_group' 11 | - 'neighbourhood' 12 | - 'room_type' 13 | continuous: 14 | - 'minimum_nights' 15 | - 'number_of_reviews' 16 | - 'reviews_per_month' 17 | - 'calculated_host_listings_count' 18 | text: 19 | - 'name' 20 | - 'host_name' 21 | excluded: 22 | - 'price' 23 | - 'id' 24 | - 'latitude' 25 | - 'longitude' 26 | - 'name' 27 | - 'host_name' 28 | - 'last_review' 29 | file_names: 30 | input_csv: AB_NYC_2019.csv 31 | pickled_input_dataframe: AB_NYC_2019_df.pkl 32 | pickled_output_dataframe: AB_NYC_2019_remove_bad_values_jun21_2020.pkl 33 | -------------------------------------------------------------------------------- /notebooks/airbnb_model_training_config.yml: -------------------------------------------------------------------------------- 1 | general: 2 | verboseout: True 3 | includetext: True # switch to determine whether text fields are included in model 4 | presaved: False # switch to determine whether to train model or load saved model 5 | savemodel: False # switch to determine whether to save model 6 | picklemodel: True # switch for code to allow pickling of Keras model 7 | save_model_plot: False 8 | tensorboard_callback: False 9 | hctextmax: 7000 10 | maxwords: 6000 11 | textmax: 50 12 | pickled_data_file: '20142018_0930.pkl' 13 | pickled_dataframe: 'AB_NYC_2019_remove_bad_values_jun20_2020.pkl' 14 | route_direction_file: 'routedirection.csv' 15 | modifier: 'jun26_2020' 16 | targetthresh: 6.0 # use as boundary between delay and no delay 17 | targetcontinuous: False # switch to determine whether target is continuous 18 | emptythresh: 6000 19 | zero_weight: 1.0 # class_weight = {0 : zero_weight, 1: one_weight} 20 | one_weight: 45.878 # default one_weight 21 | one_weight_offset: 0 # additional offset to one_weight 22 | patience_threshold: 15 23 | start_year: 2014 # date of start of refactored dataset 24 | start_month: 1 25 | start_day: 1 26 | end_year: 2019 # date of end of refactored dataset 27 | end_month: 12 28 | end_day: 31 29 | test_parms: 30 | testproportion: 0.2 # proportion of data reserved for test set 31 | trainproportion: 0.8 # proportion of non-test data dedicated to training (vs. validation) 32 | current_experiment: 1 33 | repeatable_run: False # switch to control whether runs are repeated identically 34 | get_test_train_acc: True # switch to control whether block to get test and train accuracy is after training 35 | categorical: 36 | - 'neighbourhood_group' 37 | - 'neighbourhood' 38 | - 'room_type' 39 | continuous: 40 | - 'minimum_nights' 41 | - 'number_of_reviews' 42 | - 'reviews_per_month' 43 | - 'calculated_host_listings_count' 44 | text: [] 45 | excluded: 46 | - 'price' 47 | - 'id' 48 | - 'latitude' 49 | - 'longitude' 50 | - 'host_name' 51 | - 'last_review' 52 | - 'name' 53 | - 'host_name' 54 | - 'availability_365' 55 | hyperparameters: 56 | learning_rate: 0.001 57 | dropout_rate: 0.0003 #0.003 58 | l2_lambda: 0.0003 #7.5 59 | loss_func: "binary_crossentropy" 60 | output_activation: "hard_sigmoid" # set to "linear" if target is continuous 61 | batch_size: 1000 62 | epochs: 50 -------------------------------------------------------------------------------- /notebooks/chapter5.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Pandas equivalents for common SQL operations\n", 8 | "- this notebook shows the Pandas equivalents for some common SQL operations" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "metadata": {}, 14 | "source": [ 15 | "# Import libraries and ingest data\n", 16 | "- load one sheet from the streetcar delay dataset into a Pandas dataframe" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 26, 22 | "metadata": {}, 23 | "outputs": [], 24 | "source": [ 25 | "import numpy as np # linear algebra\n", 26 | "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", 27 | "import datetime\n", 28 | "import os\n" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 27, 34 | "metadata": {}, 35 | "outputs": [ 36 | { 37 | "name": "stdout", 38 | "output_type": "stream", 39 | "text": [ 40 | "raw path is C:\\personal\\manning\\deep_learning_for_structured_data\\notebooks\n" 41 | ] 42 | } 43 | ], 44 | "source": [ 45 | "# get the directory for that this notebook is in\n", 46 | "rawpath = os.getcwd()\n", 47 | "print(\"raw path is\",rawpath)" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 28, 53 | "metadata": {}, 54 | "outputs": [ 55 | { 56 | "name": "stdout", 57 | "output_type": "stream", 58 | "text": [ 59 | "path is C:\\personal\\manning\\deep_learning_for_structured_data\\data\n" 60 | ] 61 | } 62 | ], 63 | "source": [ 64 | "# data is in a directory called \"data\" that is a sibling to the directory containing the notebook\n", 65 | "# this code assumes you have copied to this directory all the XLS files from the source dataset: https://www.toronto.ca/city-government/data-research-maps/open-data/open-data-catalogue/#e8f359f0-2f47-3058-bf64-6ec488de52da\n", 66 | "path = os.path.abspath(os.path.join(rawpath, '..', 'data'))\n", 67 | "print(\"path is\", path)" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": 29, 73 | "metadata": {}, 74 | "outputs": [], 75 | "source": [ 76 | "# dataset into Pandas dataframe\n", 77 | "file = \"Streetcar Jan 2014.csv\"\n", 78 | "streetcarjan2014=pd.read_csv(os.path.join(path,file))" 79 | ] 80 | }, 81 | { 82 | "cell_type": "markdown", 83 | "metadata": {}, 84 | "source": [ 85 | "# Show SQL / Pandas equivalent statements\n", 86 | "- in the following cells, each Python statement is preceded by a comment that shows the SQL that would produce the same result as the Python statement" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 30, 92 | "metadata": {}, 93 | "outputs": [ 94 | { 95 | "data": { 96 | "text/html": [ 97 | "
\n", 98 | "\n", 111 | "\n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | "
Report DateRouteTimeDayLocationIncidentMin DelayMin GapDirectionVehicle
02014-01-025056:31:00 AMThursdayDundas and RoncesvallesLate Leaving Garage48.0E/B4018.0
12014-01-0250412:43:00 PMThursdayKing and ShawUtilized Off Route2022.0E/B4128.0
22014-01-025012:01:00 PMThursdayKingston road and BinghamHeld By1319.0W/B4016.0
\n", 169 | "
" 170 | ], 171 | "text/plain": [ 172 | " Report Date Route Time Day Location \\\n", 173 | "0 2014-01-02 505 6:31:00 AM Thursday Dundas and Roncesvalles \n", 174 | "1 2014-01-02 504 12:43:00 PM Thursday King and Shaw \n", 175 | "2 2014-01-02 501 2:01:00 PM Thursday Kingston road and Bingham \n", 176 | "\n", 177 | " Incident Min Delay Min Gap Direction Vehicle \n", 178 | "0 Late Leaving Garage 4 8.0 E/B 4018.0 \n", 179 | "1 Utilized Off Route 20 22.0 E/B 4128.0 \n", 180 | "2 Held By 13 19.0 W/B 4016.0 " 181 | ] 182 | }, 183 | "execution_count": 30, 184 | "metadata": {}, 185 | "output_type": "execute_result" 186 | } 187 | ], 188 | "source": [ 189 | "# select * from streetcarjan2014 limit 3\n", 190 | "streetcarjan2014.head(3)" 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": 31, 196 | "metadata": {}, 197 | "outputs": [ 198 | { 199 | "data": { 200 | "text/plain": [ 201 | "1 504\n", 202 | "Name: Route, dtype: int64" 203 | ] 204 | }, 205 | "execution_count": 31, 206 | "metadata": {}, 207 | "output_type": "execute_result" 208 | } 209 | ], 210 | "source": [ 211 | "# single condition on a Select\n", 212 | "# select \"Route\" from streetcarjan2014 where \"Location \" = 'King and Shaw'\n", 213 | "streetcarjan2014[streetcarjan2014.Location == \"King and Shaw\"].Route" 214 | ] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "execution_count": 32, 219 | "metadata": {}, 220 | "outputs": [ 221 | { 222 | "data": { 223 | "text/plain": [ 224 | "array(['Late Leaving Garage', 'Utilized Off Route', 'Held By',\n", 225 | " 'Investigation', 'Mechanical', 'General Delay',\n", 226 | " 'Emergency Services', 'Diversion'], dtype=object)" 227 | ] 228 | }, 229 | "execution_count": 32, 230 | "metadata": {}, 231 | "output_type": "execute_result" 232 | } 233 | ], 234 | "source": [ 235 | "# unique entries in a column\n", 236 | "# select distinct \"Incident\" from streetcarjan2014\n", 237 | "streetcarjan2014.Incident.unique()" 238 | ] 239 | }, 240 | { 241 | "cell_type": "code", 242 | "execution_count": 33, 243 | "metadata": {}, 244 | "outputs": [ 245 | { 246 | "data": { 247 | "text/html": [ 248 | "
\n", 249 | "\n", 262 | "\n", 263 | " \n", 264 | " \n", 265 | " \n", 266 | " \n", 267 | " \n", 268 | " \n", 269 | " \n", 270 | " \n", 271 | " \n", 272 | " \n", 273 | " \n", 274 | " \n", 275 | " \n", 276 | " \n", 277 | " \n", 278 | " \n", 279 | " \n", 280 | " \n", 281 | " \n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | " \n", 289 | " \n", 290 | " \n", 291 | " \n", 292 | " \n", 293 | " \n", 294 | " \n", 295 | " \n", 296 | " \n", 297 | " \n", 298 | " \n", 299 | " \n", 300 | " \n", 301 | " \n", 302 | " \n", 303 | " \n", 304 | " \n", 305 | " \n", 306 | "
Report DateRouteTimeDayLocationIncidentMin DelayMin GapDirectionVehicle
3052014-01-195048:33:00 AMSundayKing and QueenHeld By4050.0E/B4089.0
3112014-01-195117:17:00 PMSundayBathurst and FrontInvestigation3340.0S/B4179.0
\n", 307 | "
" 308 | ], 309 | "text/plain": [ 310 | " Report Date Route Time Day Location Incident \\\n", 311 | "305 2014-01-19 504 8:33:00 AM Sunday King and Queen Held By \n", 312 | "311 2014-01-19 511 7:17:00 PM Sunday Bathurst and Front Investigation \n", 313 | "\n", 314 | " Min Delay Min Gap Direction Vehicle \n", 315 | "305 40 50.0 E/B 4089.0 \n", 316 | "311 33 40.0 S/B 4179.0 " 317 | ] 318 | }, 319 | "execution_count": 33, 320 | "metadata": {}, 321 | "output_type": "execute_result" 322 | } 323 | ], 324 | "source": [ 325 | "# multiple conditions on a Select\n", 326 | "# select * from streetcarjan2014 where 'Min Delay' > 20 and Day = \"Sunday\"\n", 327 | "streetcarjan2014[(streetcarjan2014['Min Delay'] > 20) & (streetcarjan2014['Day'] == \"Sunday\")]\n" 328 | ] 329 | }, 330 | { 331 | "cell_type": "code", 332 | "execution_count": 34, 333 | "metadata": {}, 334 | "outputs": [ 335 | { 336 | "data": { 337 | "text/html": [ 338 | "
\n", 339 | "\n", 352 | "\n", 353 | " \n", 354 | " \n", 355 | " \n", 356 | " \n", 357 | " \n", 358 | " \n", 359 | " \n", 360 | " \n", 361 | " \n", 362 | " \n", 363 | " \n", 364 | " \n", 365 | " \n", 366 | " \n", 367 | " \n", 368 | " \n", 369 | " \n", 370 | " \n", 371 | " \n", 372 | " \n", 373 | " \n", 374 | " \n", 375 | " \n", 376 | " \n", 377 | " \n", 378 | " \n", 379 | " \n", 380 | " \n", 381 | " \n", 382 | " \n", 383 | " \n", 384 | " \n", 385 | " \n", 386 | " \n", 387 | " \n", 388 | " \n", 389 | " \n", 390 | " \n", 391 | " \n", 392 | " \n", 393 | " \n", 394 | " \n", 395 | " \n", 396 | " \n", 397 | " \n", 398 | " \n", 399 | " \n", 400 | " \n", 401 | " \n", 402 | " \n", 403 | " \n", 404 | " \n", 405 | " \n", 406 | " \n", 407 | " \n", 408 | " \n", 409 | " \n", 410 | " \n", 411 | " \n", 412 | " \n", 413 | " \n", 414 | " \n", 415 | " \n", 416 | " \n", 417 | " \n", 418 | " \n", 419 | " \n", 420 | " \n", 421 | " \n", 422 | " \n", 423 | " \n", 424 | " \n", 425 | " \n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | "
RouteMin Delay
14850122
27150122
27750623
29650625
29550425
29851127
29750428
29350533
30051133
31151133
30450435
30550440
28651242
29951166
29150674
23750477
23650193
\n", 448 | "
" 449 | ], 450 | "text/plain": [ 451 | " Route Min Delay\n", 452 | "148 501 22\n", 453 | "271 501 22\n", 454 | "277 506 23\n", 455 | "296 506 25\n", 456 | "295 504 25\n", 457 | "298 511 27\n", 458 | "297 504 28\n", 459 | "293 505 33\n", 460 | "300 511 33\n", 461 | "311 511 33\n", 462 | "304 504 35\n", 463 | "305 504 40\n", 464 | "286 512 42\n", 465 | "299 511 66\n", 466 | "291 506 74\n", 467 | "237 504 77\n", 468 | "236 501 93" 469 | ] 470 | }, 471 | "execution_count": 34, 472 | "metadata": {}, 473 | "output_type": "execute_result" 474 | } 475 | ], 476 | "source": [ 477 | "# ORDER BY\n", 478 | "# select \"Route\", \"Min Delay\" from streetcarjan2014 where \"Min Delay\" > 20 order by \"Min Delay\"\n", 479 | "streetcarjan2014[['Route','Min Delay']][(streetcarjan2014['Min Delay'] > 20)].sort_values('Min Delay')\n" 480 | ] 481 | }, 482 | { 483 | "cell_type": "code", 484 | "execution_count": null, 485 | "metadata": {}, 486 | "outputs": [], 487 | "source": [] 488 | } 489 | ], 490 | "metadata": { 491 | "kernelspec": { 492 | "display_name": "Python 3", 493 | "language": "python", 494 | "name": "python3" 495 | }, 496 | "language_info": { 497 | "codemirror_mode": { 498 | "name": "ipython", 499 | "version": 3 500 | }, 501 | "file_extension": ".py", 502 | "mimetype": "text/x-python", 503 | "name": "python", 504 | "nbconvert_exporter": "python", 505 | "pygments_lexer": "ipython3", 506 | "version": "3.7.4" 507 | } 508 | }, 509 | "nbformat": 4, 510 | "nbformat_minor": 2 511 | } 512 | -------------------------------------------------------------------------------- /notebooks/custom_classes.py: -------------------------------------------------------------------------------- 1 | from sklearn.model_selection import train_test_split 2 | from sklearn.metrics import accuracy_score 3 | from sklearn.metrics import average_precision_score 4 | from sklearn.pipeline import Pipeline 5 | from sklearn.base import TransformerMixin 6 | from sklearn.base import BaseEstimator 7 | from sklearn.preprocessing import LabelEncoder, MinMaxScaler, StandardScaler 8 | 9 | 10 | 11 | 12 | 13 | class encode_categorical(BaseEstimator, TransformerMixin): 14 | '''encode categorical columns''' 15 | def __init__(self): 16 | self.le = {} 17 | self.max_dict = {} 18 | return None 19 | 20 | def set_params(self, **kwargs): 21 | self.col_list = kwargs.get('col_list', None) 22 | return self 23 | 24 | 25 | 26 | def fit(self, X, y=None, **fit_params): 27 | for col in self.col_list: 28 | print("col is ",col) 29 | self.le[col] = LabelEncoder() 30 | self.le[col].fit(X[col].tolist()) 31 | return self 32 | 33 | 34 | def transform(self, X, y=None, **tranform_params): 35 | for col in self.col_list: 36 | print("transform col is ",col) 37 | X[col] = self.le[col].transform(X[col]) 38 | print("after transform col is ",col) 39 | self.max_dict[col] = X[col].max() +1 40 | # print(X.loc[[0]]) 41 | return X 42 | 43 | class prep_for_keras_input(BaseEstimator, TransformerMixin): 44 | '''prep columns for feeding Keras model''' 45 | def __init__(self): 46 | self.dictlist = [] 47 | return None 48 | 49 | def set_params(self, **kwargs): 50 | self.collist = kwargs.get('collist', None) 51 | self.continuouscols = kwargs.get('continuouscols', None) 52 | self.textcols = kwargs.get('textcols', None) 53 | return self 54 | 55 | 56 | 57 | def fit(self, X, y=None, **fit_params): 58 | return self 59 | 60 | 61 | def transform(self, X, y=None, **tranform_params): 62 | for col in self.collist: 63 | print("cat col is",col) 64 | self.dictlist.append(np.array(X[col])) 65 | for col in self.textcols: 66 | print("text col is",col) 67 | self.dictlist.append(pad_sequences(X[col], maxlen=max_dict[col])) 68 | for col in self.continuouscols: 69 | print("cont col is",col) 70 | self.dictlist.append(np.array(X[col])) 71 | # print(X.loc[[0]]) 72 | return self.dictlist 73 | 74 | class fill_empty(BaseEstimator, TransformerMixin): 75 | '''fill empty values with placeholders''' 76 | 77 | 78 | def set_params(self, **kwargs): 79 | self.collist = kwargs.get('collist', None) 80 | self.continuouscols = kwargs.get('continuouscols', None) 81 | self.textcols = kwargs.get('textcols', None) 82 | return self 83 | 84 | 85 | def transform(self, X, **tranform_params): 86 | print("fill empty xform") 87 | for col in self.collist: 88 | X[col].fillna(value="missing", inplace=True) 89 | for col in self.continuouscols: 90 | X[col].fillna(value=0.0,inplace=True) 91 | for col in self.textcols: 92 | X[col].fillna(value="missing", inplace=True) 93 | return X 94 | 95 | def fit(self, X, y=None, **fit_params): 96 | return self 97 | 98 | class encode_text(BaseEstimator, TransformerMixin): 99 | '''encode text columns''' 100 | def __init__(self): 101 | self.tok = {} 102 | return None 103 | 104 | def set_params(self, **kwargs): 105 | self.col_list = kwargs.get('col_list', None) 106 | return self 107 | 108 | 109 | 110 | def fit(self, X, y=None, **fit_params): 111 | for col in self.col_list: 112 | print("col is ",col) 113 | self.tok[col] = Tokenizer(num_words=maxwords,lower=True) 114 | self.tok[col].fit_on_texts(X[col]) 115 | return self 116 | 117 | 118 | def transform(self, X, y=None, **tranform_params): 119 | for col in self.col_list: 120 | print("transform col is ",col) 121 | X[col] = self.tok[col].texts_to_sequences(X[col]) 122 | print("after transform col is ",col) 123 | self.max_dict[col] = np.max(X[(X[col].map(len) != 0)][col].map(max)) 124 | if self.max_dict[cols] > textmax: 125 | textmax = self.max_dict[cols] 126 | # print(X.loc[[0]]) 127 | return X 128 | -------------------------------------------------------------------------------- /notebooks/keras_functional_api_mnist.py: -------------------------------------------------------------------------------- 1 | # minimal Kera functional API model for MNIST 2 | # adapted from https://www.tensorflow.org/guide/keras/functional 3 | 4 | #import libraries 5 | 6 | import numpy as np 7 | import tensorflow as tf 8 | from tensorflow import keras 9 | from tensorflow.keras import layers 10 | 11 | # define model layers 12 | 13 | inputs = keras.Input(shape=(784,)) 14 | flatten = layers.Flatten(input_shape=(28, 28)) 15 | flattened = flatten(inputs) 16 | dense = layers.Dense(128, activation='relu')(flattened) 17 | dropout = layers.Dropout(0.2) (dense) 18 | outputs = layers.Dense(10) (dropout) 19 | 20 | # define model inputs and outputs (taken from layer definition) 21 | 22 | model = keras.Model(inputs=inputs, outputs=outputs, name='mnist_model') 23 | (x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data() 24 | 25 | x_train = x_train.reshape(60000, 784).astype('float32') / 255 26 | x_test = x_test.reshape(10000, 784).astype('float32') / 255 27 | 28 | # compile model, including specifying the loss function, optimizer, and metrics 29 | 30 | model.compile(loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), 31 | optimizer=keras.optimizers.RMSprop(), 32 | metrics=['accuracy']) 33 | 34 | # train model 35 | 36 | history = model.fit(x_train, y_train, 37 | batch_size=64, 38 | epochs=5, 39 | validation_split=0.2) 40 | 41 | # assess model performance 42 | 43 | test_scores = model.evaluate(x_test, y_test, verbose=2) 44 | print('Test loss:', test_scores[0]) 45 | print('Test accuracy:', test_scores[1]) 46 | -------------------------------------------------------------------------------- /notebooks/keras_sequential_api_mnist.py: -------------------------------------------------------------------------------- 1 | # minimal Kera sequential API model for MNIST 2 | # adapted from https://github.com/tensorflow/docs/blob/master/site/en/tutorials/quickstart/beginner.ipynb 3 | 4 | #import libraries 5 | 6 | import tensorflow as tf 7 | import pydotplus 8 | from tensorflow.keras.utils import plot_model 9 | 10 | mnist = tf.keras.datasets.mnist 11 | 12 | # define model inputs 13 | 14 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 15 | x_train, x_test = x_train / 255.0, x_test / 255.0 16 | 17 | # define model layers 18 | 19 | model = tf.keras.models.Sequential([ 20 | tf.keras.layers.Flatten(input_shape=(28, 28)), 21 | tf.keras.layers.Dense(128, activation='relu'), 22 | tf.keras.layers.Dropout(0.2), 23 | tf.keras.layers.Dense(10) 24 | ]) 25 | 26 | # compile model, including specifying the loss function, optimizer, and metrics 27 | 28 | model.compile(optimizer='adam', 29 | loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), 30 | metrics=['accuracy']) 31 | 32 | # train model 33 | 34 | history = model.fit(x_train, y_train, 35 | batch_size=64, 36 | epochs=5, 37 | validation_split=0.2) 38 | 39 | # assess model performance 40 | 41 | test_scores = model.evaluate(x_test, y_test, verbose=2) 42 | print('Test loss:', test_scores[0]) 43 | print('Test accuracy:', test_scores[1]) 44 | -------------------------------------------------------------------------------- /notebooks/streetcar_data_preparation_config.yml: -------------------------------------------------------------------------------- 1 | general: 2 | load_from_scratch: False 3 | save_transformed_dataframe: True 4 | remove_bad_values: True 5 | file_names: 6 | pickled_input_dataframe: 2014_2019.pkl 7 | pickled_output_dataframe: 2014_2019_df_cleaned_remove_bad_values_may16_2020.pkl 8 | -------------------------------------------------------------------------------- /notebooks/streetcar_model_training_config.yml: -------------------------------------------------------------------------------- 1 | general: 2 | verboseout: True 3 | includetext: True # switch to determine whether text fields are included in model 4 | presaved: False # switch to determine whether to train model or load saved model 5 | savemodel: False # switch to determine whether to save model 6 | picklemodel: True # switch for code to allow pickling of Keras model 7 | save_model_plot: False 8 | tensorboard_callback: False 9 | hctextmax: 7000 10 | maxwords: 6000 11 | textmax: 50 12 | pickled_data_file: '20142018_0930.pkl' 13 | pickled_dataframe: '2014_2019_df_cleaned_remove_bad_values_may16_2020.pkl' 14 | route_direction_file: 'routedirection.csv' 15 | modifier: 'may16_2020' 16 | targetthresh: 6.0 # use as boundary between delay and no delay 17 | targetcontinuous: False # switch to determine whether target is continuous 18 | emptythresh: 6000 19 | zero_weight: 1.0 # class_weight = {0 : zero_weight, 1: one_weight} 20 | one_weight: 45.878 # default one_weight 21 | one_weight_offset: 0 # additional offset to one_weight 22 | patience_threshold: 15 23 | start_year: 2014 # date of start of refactored dataset 24 | start_month: 1 25 | start_day: 1 26 | end_year: 2019 # date of end of refactored dataset 27 | end_month: 12 28 | end_day: 31 29 | test_parms: 30 | testproportion: 0.2 # proportion of data reserved for test set 31 | trainproportion: 0.8 # proportion of non-test data dedicated to training (vs. validation) 32 | current_experiment: 5 33 | repeatable_run: False # switch to control whether runs are repeated identically 34 | get_test_train_acc: True # switch to control whether block to get test and train accuracy is after training 35 | hyperparameters: 36 | learning_rate: 0.001 37 | dropout_rate: 0.0003 #0.003 38 | l2_lambda: 0.0003 #7.5 39 | loss_func: "binary_crossentropy" 40 | output_activation: "hard_sigmoid" # set to "linear" if target is continuous 41 | batch_size: 1000 42 | epochs: 50 -------------------------------------------------------------------------------- /pipelines/sc_delay_pipeline_dec27b.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipeline_dec27b.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipeline_keras_prep_dec27b.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipeline_keras_prep_dec27b.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipelinedec27.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipelinedec27.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr11_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr11_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr11b_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr11b_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr11c_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr11c_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr12020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr12020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr12_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr12_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr12b_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr12b_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr16_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr16_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr17_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr17_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr18_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr18_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr19_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr19_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr22020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr22020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr26_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr26_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr2b2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr2b2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr2d2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr2d2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr52020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr52020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr5_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr5_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr5_bv_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr5_bv_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr5_keep_bad_directions_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr5_keep_bad_directions_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr6_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr6_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr9_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr9_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr9b_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr9b_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepapr9c_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepapr9c_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepmar26b2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepmar26b2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepmar272020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepmar272020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepmar302020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepmar302020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepmay16_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepmay16_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_pipleline_keras_prepxgb_apr9d_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_pipleline_keras_prepxgb_apr9d_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr11_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr11_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr11b_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr11b_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr11c_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr11c_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr12020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr12020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr12_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr12_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr12b_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr12b_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr16_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr16_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr17_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr17_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr18_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr18_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr19_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr19_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr22020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr22020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr26_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr26_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr2b2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr2b2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr2d2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr2d2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr52020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr52020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr5_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr5_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr5_bv_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr5_bv_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr5_keep_bad_directions_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr5_keep_bad_directions_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr6_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr6_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr9_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr9_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr9b_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr9b_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelineapr9c_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelineapr9c_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelinemar26b2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelinemar26b2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelinemar272020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelinemar272020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelinemar302020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelinemar302020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelinemay16_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelinemay16_2020.pkl -------------------------------------------------------------------------------- /pipelines/sc_delay_piplelinexgb_apr9d_2020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanmark1867/deep_learning_for_structured_data/ad5111b7c48d5393495a2243aa0dc467cdccf01a/pipelines/sc_delay_piplelinexgb_apr9d_2020.pkl -------------------------------------------------------------------------------- /sql/streetcarjan2014.sql: -------------------------------------------------------------------------------- 1 | -- SQL (suitable for Postgres) to create table that can be loaded with "Streetcar Jan 2014.csv" 2 | 3 | CREATE TABLE public.streetcarjan2014 4 | ( 5 | "Report Date" character varying(10), 6 | "Route" character varying(10), 7 | "Time" timestamp without time zone, 8 | "Day" character varying(10), 9 | "Location " character varying(200), 10 | "Incident" character varying(80), 11 | "Min Delay" integer, 12 | "Min Gap " integer, 13 | "Direction" character varying(6), 14 | "Vehicle" character varying(10) 15 | ) 16 | WITH ( 17 | OIDS = FALSE 18 | ) 19 | TABLESPACE pg_default; 20 | 21 | ALTER TABLE public.streetcarjan2014 22 | OWNER to postgres; --------------------------------------------------------------------------------