├── lib ├── __init__.py ├── metrics.py ├── metrics_test.py ├── AMSGrad.py └── utils.py ├── model ├── __init__.py ├── prnn_model.py ├── prnn_cell.py ├── prnn_supervisor.py └── utils_tf.py ├── scripts ├── __init__.py ├── gen_adj_mx.py ├── generate_training_data.py └── eval_baseline_methods.py ├── ICML_final_overall_architecture.png ├── .idea ├── misc.xml ├── vcs.xml ├── modules.xml └── rnn_flow_lab.iml ├── result_pretrained ├── gru_mae_predictions_pemsd4__pretrained_pemsd4_gru_mae__result.txt ├── gru_mae_predictions_pemsd7__pretrained_pemsd7_gru_mae__result.txt ├── gru_mae_predictions_pemsd8__pretrained_pemsd8_gru_mae__result.txt ├── gru_nll_predictions_pemsd4__pretrained_pemsd4_gru_nll__result.txt ├── gru_nll_predictions_pemsd7__pretrained_pemsd7_gru_nll__result.txt ├── gru_nll_predictions_pemsd8__pretrained_pemsd8_gru_nll__result.txt ├── agcgru_mae_predictions_pemsd4__pretrained_pemsd4_agcgru_mae__result.txt ├── agcgru_mae_predictions_pemsd7__pretrained_pemsd7_agcgru_mae__result.txt ├── agcgru_mae_predictions_pemsd8__pretrained_pemsd8_agcgru_mae__result.txt ├── agcgru_nll_predictions_pemsd4__pretrained_pemsd4_agcgru_nll__result.txt ├── agcgru_nll_predictions_pemsd7__pretrained_pemsd7_agcgru_nll__result.txt ├── agcgru_nll_predictions_pemsd8__pretrained_pemsd8_agcgru_nll__result.txt ├── dcgru_mae_predictions_pemsd4__pretrained_pemsd4_dcgru_mae__result.txt ├── dcgru_mae_predictions_pemsd7__pretrained_pemsd7_dcgru_mae__result.txt ├── dcgru_mae_predictions_pemsd8__pretrained_pemsd8_dcgru_mae__result.txt ├── dcgru_nll_predictions_pemsd4__pretrained_pemsd4_dcgru_nll__result.txt ├── dcgru_nll_predictions_pemsd7__pretrained_pemsd7_dcgru_nll__result.txt ├── dcgru_nll_predictions_pemsd8__pretrained_pemsd8_dcgru_nll__result.txt ├── gru_mae_predictions_pemsd3__pretrained_pemsd3_gru_mae__result.txt ├── gru_nll_predictions_pemsd3__pretrained_pemsd3_gru_nll__result.txt ├── dcgru_mae_predictions_pemsd3__pretrained_pemsd3_dcgru_mae__result.txt ├── dcgru_nll_predictions_pemsd3__pretrained_pemsd3_dcgru_nll__result.txt ├── agcgru_mae_predictions_pemsd3__pretrained_pemsd3_agcgru_mae__result.txt └── agcgru_nll_predictions_pemsd3__pretrained_pemsd3_agcgru_nll__result.txt ├── README.MD ├── inference_for_dir.py ├── training.py ├── eval_metrics ├── metrics.py ├── metrics_sample.py ├── utils.py └── crps.py └── inference.py /lib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scripts/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ICML_final_overall_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/networkslab/rnn_flow/HEAD/ICML_final_overall_architecture.png -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/rnn_flow_lab.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 11 | -------------------------------------------------------------------------------- /result_pretrained/gru_mae_predictions_pemsd4__pretrained_pemsd4_gru_mae__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: gru Cost: mae Dataset: pemsd4 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 1.37 & 2.70 & 2.95 5 | Horizon-5 & 1.76 & 3.74 & 4.05 6 | Horizon-8 & 2.02 & 4.53 & 4.74 7 | Horizon-11 & 2.23 & 5.15 & 5.23------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 1.34 & 2.25 & 1.87 10 | Horizon-5& 1.72 & 2.95 & 2.33 11 | Horizon-8& 1.97 & 3.48 & 2.59 12 | Horizon-11& 2.18 & 3.92 & 2.79 -------------------------------------------------------------------------------- /result_pretrained/gru_mae_predictions_pemsd7__pretrained_pemsd7_gru_mae__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: gru Cost: mae Dataset: pemsd7 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 2.24 & 5.27 & 4.28 5 | Horizon-5 & 3.02 & 7.59 & 5.96 6 | Horizon-8 & 3.55 & 9.30 & 7.00 7 | Horizon-11 & 3.96 & 10.61 & 7.73------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 2.18 & 4.04 & 3.29 10 | Horizon-5& 2.94 & 5.65 & 4.24 11 | Horizon-8& 3.46 & 6.90 & 4.73 12 | Horizon-11& 3.86 & 7.81 & 5.16 -------------------------------------------------------------------------------- /result_pretrained/gru_mae_predictions_pemsd8__pretrained_pemsd8_gru_mae__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: gru Cost: mae Dataset: pemsd8 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 1.12 & 2.17 & 2.55 5 | Horizon-5 & 1.41 & 2.94 & 3.47 6 | Horizon-8 & 1.59 & 3.50 & 4.02 7 | Horizon-11 & 1.74 & 3.92 & 4.40------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 1.09 & 1.86 & 1.48 10 | Horizon-5& 1.37 & 2.42 & 1.77 11 | Horizon-8& 1.55 & 2.80 & 1.94 12 | Horizon-11& 1.69 & 3.11 & 2.08 -------------------------------------------------------------------------------- /result_pretrained/gru_nll_predictions_pemsd4__pretrained_pemsd4_gru_nll__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: gru Cost: nll Dataset: pemsd4 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 1.53 & 3.05 & 3.10 5 | Horizon-5 & 2.00 & 4.24 & 4.20 6 | Horizon-8 & 2.32 & 5.10 & 4.88 7 | Horizon-11 & 2.59 & 5.80 & 5.37------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 1.14 & 1.36 & 1.11 10 | Horizon-5& 1.50 & 1.87 & 1.43 11 | Horizon-8& 1.75 & 2.24 & 1.63 12 | Horizon-11& 1.95 & 2.56 & 1.78 -------------------------------------------------------------------------------- /result_pretrained/gru_nll_predictions_pemsd7__pretrained_pemsd7_gru_nll__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: gru Cost: nll Dataset: pemsd7 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 2.50 & 5.89 & 4.45 5 | Horizon-5 & 3.46 & 8.48 & 6.18 6 | Horizon-8 & 4.13 & 10.35 & 7.23 7 | Horizon-11 & 4.63 & 11.79 & 7.95------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 1.88 & 2.50 & 2.02 10 | Horizon-5& 2.61 & 3.57 & 2.74 11 | Horizon-8& 3.10 & 4.29 & 3.16 12 | Horizon-11& 3.46 & 4.84 & 3.44 -------------------------------------------------------------------------------- /result_pretrained/gru_nll_predictions_pemsd8__pretrained_pemsd8_gru_nll__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: gru Cost: nll Dataset: pemsd8 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 1.28 & 2.55 & 2.71 5 | Horizon-5 & 1.64 & 3.45 & 3.61 6 | Horizon-8 & 1.88 & 4.05 & 4.15 7 | Horizon-11 & 2.07 & 4.51 & 4.54------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 0.95 & 1.12 & 0.94 10 | Horizon-5& 1.23 & 1.52 & 1.18 11 | Horizon-8& 1.42 & 1.80 & 1.33 12 | Horizon-11& 1.57 & 2.04 & 1.44 -------------------------------------------------------------------------------- /result_pretrained/agcgru_mae_predictions_pemsd4__pretrained_pemsd4_agcgru_mae__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: agcgru Cost: mae Dataset: pemsd4 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 1.35 & 2.67 & 2.88 5 | Horizon-5 & 1.63 & 3.44 & 3.77 6 | Horizon-8 & 1.78 & 3.87 & 4.20 7 | Horizon-11 & 1.88 & 4.17 & 4.46------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 1.29 & 2.15 & 1.77 10 | Horizon-5& 1.56 & 2.64 & 2.11 11 | Horizon-8& 1.71 & 2.92 & 2.29 12 | Horizon-11& 1.81 & 3.12 & 2.39 -------------------------------------------------------------------------------- /result_pretrained/agcgru_mae_predictions_pemsd7__pretrained_pemsd7_agcgru_mae__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: agcgru Cost: mae Dataset: pemsd7 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 2.15 & 5.13 & 4.11 5 | Horizon-5 & 2.71 & 6.75 & 5.46 6 | Horizon-8 & 2.99 & 7.62 & 6.12 7 | Horizon-11 & 3.19 & 8.18 & 6.54------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 2.08 & 3.79 & 3.13 10 | Horizon-5& 2.61 & 4.85 & 3.85 11 | Horizon-8& 2.89 & 5.41 & 4.22 12 | Horizon-11& 3.08 & 5.81 & 4.48 -------------------------------------------------------------------------------- /result_pretrained/agcgru_mae_predictions_pemsd8__pretrained_pemsd8_agcgru_mae__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: agcgru Cost: mae Dataset: pemsd8 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 1.13 & 2.30 & 2.59 5 | Horizon-5 & 1.37 & 3.02 & 3.45 6 | Horizon-8 & 1.49 & 3.40 & 3.85 7 | Horizon-11 & 1.57 & 3.65 & 4.10------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 1.08 & 1.87 & 1.41 10 | Horizon-5& 1.32 & 2.35 & 1.63 11 | Horizon-8& 1.43 & 2.61 & 1.74 12 | Horizon-11& 1.51 & 2.79 & 1.79 -------------------------------------------------------------------------------- /result_pretrained/agcgru_nll_predictions_pemsd4__pretrained_pemsd4_agcgru_nll__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: agcgru Cost: nll Dataset: pemsd4 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 1.45 & 2.92 & 2.98 5 | Horizon-5 & 1.77 & 3.78 & 3.84 6 | Horizon-8 & 1.96 & 4.33 & 4.32 7 | Horizon-11 & 2.08 & 4.72 & 4.61------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 1.08 & 1.28 & 1.05 10 | Horizon-5& 1.32 & 1.62 & 1.26 11 | Horizon-8& 1.46 & 1.82 & 1.37 12 | Horizon-11& 1.56 & 1.97 & 1.45 -------------------------------------------------------------------------------- /result_pretrained/agcgru_nll_predictions_pemsd7__pretrained_pemsd7_agcgru_nll__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: agcgru Cost: nll Dataset: pemsd7 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 2.32 & 5.51 & 4.22 5 | Horizon-5 & 2.92 & 7.22 & 5.49 6 | Horizon-8 & 3.24 & 8.18 & 6.12 7 | Horizon-11 & 3.45 & 8.78 & 6.49------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 1.73 & 2.27 & 1.83 10 | Horizon-5& 2.19 & 2.98 & 2.25 11 | Horizon-8& 2.43 & 3.36 & 2.48 12 | Horizon-11& 2.59 & 3.60 & 2.62 -------------------------------------------------------------------------------- /result_pretrained/agcgru_nll_predictions_pemsd8__pretrained_pemsd8_agcgru_nll__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: agcgru Cost: nll Dataset: pemsd8 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 1.21 & 2.47 & 2.69 5 | Horizon-5 & 1.45 & 3.12 & 3.47 6 | Horizon-8 & 1.59 & 3.51 & 3.88 7 | Horizon-11 & 1.68 & 3.77 & 4.15------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 0.91 & 1.10 & 0.88 10 | Horizon-5& 1.10 & 1.43 & 1.01 11 | Horizon-8& 1.20 & 1.61 & 1.09 12 | Horizon-11& 1.28 & 1.73 & 1.14 -------------------------------------------------------------------------------- /result_pretrained/dcgru_mae_predictions_pemsd4__pretrained_pemsd4_dcgru_mae__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: dcgru Cost: mae Dataset: pemsd4 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 1.38 & 2.72 & 2.93 5 | Horizon-5 & 1.71 & 3.64 & 3.93 6 | Horizon-8 & 1.92 & 4.23 & 4.49 7 | Horizon-11 & 2.08 & 4.67 & 4.87------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 1.33 & 2.22 & 1.85 10 | Horizon-5& 1.66 & 2.82 & 2.23 11 | Horizon-8& 1.86 & 3.19 & 2.46 12 | Horizon-11& 2.01 & 3.49 & 2.64 -------------------------------------------------------------------------------- /result_pretrained/dcgru_mae_predictions_pemsd7__pretrained_pemsd7_dcgru_mae__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: dcgru Cost: mae Dataset: pemsd7 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 2.19 & 5.16 & 4.16 5 | Horizon-5 & 2.87 & 7.18 & 5.67 6 | Horizon-8 & 3.29 & 8.48 & 6.55 7 | Horizon-11 & 3.61 & 9.42 & 7.14------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 2.14 & 3.92 & 3.26 10 | Horizon-5& 2.79 & 5.32 & 4.06 11 | Horizon-8& 3.20 & 6.19 & 4.53 12 | Horizon-11& 3.50 & 6.84 & 4.90 -------------------------------------------------------------------------------- /result_pretrained/dcgru_mae_predictions_pemsd8__pretrained_pemsd8_dcgru_mae__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: dcgru Cost: mae Dataset: pemsd8 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 1.17 & 2.35 & 2.64 5 | Horizon-5 & 1.44 & 3.12 & 3.54 6 | Horizon-8 & 1.58 & 3.57 & 4.00 7 | Horizon-11 & 1.70 & 3.87 & 4.28------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 1.13 & 1.94 & 1.50 10 | Horizon-5& 1.39 & 2.49 & 1.75 11 | Horizon-8& 1.53 & 2.80 & 1.88 12 | Horizon-11& 1.64 & 3.04 & 1.97 -------------------------------------------------------------------------------- /result_pretrained/dcgru_nll_predictions_pemsd4__pretrained_pemsd4_dcgru_nll__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: dcgru Cost: nll Dataset: pemsd4 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 1.52 & 3.02 & 3.09 5 | Horizon-5 & 1.91 & 4.11 & 4.10 6 | Horizon-8 & 2.18 & 4.88 & 4.72 7 | Horizon-11 & 2.40 & 5.48 & 5.16------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 1.13 & 1.33 & 1.10 10 | Horizon-5& 1.43 & 1.75 & 1.34 11 | Horizon-8& 1.63 & 2.05 & 1.50 12 | Horizon-11& 1.79 & 2.30 & 1.61 -------------------------------------------------------------------------------- /result_pretrained/dcgru_nll_predictions_pemsd7__pretrained_pemsd7_dcgru_nll__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: dcgru Cost: nll Dataset: pemsd7 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 2.48 & 5.87 & 4.39 5 | Horizon-5 & 3.37 & 8.42 & 6.01 6 | Horizon-8 & 3.97 & 10.13 & 6.97 7 | Horizon-11 & 4.43 & 11.43 & 7.63------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 1.85 & 2.41 & 2.00 10 | Horizon-5& 2.51 & 3.35 & 2.62 11 | Horizon-8& 2.95 & 3.97 & 3.00 12 | Horizon-11& 3.27 & 4.42 & 3.29 -------------------------------------------------------------------------------- /result_pretrained/dcgru_nll_predictions_pemsd8__pretrained_pemsd8_dcgru_nll__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: dcgru Cost: nll Dataset: pemsd8 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 1.27 & 2.48 & 2.69 5 | Horizon-5 & 1.59 & 3.31 & 3.55 6 | Horizon-8 & 1.80 & 3.90 & 4.08 7 | Horizon-11 & 1.98 & 4.33 & 4.44------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 0.94 & 1.10 & 0.93 10 | Horizon-5& 1.18 & 1.43 & 1.13 11 | Horizon-8& 1.35 & 1.67 & 1.25 12 | Horizon-11& 1.48 & 1.86 & 1.34 -------------------------------------------------------------------------------- /result_pretrained/gru_mae_predictions_pemsd3__pretrained_pemsd3_gru_mae__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: gru Cost: mae Dataset: pemsd3 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 14.40 & 14.57 & 23.07 5 | Horizon-5 & 16.10 & 15.98 & 26.18 6 | Horizon-8 & 17.63 & 17.33 & 28.65 7 | Horizon-11 & 19.18 & 18.89 & 30.99------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 13.95 & 7.88 & 7.72 10 | Horizon-5& 15.60 & 8.81 & 8.67 11 | Horizon-8& 17.10 & 9.69 & 9.48 12 | Horizon-11& 18.60 & 10.49 & 10.38 -------------------------------------------------------------------------------- /result_pretrained/gru_nll_predictions_pemsd3__pretrained_pemsd3_gru_nll__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: gru Cost: nll Dataset: pemsd3 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 15.47 & 15.87 & 24.85 5 | Horizon-5 & 17.47 & 17.43 & 28.72 6 | Horizon-8 & 19.24 & 18.91 & 31.59 7 | Horizon-11 & 20.97 & 20.68 & 34.20------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 11.24 & 4.19 & 4.33 10 | Horizon-5& 12.70 & 4.71 & 4.93 11 | Horizon-8& 13.99 & 5.14 & 5.48 12 | Horizon-11& 15.25 & 5.56 & 6.04 -------------------------------------------------------------------------------- /result_pretrained/dcgru_mae_predictions_pemsd3__pretrained_pemsd3_dcgru_mae__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: dcgru Cost: mae Dataset: pemsd3 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 14.48 & 15.07 & 23.71 5 | Horizon-5 & 15.67 & 16.07 & 26.04 6 | Horizon-8 & 16.52 & 16.91 & 27.57 7 | Horizon-11 & 17.36 & 17.84 & 28.81------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 14.06 & 8.47 & 7.30 10 | Horizon-5& 15.22 & 9.39 & 7.69 11 | Horizon-8& 16.05 & 9.94 & 8.08 12 | Horizon-11& 16.86 & 10.46 & 8.48 -------------------------------------------------------------------------------- /result_pretrained/dcgru_nll_predictions_pemsd3__pretrained_pemsd3_dcgru_nll__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: dcgru Cost: nll Dataset: pemsd3 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 15.32 & 15.89 & 24.61 5 | Horizon-5 & 16.56 & 16.95 & 26.88 6 | Horizon-8 & 17.53 & 17.86 & 28.36 7 | Horizon-11 & 18.56 & 18.92 & 30.38------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 11.20 & 4.28 & 4.30 10 | Horizon-5& 12.14 & 4.69 & 4.67 11 | Horizon-8& 12.87 & 4.99 & 4.97 12 | Horizon-11& 13.64 & 5.28 & 5.31 -------------------------------------------------------------------------------- /result_pretrained/agcgru_mae_predictions_pemsd3__pretrained_pemsd3_agcgru_mae__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: agcgru Cost: mae Dataset: pemsd3 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 13.79 & 14.01 & 22.06 5 | Horizon-5 & 14.84 & 14.74 & 24.22 6 | Horizon-8 & 15.58 & 15.34 & 25.56 7 | Horizon-11 & 16.06 & 15.80 & 26.45------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 13.38 & 7.97 & 7.02 10 | Horizon-5& 14.45 & 8.77 & 7.47 11 | Horizon-8& 15.19 & 9.34 & 7.76 12 | Horizon-11& 15.67 & 9.67 & 8.00 -------------------------------------------------------------------------------- /result_pretrained/agcgru_nll_predictions_pemsd3__pretrained_pemsd3_agcgru_nll__result.txt: -------------------------------------------------------------------------------- 1 | Rnn_type: agcgru Cost: nll Dataset: pemsd3 2 | ------------------------------- 3 | Metrics & MAE & MAPE & RMSE 4 | Horizon-2 & 14.41 & 14.92 & 23.93 5 | Horizon-5 & 15.52 & 15.79 & 25.98 6 | Horizon-8 & 16.33 & 16.50 & 27.64 7 | Horizon-11 & 16.88 & 17.01 & 28.85------------------------------- 8 | Metrics & CRPS & P10QL & P90QL 9 | Horizon-2& 10.53 & 4.02 & 4.06 10 | Horizon-5& 11.39 & 4.44 & 4.38 11 | Horizon-8& 12.04 & 4.76 & 4.63 12 | Horizon-11& 12.48 & 4.97 & 4.82 -------------------------------------------------------------------------------- /scripts/gen_adj_mx.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import argparse 6 | import numpy as np 7 | import pandas as pd 8 | import pickle 9 | 10 | 11 | def get_adjacency_matrix(distance_df, sensor_ids, normalized_k=0.1): 12 | """ 13 | 14 | :param distance_df: data frame with three columns: [from, to, distance]. 15 | :param sensor_ids: list of sensor ids. 16 | :param normalized_k: entries that become lower than normalized_k after normalization are set to zero for sparsity. 17 | :return: 18 | """ 19 | num_sensors = len(sensor_ids) 20 | dist_mx = np.zeros((num_sensors, num_sensors), dtype=np.float32) 21 | dist_mx[:] = np.inf 22 | # Builds sensor id to index map. 23 | sensor_id_to_ind = {} 24 | for i, sensor_id in enumerate(sensor_ids): 25 | sensor_id_to_ind[sensor_id] = i 26 | 27 | # Fills cells in the matrix with distances. 28 | for row in distance_df.values: 29 | if row[0] not in sensor_id_to_ind or row[1] not in sensor_id_to_ind: 30 | continue 31 | dist_mx[sensor_id_to_ind[row[0]], sensor_id_to_ind[row[1]]] = row[2] 32 | 33 | # Calculates the standard deviation as theta. 34 | distances = dist_mx[~np.isinf(dist_mx)].flatten() 35 | std = distances.std() 36 | adj_mx = np.exp(-np.square(dist_mx / std)) 37 | # Make the adjacent matrix symmetric by taking the max. 38 | # adj_mx = np.maximum.reduce([adj_mx, adj_mx.T]) 39 | 40 | # Sets entries that lower than a threshold, i.e., k, to zero for sparsity. 41 | adj_mx[adj_mx < normalized_k] = 0 42 | return sensor_ids, sensor_id_to_ind, adj_mx 43 | 44 | 45 | if __name__ == '__main__': 46 | parser = argparse.ArgumentParser() 47 | parser.add_argument('--sensor_ids_filename', type=str, default='data/sensor_graph/graph_sensor_ids.txt', 48 | help='File containing sensor ids separated by comma.') 49 | parser.add_argument('--distances_filename', type=str, default='data/sensor_graph/distances_la_2012.csv', 50 | help='CSV file containing sensor distances with three columns: [from, to, distance].') 51 | parser.add_argument('--normalized_k', type=float, default=0.1, 52 | help='Entries that become lower than normalized_k after normalization are set to zero for sparsity.') 53 | parser.add_argument('--output_pkl_filename', type=str, default='data/sensor_graph/adj_mat.pkl', 54 | help='Path of the output file.') 55 | args = parser.parse_args() 56 | 57 | with open(args.sensor_ids_filename) as f: 58 | sensor_ids = f.read().strip().split(',') 59 | distance_df = pd.read_csv(args.distances_filename, dtype={'from': 'str', 'to': 'str'}) 60 | normalized_k = args.normalized_k 61 | _, sensor_id_to_ind, adj_mx = get_adjacency_matrix(distance_df, sensor_ids, normalized_k) 62 | # Save to pickle file. 63 | with open(args.output_pkl_filename, 'wb') as f: 64 | pickle.dump([sensor_ids, sensor_id_to_ind, adj_mx], f, protocol=2) 65 | -------------------------------------------------------------------------------- /README.MD: -------------------------------------------------------------------------------- 1 | # RNN with Particle Flow for Probabilistic Spatio-temporal Forecasting 2 | 3 | This is an implementation of the methodology in "RNN with Particle Flow for Probabilistic Spatio-temporal Forecasting". The arXiv version is available at [https://arxiv.org/pdf/2106.06064.pdf](https://arxiv.org/pdf/2106.06064.pdf) 4 | 5 | ![](ICML_final_overall_architecture.png) 6 | 7 | You can 8 | - reproduce the results for AGCGRU+flow, DCGRU+flow, and GRU+flow algorithms in our paper and supplementary material with the checkpoints of our trained models. 9 | - re-train the models with the same hyperparameter setting and evaluate the forecasts. 10 | 11 |   12 |   13 |   14 | 15 | --- 16 | ## To reproduce the results in the paper with the pre-trained checkpoints 17 | 18 | 1. download the preprocessed datasets and checkpoints from [https://drive.google.com/file/d/1tnaTTBVxUOdSpc5hxYUpCqUriYVVr5BP/view?usp=sharing](https://drive.google.com/file/d/1tnaTTBVxUOdSpc5hxYUpCqUriYVVr5BP/view?usp=sharing) 19 | 2. unzip the downloaded zip file inside the folder ```rnn_flow``` 20 | 3. run script ```python inference_for_dir.py --search_dir ckpt/pretrained --output_dir ./result_pretrained --gpu_id 0``` 21 | 22 |   23 | 24 | Then the results will be saved in the ```./result_pretrained```. 25 | 26 | 27 |   28 |   29 |   30 | 31 | ## To train the model 32 | 1. download the preprocessed datasets and checkpoints and decompress the downloaded zip file inside the folder ```rnn_flow``` 33 | 2. train model with script (e.g.) : ```python training.py --dataset PEMS07 --rnn_type agcgru --cost mae --gpu_id 0 --trial_id --ckpt_dir ./ckpt --data_dir ./data ``` 34 | 3. evaluate with script: ```python inference.py --dataset PEMS07 --rnn_type agcgru --cost mae --gpu_id 0 --trial_id 0 --ckpt_dir ./ckpt --output_dir ./result``` 35 |   36 | 37 | Note that: 38 | - the setting of ```--dataset```, ```--rnn_type```, ```--cost```, ```--trial_id``` and ```--ckpt_dir``` should be consistent in training and evaluation. 39 | - ```--trial_id``` controls the random seed; by changing it, we can get multiple-trials of results with different random seeds. 40 | 41 |   42 | 43 | 44 | ### Settings 45 | ```--rnn_type```: 46 | - ```agcgru``` 47 | - ```dcgru``` 48 | - ```gru``` 49 | 50 | ```--cost```: 51 | - ```mae``` (for MAE, MAPE and RMSE computation) 52 | - ```nll``` (for CRPS, P10QL and P90QL computation) 53 | 54 |   55 |   56 | 57 | ### Other Settings 58 | The adjacency matrices of the graphs of the datasets are stored in ```./data/sensor_graph``` (```--graph_dir```). 59 | 60 | The model configs are stored in ```./data/model_config``` (```--config_dir```) 61 | 62 | If the folders are changed, please make the arguments of ```train.py``` and ```inference.py``` consistent. 63 | 64 |   65 |   66 |   67 | 68 | ---- 69 | 70 | 71 | ## Requirements 72 | The code has been tested with ```python 3.7.0``` with the following packages installed (along with their dependencies): 73 | - ```tensorflow-gpu==1.15.0``` 74 | - ```numpy==1.19.2``` 75 | - ```networkx==2.5``` 76 | - ```pandas==1.1.3``` 77 | - ```scikit-learn==0.23.2``` 78 | - ```PyYAML==5.3.1``` 79 | - ```scipy==1.5.4``` 80 | 81 | In addition, for running with gpu, the code has been tested with ```CUDA 10.0``` and ```cuDNN 7.4```. 82 | 83 | ## Cite 84 | 85 | Please cite our paper if you use this code in your own work: 86 | 87 | ``` 88 | @inproceedings{pal2021, 89 | author={S. Pal and L. Ma and Y. Zhang and M. Coates}, 90 | booktitle={Proc. Int. Conf. Machine Learning}, 91 | title={{RNN} with Particle Flow for Probabilistic Spatio-temporal Forecasting}, 92 | month={Jul.}, 93 | year={2021}, 94 | address={Virtual Conference}} 95 | ``` 96 | -------------------------------------------------------------------------------- /inference_for_dir.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import os 4 | from os.path import join, exists 5 | import sys 6 | import glob 7 | from eval_metrics.metrics import eval_deter 8 | from eval_metrics.metrics_sample import eval_samples 9 | from inference import run_inference 10 | import copy 11 | 12 | 13 | 14 | if __name__ == '__main__': 15 | sys.path.append(os.getcwd()) 16 | parser = argparse.ArgumentParser() 17 | # ----------- args for load checkpoint of trained models 18 | parser.add_argument("--search_dir", default="ckpt/pretrained") 19 | parser.add_argument('--pretrained_dir', default=None, type=str, 20 | help='the config dir for trained model; if set as None, will be automatically set based on {ckpt_dir},{dataset},{trial_id}') 21 | parser.add_argument("--config_filename", default=None, type=str, help="The config file to used; if None, automatically find the latest checkpoint under {config_dir}") 22 | # ----------- if config_filename and pretrained_dir are not set; please ensure the following setting the same as the 'training.py' in order to evaluate the model on test set. 23 | parser.add_argument("--rnn_type", type=str, default=None, help="The type of rnn architecture of rnn_flow; if None, following the setting in the config file") 24 | parser.add_argument("--cost", type=str, default=None, help="The type of loss function (e.g., [mae], [nll]); if None, following the setting of the config file") 25 | parser.add_argument("--dataset", default="PEMSD8", help="name of datasets") 26 | parser.add_argument("--ckpt_dir", default="./ckpt", help="the dir to store checkpoints") 27 | parser.add_argument("--trial_id", type=int, default=123, help="id of the trial. Used as the random seed in multiple trials training") 28 | # ----------- the args to load data, please keep consistent with 'training.py' 29 | parser.add_argument("--data_dir", default="./data", help="the dir storing dataset") 30 | parser.add_argument("--graph_dir", default=None, help="the dir storing the graph information; if None, will be set as the '{data_dir}/sensor_graph'.") 31 | parser.add_argument("--output_dir", type=str, default="./result", help="The dir to store the output result") 32 | parser.add_argument('--output_filename', default=None, help="the name of output file; if None, automatically set as p{rnn_type}_{cost}_prediction_{dataset}_trial_{trial_id}.npz ") 33 | # ----------- 34 | parser.add_argument('--gpu_id', default=0, type=int, help='GPU id to use; by default using 0') 35 | parser.add_argument('--use_cpu_only', action="store_true", help='Add this if want to train in cpu only') 36 | args = parser.parse_args() 37 | 38 | if not args.use_cpu_only: 39 | os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu_id)# the GPU number 40 | else: 41 | os.environ["CUDA_VISIBLE_DEVICES"] = str(-1)# the GPU number 42 | 43 | if args.graph_dir is None: 44 | args.graph_dir = join(args.data_dir, "sensor_graph") 45 | 46 | args.dataset = args.dataset.lower() 47 | 48 | dirs = glob.glob(args.search_dir + "/*/*/") 49 | args.pre_set = True 50 | 51 | 52 | bad_files = [] 53 | bad_reasons = [] 54 | for d in dirs: 55 | args.pretrained_dir = d 56 | config = copy.deepcopy(args) 57 | print(d) 58 | try: 59 | run_inference(config) 60 | except Exception as e: 61 | print("\n-----------------------\n {} \n {} \n----------------------- ".format(d, e)) 62 | bad_files.append(d) 63 | bad_reasons.append(e) 64 | 65 | 66 | print("\n\n\n\n----------------------- bad_files ----------------------- ") 67 | if len(bad_files) == 0: 68 | print("no bad files") 69 | for i in range(len(bad_files)): 70 | print(bad_files[i]) 71 | print(bad_reasons[i]) 72 | print("-------") 73 | 74 | 75 | 76 | -------------------------------------------------------------------------------- /training.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import os 6 | import argparse 7 | from os.path import join, exists 8 | import numpy as np 9 | 10 | if __name__ == '__main__': 11 | 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument("--dataset", default="PEMSD8", help="name of datasets") 14 | parser.add_argument("--data_dir", default="./data", help="the dir storing dataset") 15 | parser.add_argument("--ckpt_dir", default="./ckpt", help="the dir to store checkpoints") 16 | parser.add_argument("--graph_dir", default=None, help="the dir storing the graph information; if None, will be set as the '{data_dir}/sensor_graph'.") 17 | parser.add_argument('--config_dir', default=None, type=str, 18 | help="The dir storing the detailed configuration of model (including hyperparameters); if None, will be set as the '{data_dir}/model_config'.") 19 | parser.add_argument('--gpu_id', default=0, type=int, help='GPU id to use; by default using 0') 20 | parser.add_argument('--use_cpu_only', action="store_true", help='Add this if want to train in cpu only') 21 | parser.add_argument("--trial_id", type=int, default=1, help="id of the trial. Used as the random seed in multiple trials training") 22 | # -------------- setting of the models: 23 | # - recommend to set this two hyperparmeters here; which is essential to find the saved checkpoitns in 'inference.py' 24 | parser.add_argument("--rnn_type", type=str, default=None, help="The type of rnn architecture of rnn_flow; if None, following the setting in the config file") 25 | parser.add_argument("--cost", type=str, default=None, help="The type of loss function (e.g., [mae], [nll]); if None, following the setting of the config file") 26 | args = parser.parse_args() 27 | 28 | if args.graph_dir is None: 29 | args.graph_dir = join(args.data_dir, "sensor_graph") 30 | 31 | if args.config_dir is None: 32 | args.config_dir = join(args.data_dir, "model_config") 33 | 34 | 35 | if args.use_cpu_only: 36 | os.environ["CUDA_VISIBLE_DEVICES"] = str(-1) 37 | else: 38 | os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu_id) 39 | 40 | if args.graph_dir is None: 41 | args.graph_dir = join(args.data_dir, "sensor_graph") 42 | 43 | args.dataset = args.dataset.lower() 44 | 45 | 46 | 47 | 48 | import tensorflow.compat.v1 as tf 49 | import yaml 50 | from scipy.sparse import csr_matrix 51 | from lib.utils import load_graph_data 52 | from model.prnn_supervisor import PRNNSupervisor 53 | import inference 54 | 55 | 56 | def main(args): 57 | print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) 58 | # if not tf. 59 | 60 | args.config_filename = join(args.config_dir, "prnn_" + args.dataset + ".yaml") 61 | with open(args.config_filename) as f: 62 | 63 | supervisor_config = yaml.load(f) 64 | 65 | if args.rnn_type is not None: 66 | supervisor_config["model"]["rnn_type"] = args.rnn_type 67 | if args.cost is not None: 68 | supervisor_config["train"]["cost"] = args.cost 69 | 70 | 71 | graph_pkl_filename = join(args.graph_dir, "adj_" + args.dataset + ".pkl") 72 | supervisor_config['data']["graph_pkl_filename"] = graph_pkl_filename 73 | 74 | adj_mx = load_graph_data(graph_pkl_filename).astype('float32') 75 | 76 | tf_config = tf.ConfigProto() 77 | if args.use_cpu_only: 78 | tf_config = tf.ConfigProto(device_count={'GPU': 0}) 79 | else: 80 | tf_config = tf.ConfigProto(device_count={'GPU': 1}) 81 | 82 | tf_config.gpu_options.allow_growth = True 83 | 84 | with tf.Session(config=tf_config) as sess: 85 | supervisor = PRNNSupervisor(adj_mx=adj_mx, args=args, inference=False, pretrained_dir=None, **supervisor_config) 86 | args.pretrained_dir = supervisor._log_dir 87 | supervisor.train(sess=sess) 88 | 89 | 90 | print("the checkpoint files are saved in :", args.pretrained_dir) 91 | 92 | 93 | if __name__ == '__main__': 94 | main(args) 95 | -------------------------------------------------------------------------------- /scripts/generate_training_data.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | from __future__ import unicode_literals 5 | 6 | import argparse 7 | import numpy as np 8 | import os 9 | import pandas as pd 10 | 11 | 12 | def generate_graph_seq2seq_io_data( 13 | df, x_offsets, y_offsets, add_time_in_day=False, add_day_in_week=False, scaler=None 14 | ): 15 | """ 16 | Generate samples from 17 | :param df: 18 | :param x_offsets: 19 | :param y_offsets: 20 | :param add_time_in_day: 21 | :param add_day_in_week: 22 | :param scaler: 23 | :return: 24 | # x: (epoch_size, input_length, num_nodes, input_dim) 25 | # y: (epoch_size, output_length, num_nodes, output_dim) 26 | """ 27 | 28 | num_samples, num_nodes = df.shape 29 | #data = np.expand_dims(df.values, axis=-1) 30 | data = np.expand_dims(df, axis=-1) 31 | data_list = [data] 32 | #if add_time_in_day: 33 | # time_ind = (df.index.values - df.index.values.astype("datetime64[D]")) / np.timedelta64(1, "D") 34 | # time_in_day = np.tile(time_ind, [1, num_nodes, 1]).transpose((2, 1, 0)) 35 | # data_list.append(time_in_day) 36 | #if add_day_in_week: 37 | # day_in_week = np.zeros(shape=(num_samples, num_nodes, 7)) 38 | # day_in_week[np.arange(num_samples), :, df.index.dayofweek] = 1 39 | # data_list.append(day_in_week) 40 | 41 | data = np.concatenate(data_list, axis=-1) 42 | # epoch_len = num_samples + min(x_offsets) - max(y_offsets) 43 | x, y = [], [] 44 | # t is the index of the last observation. 45 | min_t = abs(min(x_offsets)) 46 | max_t = abs(num_samples - abs(max(y_offsets))) # Exclusive 47 | for t in range(min_t, max_t): 48 | x_t = data[t + x_offsets, ...] 49 | y_t = data[t + y_offsets, ...] 50 | x.append(x_t) 51 | y.append(y_t) 52 | x = np.stack(x, axis=0) 53 | y = np.stack(y, axis=0) 54 | return x, y 55 | 56 | 57 | def generate_train_val_test(args): 58 | #df = pd.read_hdf(args.traffic_df_filename) 59 | #df = pd.read_csv(args.traffic_df_filename).astype('float32') 60 | df = np.load(args.traffic_df_filename)['data'][:,:,-1].astype('float32') 61 | 62 | 63 | # 0 is the latest observed sample. 64 | x_offsets = np.sort( 65 | # np.concatenate(([-week_size + 1, -day_size + 1], np.arange(-11, 1, 1))) 66 | np.concatenate((np.arange(-11, 1, 1),)) 67 | ) 68 | # Predict the next one hour 69 | y_offsets = np.sort(np.arange(1, 13, 1)) 70 | # x: (num_samples, input_length, num_nodes, input_dim) 71 | # y: (num_samples, output_length, num_nodes, output_dim) 72 | x, y = generate_graph_seq2seq_io_data( 73 | df, 74 | x_offsets=x_offsets, 75 | y_offsets=y_offsets, 76 | add_time_in_day=True, 77 | add_day_in_week=False, 78 | ) 79 | 80 | print("x shape: ", x.shape, ", y shape: ", y.shape) 81 | # Write the data into npz file. 82 | # num_test = 6831, using the last 6831 examples as testing. 83 | # for the rest: 7/8 is used for training, and 1/8 is used for validation. 84 | num_samples = x.shape[0] 85 | num_test = int(round(num_samples * 0.2)) 86 | num_train = int(round(num_samples * 0.7)) 87 | num_val = num_samples - num_test - num_train 88 | 89 | # train 90 | x_train, y_train = x[:num_train], y[:num_train] 91 | # val 92 | x_val, y_val = ( 93 | x[num_train: num_train + num_val], 94 | y[num_train: num_train + num_val], 95 | ) 96 | # test 97 | x_test, y_test = x[-num_test:], y[-num_test:] 98 | 99 | for cat in ["train", "val", "test"]: 100 | _x, _y = locals()["x_" + cat], locals()["y_" + cat] 101 | print(cat, "x: ", _x.shape, "y:", _y.shape) 102 | np.savez_compressed( 103 | os.path.join(args.output_dir, "%s.npz" % cat), 104 | x=_x, 105 | y=_y, 106 | x_offsets=x_offsets.reshape(list(x_offsets.shape) + [1]), 107 | y_offsets=y_offsets.reshape(list(y_offsets.shape) + [1]), 108 | ) 109 | 110 | 111 | def main(args): 112 | print("Generating training data") 113 | generate_train_val_test(args) 114 | 115 | 116 | if __name__ == "__main__": 117 | parser = argparse.ArgumentParser() 118 | parser.add_argument( 119 | "--output_dir", type=str, default="data/pemsd7", help="Output directory." 120 | ) 121 | parser.add_argument( 122 | "--traffic_df_filename", 123 | type=str, 124 | default="data/pemsd7.npz", 125 | help="Raw traffic readings.", 126 | ) 127 | args = parser.parse_args() 128 | main(args) 129 | -------------------------------------------------------------------------------- /eval_metrics/metrics.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from eval_metrics.utils import * 3 | from os.path import join 4 | 5 | 6 | 7 | def eval_deter(args, horizons=None): 8 | 9 | dataset = args.dataset 10 | algorithm = args.rnn_type 11 | 12 | result = np.load(args.output_filename, allow_pickle=True) 13 | 14 | true = result['groundtruth'] 15 | pred = result['predictions'] 16 | 17 | # print(true.shape) 18 | # print(pred.shape) 19 | # print(result['predictions_samples'].shape) 20 | 21 | try: 22 | test_mask = np.load(join(args.data_dir, args.dataset, "test_mask_y.npz"))["test_mask"] 23 | test_mask = np.transpose(test_mask, [1, 0, 2]) 24 | except Exception: 25 | test_mask = np.ones_like(true) 26 | 27 | print(test_mask.shape) 28 | 29 | print('------------------------------------------------------------------------------------') 30 | print('Dataset : ' + dataset) 31 | print('Algorithm : ' + algorithm) 32 | print('------------------------------------------------------------------------------------') 33 | print('Horizon : 15/30/45/60 minutes') 34 | 35 | metrics = np.zeros((3, 4)) 36 | results = ["Metrics & MAE & MAPE & RMSE"] 37 | 38 | if horizons is None: 39 | if pred.shape[0] == 12: 40 | horizons = [2, 5, 8, 11] 41 | elif pred.shape[0] == 4: 42 | horizons = [0, 1, 2, 3] 43 | else: 44 | raise NotImplementedError("Plase specify the horizons to evaluate") 45 | 46 | for horizon in horizons: 47 | 48 | mae = masked_mae_np(preds=pred[horizon, :, :], labels=true[horizon, :, :], mask=test_mask[horizon, :, :]) 49 | mape = masked_mape_np(preds=pred[horizon, :, :], labels=true[horizon, :, :], mask=test_mask[horizon, :, :]) 50 | rmse = masked_rmse_np(preds=pred[horizon, :, :], labels=true[horizon, :, :], mask=test_mask[horizon, :, :]) 51 | 52 | metrics[0, int((horizon - 2) / 3)] = mae 53 | metrics[1, int((horizon - 2) / 3)] = mape 54 | metrics[2, int((horizon - 2) / 3)] = rmse 55 | 56 | print('MAE, MAPE, RMSE : &%.2f &%.2f &%.2f' % (mae, mape, rmse)) 57 | results.append("Horizon-{}".format(horizon)+ ' & %.2f & %.2f & %.2f' % (mae, mape, rmse)) 58 | 59 | np.set_printoptions(precision=2) 60 | 61 | return results 62 | # print(metrics) 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | if __name__ == "__main__": 77 | 78 | dataset = 'PEMS08' 79 | algorithm = 'pgagru_mae' 80 | 81 | result = np.load('/home/soumyasundar/CRPS/log/' + algorithm + '_predictions_' + dataset + '.npz', allow_pickle=True) 82 | print(result.files) 83 | 84 | # res = result['arr_0'] 85 | # result = res.tolist() 86 | # print(result.keys) 87 | 88 | true = result['groundtruth'] 89 | pred = result['predictions'] 90 | 91 | # true = np.transpose(np.tile(result['truth'][:, :, np.newaxis], [1, 1, 12]), [2, 0, 1]) 92 | # # pred = np.transpose(np.tile(result['pred'][:, :, np.newaxis], [1, 1, 12]), [2, 0, 1]) 93 | # pred = np.transpose(np.repeat(result['pred'], 3, axis=2), [2, 0, 1]) 94 | 95 | # true = np.transpose(np.squeeze(result['ground_truth']), [2, 0, 1]) 96 | # pred = np.transpose(np.squeeze(result['prediction']), [2, 0, 1]) 97 | 98 | # true = np.transpose(result['truth'], [1, 0, 2]) 99 | # pred = np.transpose(result['preds'], [1, 0, 2]) 100 | 101 | # true = np.transpose(np.repeat(np.squeeze(result['ground_truth']), 3, axis=1), [1, 0, 2]) 102 | # pred = np.transpose(np.repeat(np.squeeze(result['predictions']), 3, axis=0), [0, 1, 2]) 103 | 104 | print(true.shape) 105 | print(pred.shape) 106 | print(result['predictions_samples'].shape) 107 | 108 | test_mask = np.load('/home/soumyasundar/CRPS/log/test_mask_y_' + dataset + '.npz')['test_mask'] 109 | test_mask = np.transpose(test_mask, [1, 0, 2]) 110 | 111 | print(test_mask.shape) 112 | 113 | print('------------------------------------------------------------------------------------') 114 | print('Dataset : ' + dataset) 115 | print('Algorithm : ' + algorithm) 116 | print('------------------------------------------------------------------------------------') 117 | print('Horizon : 15/30/45/60 minutes') 118 | 119 | metrics = np.zeros((3, 4)) 120 | 121 | for horizon in [2, 5, 8, 11]: 122 | 123 | mae = masked_mae_np(preds=pred[horizon, :, :], labels=true[horizon, :, :], mask=test_mask[horizon, :, :]) 124 | mape = masked_mape_np(preds=pred[horizon, :, :], labels=true[horizon, :, :], mask=test_mask[horizon, :, :]) 125 | rmse = masked_rmse_np(preds=pred[horizon, :, :], labels=true[horizon, :, :], mask=test_mask[horizon, :, :]) 126 | 127 | metrics[0, int((horizon - 2) / 3)] = mae 128 | metrics[1, int((horizon - 2) / 3)] = mape 129 | metrics[2, int((horizon - 2) / 3)] = rmse 130 | 131 | print('MAE, MAPE, RMSE : &%.2f &%.2f &%.2f' % (mae, mape, rmse)) 132 | 133 | np.set_printoptions(precision=2) 134 | print(metrics) -------------------------------------------------------------------------------- /model/prnn_model.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | from model.utils_tf import AGCGRU, DCGRU, GRU, training 5 | 6 | import tensorflow as tf 7 | import numpy as np 8 | 9 | 10 | class PRNNModel(object): 11 | def __init__(self, is_training, batch_size, scaler, adj_mx, **model_kwargs): 12 | # Scaler for data normalization. 13 | self._scaler = scaler 14 | 15 | # Train and loss 16 | self._loss = None 17 | self._mae = None 18 | self._train_op = None 19 | 20 | nParticle = int(model_kwargs.get('nParticle', 10)) 21 | nParticle_test = int(model_kwargs.get('nParticle_test', 25)) 22 | 23 | max_diffusion_step = int(model_kwargs.get('max_diffusion_step', 2)) 24 | cl_decay_steps = int(model_kwargs.get('cl_decay_steps', 1000)) 25 | filter_type = model_kwargs.get('filter_type', 'laplacian') 26 | horizon = int(model_kwargs.get('horizon', 1)) 27 | max_grad_norm = float(model_kwargs.get('max_grad_norm', 5.0)) 28 | num_nodes = int(model_kwargs.get('num_nodes', 1)) 29 | num_rnn_layers = int(model_kwargs.get('num_rnn_layers', 1)) 30 | rnn_units = int(model_kwargs.get('rnn_units')) 31 | seq_len = int(model_kwargs.get('seq_len')) 32 | use_curriculum_learning = bool(model_kwargs.get('use_curriculum_learning', False)) 33 | input_dim = int(model_kwargs.get('input_dim', 1)) 34 | output_dim = int(model_kwargs.get('output_dim', 1)) 35 | embed_dim = int(model_kwargs.get('embed_dim', 10)) 36 | rho = float(model_kwargs.get('rho', 1.0)) 37 | rnn_type = model_kwargs.get('rnn_type', 'agcgru') # agcgru/dcgru/gru 38 | 39 | # Input (batch_size, timesteps, num_sensor, input_dim) 40 | self._inputs = tf.placeholder(tf.float32, shape=(batch_size, seq_len, num_nodes, input_dim), name='inputs') 41 | # Labels: (batch_size, timesteps, num_sensor, input_dim), same format with input except the temporal dimension. 42 | self._labels = tf.placeholder(tf.float32, shape=(batch_size, horizon, num_nodes, input_dim), name='labels') 43 | 44 | node_embedding = tf.get_variable("node_embedding", [num_nodes, embed_dim], dtype=tf.float32, 45 | initializer=tf.contrib.layers.xavier_initializer()) 46 | 47 | if rnn_type == 'agcgru': 48 | rnn_0 = AGCGRU(adj_mx=adj_mx, input_dim=input_dim, embed_dim=embed_dim, num_units=rnn_units, 49 | max_diffusion_step=max_diffusion_step, scope="layer0") 50 | rnn_1 = AGCGRU(adj_mx=adj_mx, input_dim=rnn_units, embed_dim=embed_dim, num_units=rnn_units, 51 | max_diffusion_step=max_diffusion_step, scope="layer1") 52 | elif rnn_type == 'dcgru': 53 | rnn_0 = DCGRU(adj_mx=adj_mx, input_dim=input_dim, num_units=rnn_units, 54 | max_diffusion_step=max_diffusion_step, scope="layer0") 55 | rnn_1 = DCGRU(adj_mx=adj_mx, input_dim=rnn_units, num_units=rnn_units, 56 | max_diffusion_step=max_diffusion_step, scope="layer1") 57 | elif rnn_type == 'gru': 58 | rnn_0 = GRU(adj_mx=adj_mx, input_dim=input_dim, num_units=rnn_units, scope="layer0") 59 | rnn_1 = GRU(adj_mx=adj_mx, input_dim=rnn_units, num_units=rnn_units, scope="layer1") 60 | else: 61 | print('ERROR! ERROR! ERROR!') 62 | exit(0) 63 | 64 | weight = tf.get_variable("weight", [rnn_units, output_dim], dtype=tf.float32, 65 | initializer=tf.contrib.layers.xavier_initializer()) 66 | weight_delta = tf.get_variable("weight_delta", [rnn_units, output_dim], dtype=tf.float32, 67 | initializer=tf.contrib.layers.xavier_initializer()) 68 | 69 | global_step = tf.train.get_or_create_global_step() 70 | 71 | with tf.variable_scope('DCRNN_SEQ'): 72 | curriculam_prob = self._compute_sampling_threshold(global_step, cl_decay_steps) 73 | 74 | if is_training: 75 | output_samples, output_mu, output_sigma = training(self._inputs, self._labels, rnn_0, rnn_1, weight, weight_delta, 76 | rho, node_embedding, rnn_units, nParticle, is_training, curriculam_prob, rnn_type) 77 | else: 78 | output_samples, output_mu, output_sigma = training(self._inputs, self._labels, rnn_0, rnn_1, weight, weight_delta, 79 | rho, node_embedding, rnn_units, nParticle_test, is_training, curriculam_prob, rnn_type) 80 | 81 | # Project the output to output_dim. 82 | outputs = tf.reduce_mean(output_samples, axis=3) 83 | self._outputs = tf.reshape(outputs, (batch_size, horizon, num_nodes, output_dim), name='outputs') 84 | self._outputs_samples = output_samples 85 | self._outputs_mu = output_mu 86 | self._outputs_sigma = output_sigma 87 | self._merged = tf.summary.merge_all() 88 | 89 | @staticmethod 90 | def _compute_sampling_threshold(global_step, k): 91 | """ 92 | Computes the sampling probability for scheduled sampling using inverse sigmoid. 93 | :param global_step: 94 | :param k: 95 | :return: 96 | """ 97 | return tf.cast(k / (k + tf.exp(global_step / k)), tf.float32) 98 | 99 | @property 100 | def inputs(self): 101 | return self._inputs 102 | 103 | @property 104 | def labels(self): 105 | return self._labels 106 | 107 | @property 108 | def loss(self): 109 | return self._loss 110 | 111 | @property 112 | def mae(self): 113 | return self._mae 114 | 115 | @property 116 | def merged(self): 117 | return self._merged 118 | 119 | @property 120 | def outputs(self): 121 | return self._outputs 122 | 123 | @property 124 | def outputs_samples(self): 125 | return self._outputs_samples 126 | 127 | @property 128 | def outputs_mu(self): 129 | return self._outputs_mu 130 | 131 | @property 132 | def outputs_sigma(self): 133 | return self._outputs_sigma 134 | -------------------------------------------------------------------------------- /eval_metrics/metrics_sample.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from eval_metrics.utils import * 3 | from os.path import join 4 | 5 | 6 | def eval_samples(args, horizons=None): 7 | dataset = args.dataset 8 | algorithm = args.rnn_type 9 | 10 | sample_based = False 11 | 12 | result = np.load(args.output_filename) 13 | # print(result.files) 14 | 15 | true = result['groundtruth'] 16 | pred_samples = result['predictions_samples'] 17 | 18 | # print(true.shape) 19 | # print(pred_samples.shape) 20 | 21 | mu = np.mean(pred_samples, axis=3) 22 | sigma = np.std(pred_samples, axis=3) 23 | try: 24 | test_mask = np.load(join(args.data_dir, args.dataset, "test_mask_y.npz"))["test_mask"] 25 | test_mask = np.transpose(test_mask, [1, 0, 2]) 26 | except Exception: 27 | test_mask = np.ones_like(true) 28 | # print(test_mask.shape) 29 | 30 | print('------------------------------------------------------------------------------------') 31 | print('Dataset : ' + dataset) 32 | print('Algorithm : ' + algorithm) 33 | 34 | print('------------------------------------------------------------------------------------') 35 | print('Horizon : 15/30/45/60 minutes') 36 | 37 | if horizons is None: 38 | if pred_samples.shape[0] == 12: 39 | horizons = [2, 5, 8, 11] 40 | elif pred_samples.shape[0] == 4: 41 | horizons = [0, 1, 2, 3] 42 | else: 43 | raise NotImplementedError("Plase specify the horizons to evaluate") 44 | 45 | metrics = np.zeros((3, 4)) 46 | results = ["Metrics & CRPS & P10QL & P90QL"] 47 | 48 | for horizon in horizons: 49 | if sample_based: 50 | crps = masked_crps_samples(pred_samples[horizon, :, :, :], true[horizon, :, :], test_mask[horizon, :, :]) 51 | #coverage = empirical_coverage_samples(pred_samples[horizon, :, :, :], true[horizon, :, :], test_mask[horizon, :, :], interval=0.95) 52 | loss_10 = percentage_quantile_loss_samples(pred_samples[horizon, :, :, :], true[horizon, :, :], test_mask[horizon, :, :], q=0.1) 53 | loss_90 = percentage_quantile_loss_samples(pred_samples[horizon, :, :, :], true[horizon, :, :], test_mask[horizon, :, :], q=0.9) 54 | else: 55 | crps = masked_crps_dist(mu[horizon, :, :], sigma[horizon, :, :], true[horizon, :, :], test_mask[horizon, :, :]) 56 | #coverage = empirical_coverage_dist(mu[horizon, :, :], sigma[horizon, :, :], true[horizon, :, :], test_mask[horizon, :, :], interval=0.95) 57 | loss_10 = percentage_quantile_loss_dist(mu[horizon, :, :], sigma[horizon, :, :], true[horizon, :, :], test_mask[horizon, :, :], q=0.1) 58 | loss_90 = percentage_quantile_loss_dist(mu[horizon, :, :], sigma[horizon, :, :], true[horizon, :, :], test_mask[horizon, :, :], q=0.9) 59 | 60 | 61 | metrics[0, int((horizon - 2) / 3)] = crps 62 | metrics[1, int((horizon - 2) / 3)] = loss_10 63 | metrics[2, int((horizon - 2) / 3)] = loss_90 64 | 65 | print('CRPS, P10QL, P90QL : &%.2f &%.2f &%.2f' % (crps, loss_10, loss_90)) 66 | results.append("Horizon-{}".format(horizon) + '& %.2f & %.2f & %.2f' % (crps, loss_10, loss_90)) 67 | #print('CRPS, Coverage(95), P10QL, P90QL : &%.2f &%.2f &%.2f &%.2f' % (crps, coverage, loss_10, loss_90)) 68 | #print('P10QL, P90QL : &%.2f &%.2f' % (loss_10, loss_90)) 69 | 70 | 71 | np.set_printoptions(precision=2) 72 | # print(metrics) 73 | return results 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | if __name__ == "__main__": 89 | 90 | dataset = 'PEMS03' 91 | algorithm = 'pgagru_nll' 92 | sample_based = False 93 | 94 | result = np.load('/home/soumyasundar/CRPS/log/' + algorithm + '_predictions_' + dataset + '.npz') 95 | print(result.files) 96 | 97 | true = result['groundtruth'] 98 | pred_samples = result['predictions_samples'] 99 | 100 | print(true.shape) 101 | print(pred_samples.shape) 102 | 103 | mu = np.mean(pred_samples, axis=3) 104 | sigma = np.std(pred_samples, axis=3) 105 | test_mask = np.load('/home/soumyasundar/CRPS/log/test_mask_y_' + dataset + '.npz')['test_mask'] 106 | test_mask = np.transpose(test_mask, [1, 0, 2]) 107 | print(test_mask.shape) 108 | 109 | print('------------------------------------------------------------------------------------') 110 | print('Dataset : ' + dataset) 111 | print('Algorithm : ' + algorithm) 112 | 113 | print('------------------------------------------------------------------------------------') 114 | print('Horizon : 15/30/45/60 minutes') 115 | 116 | metrics = np.zeros((3, 4)) 117 | 118 | for horizon in [2, 5, 8, 11]: 119 | if sample_based: 120 | crps = masked_crps_samples(pred_samples[horizon, :, :, :], true[horizon, :, :], test_mask[horizon, :, :]) 121 | #coverage = empirical_coverage_samples(pred_samples[horizon, :, :, :], true[horizon, :, :], test_mask[horizon, :, :], interval=0.95) 122 | loss_10 = percentage_quantile_loss_samples(pred_samples[horizon, :, :, :], true[horizon, :, :], test_mask[horizon, :, :], q=0.1) 123 | loss_90 = percentage_quantile_loss_samples(pred_samples[horizon, :, :, :], true[horizon, :, :], test_mask[horizon, :, :], q=0.9) 124 | else: 125 | crps = masked_crps_dist(mu[horizon, :, :], sigma[horizon, :, :], true[horizon, :, :], test_mask[horizon, :, :]) 126 | #coverage = empirical_coverage_dist(mu[horizon, :, :], sigma[horizon, :, :], true[horizon, :, :], test_mask[horizon, :, :], interval=0.95) 127 | loss_10 = percentage_quantile_loss_dist(mu[horizon, :, :], sigma[horizon, :, :], true[horizon, :, :], test_mask[horizon, :, :], q=0.1) 128 | loss_90 = percentage_quantile_loss_dist(mu[horizon, :, :], sigma[horizon, :, :], true[horizon, :, :], test_mask[horizon, :, :], q=0.9) 129 | 130 | 131 | metrics[0, int((horizon - 2) / 3)] = crps 132 | metrics[1, int((horizon - 2) / 3)] = loss_10 133 | metrics[2, int((horizon - 2) / 3)] = loss_90 134 | 135 | print('CRPS, P10QL, P90QL : &%.2f &%.2f &%.2f' % (crps, loss_10, loss_90)) 136 | #print('CRPS, Coverage(95), P10QL, P90QL : &%.2f &%.2f &%.2f &%.2f' % (crps, coverage, loss_10, loss_90)) 137 | #print('P10QL, P90QL : &%.2f &%.2f' % (loss_10, loss_90)) 138 | 139 | 140 | np.set_printoptions(precision=2) 141 | print(metrics) 142 | -------------------------------------------------------------------------------- /scripts/eval_baseline_methods.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import pandas as pd 4 | 5 | from statsmodels.tsa.vector_ar.var_model import VAR 6 | 7 | from lib import utils 8 | from lib.metrics import masked_rmse_np, masked_mape_np, masked_mae_np 9 | from lib.utils import StandardScaler 10 | 11 | 12 | def historical_average_predict(df, period=12 * 24 * 7, test_ratio=0.2, null_val=0.): 13 | """ 14 | Calculates the historical average of sensor reading. 15 | :param df: 16 | :param period: default 1 week. 17 | :param test_ratio: 18 | :param null_val: default 0. 19 | :return: 20 | """ 21 | n_sample, n_sensor = df.shape 22 | n_test = int(round(n_sample * test_ratio)) 23 | n_train = n_sample - n_test 24 | y_test = df[-n_test:] 25 | y_predict = pd.DataFrame.copy(y_test) 26 | 27 | for i in range(n_train, min(n_sample, n_train + period)): 28 | inds = [j for j in range(i % period, n_train, period)] 29 | historical = df.iloc[inds, :] 30 | y_predict.iloc[i - n_train, :] = historical[historical != null_val].mean() 31 | # Copy each period. 32 | for i in range(n_train + period, n_sample, period): 33 | size = min(period, n_sample - i) 34 | start = i - n_train 35 | y_predict.iloc[start:start + size, :] = y_predict.iloc[start - period: start + size - period, :].values 36 | return y_predict, y_test 37 | 38 | 39 | def static_predict(df, n_forward, test_ratio=0.2): 40 | """ 41 | Assumes $x^{t+1} = x^{t}$ 42 | :param df: 43 | :param n_forward: 44 | :param test_ratio: 45 | :return: 46 | """ 47 | test_num = int(round(df.shape[0] * test_ratio)) 48 | y_test = df[-test_num:] 49 | y_predict = df.shift(n_forward).iloc[-test_num:] 50 | return y_predict, y_test 51 | 52 | 53 | def var_predict(df, n_forwards=(1, 3), n_lags=4, test_ratio=0.2): 54 | """ 55 | Multivariate time series forecasting using Vector Auto-Regressive Model. 56 | :param df: pandas.DataFrame, index: time, columns: sensor id, content: data. 57 | :param n_forwards: a tuple of horizons. 58 | :param n_lags: the order of the VAR model. 59 | :param test_ratio: 60 | :return: [list of prediction in different horizon], dt_test 61 | """ 62 | n_sample, n_output = df.shape 63 | n_test = int(round(n_sample * test_ratio)) 64 | n_train = n_sample - n_test 65 | df_train, df_test = df[:n_train], df[n_train:] 66 | 67 | scaler = StandardScaler(mean=df_train.values.mean(), std=df_train.values.std()) 68 | data = scaler.transform(df_train.values) 69 | var_model = VAR(data) 70 | var_result = var_model.fit(n_lags) 71 | max_n_forwards = np.max(n_forwards) 72 | # Do forecasting. 73 | result = np.zeros(shape=(len(n_forwards), n_test, n_output)) 74 | start = n_train - n_lags - max_n_forwards + 1 75 | for input_ind in range(start, n_sample - n_lags): 76 | prediction = var_result.forecast(scaler.transform(df.values[input_ind: input_ind + n_lags]), max_n_forwards) 77 | for i, n_forward in enumerate(n_forwards): 78 | result_ind = input_ind - n_train + n_lags + n_forward - 1 79 | if 0 <= result_ind < n_test: 80 | result[i, result_ind, :] = prediction[n_forward - 1, :] 81 | 82 | df_predicts = [] 83 | for i, n_forward in enumerate(n_forwards): 84 | df_predict = pd.DataFrame(scaler.inverse_transform(result[i]), index=df_test.index, columns=df_test.columns) 85 | df_predicts.append(df_predict) 86 | return df_predicts, df_test 87 | 88 | 89 | def eval_static(traffic_reading_df): 90 | logger.info('Static') 91 | horizons = [1, 3, 6, 12] 92 | logger.info('\t'.join(['Model', 'Horizon', 'RMSE', 'MAPE', 'MAE'])) 93 | for horizon in horizons: 94 | y_predict, y_test = static_predict(traffic_reading_df, n_forward=horizon, test_ratio=0.2) 95 | rmse = masked_rmse_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0) 96 | mape = masked_mape_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0) 97 | mae = masked_mae_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0) 98 | line = 'Static\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae) 99 | logger.info(line) 100 | 101 | 102 | def eval_historical_average(traffic_reading_df, period): 103 | y_predict, y_test = historical_average_predict(traffic_reading_df, period=period, test_ratio=0.2) 104 | rmse = masked_rmse_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0) 105 | mape = masked_mape_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0) 106 | mae = masked_mae_np(preds=y_predict.as_matrix(), labels=y_test.as_matrix(), null_val=0) 107 | logger.info('Historical Average') 108 | logger.info('\t'.join(['Model', 'Horizon', 'RMSE', 'MAPE', 'MAE'])) 109 | for horizon in [1, 3, 6, 12]: 110 | line = 'HA\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae) 111 | logger.info(line) 112 | 113 | 114 | def eval_var(traffic_reading_df, n_lags=3): 115 | n_forwards = [1, 3, 6, 12] 116 | y_predicts, y_test = var_predict(traffic_reading_df, n_forwards=n_forwards, n_lags=n_lags, 117 | test_ratio=0.2) 118 | logger.info('VAR (lag=%d)' % n_lags) 119 | logger.info('Model\tHorizon\tRMSE\tMAPE\tMAE') 120 | for i, horizon in enumerate(n_forwards): 121 | rmse = masked_rmse_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0) 122 | mape = masked_mape_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0) 123 | mae = masked_mae_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0) 124 | line = 'VAR\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae) 125 | logger.info(line) 126 | 127 | 128 | def main(args): 129 | traffic_reading_df = pd.read_hdf(args.traffic_reading_filename) 130 | eval_static(traffic_reading_df) 131 | eval_historical_average(traffic_reading_df, period=7 * 24 * 12) 132 | eval_var(traffic_reading_df, n_lags=3) 133 | 134 | 135 | if __name__ == '__main__': 136 | logger = utils.get_logger('data/model', 'Baseline') 137 | parser = argparse.ArgumentParser() 138 | parser.add_argument('--traffic_reading_filename', default="data/metr-la.h5", type=str, 139 | help='Path to the traffic Dataframe.') 140 | args = parser.parse_args() 141 | main(args) 142 | -------------------------------------------------------------------------------- /eval_metrics/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.stats 3 | from eval_metrics.crps import crps_ensemble, crps_gaussian 4 | 5 | 6 | def masked_rmse_np(preds, labels, mask): 7 | return np.sqrt(masked_mse_np(preds=preds, labels=labels, mask=mask)) 8 | 9 | 10 | def masked_mse_np(preds, labels, mask): 11 | mask = mask.astype('float32') 12 | mask /= np.mean(mask) 13 | rmse = np.square(np.subtract(preds, labels)).astype('float32') 14 | rmse = np.nan_to_num(rmse * mask) 15 | return np.mean(rmse) 16 | 17 | 18 | def masked_mae_np(preds, labels, mask): 19 | mask = mask.astype('float32') 20 | mask /= np.mean(mask) 21 | mae = np.abs(np.subtract(preds, labels)).astype('float32') 22 | mae = np.nan_to_num(mae * mask) 23 | return np.mean(mae) 24 | 25 | 26 | def masked_mape_np(preds, labels, mask): 27 | mask = mask.astype('float32') 28 | mask /= np.mean(mask) 29 | # add labels masking for zeros 30 | zero_labels = np.where(labels == 0) 31 | zero_masking = np.ones_like(labels) 32 | zero_masking[zero_labels] = 1 33 | valid_num = np.sum(zero_masking) 34 | # 35 | mape = np.abs(np.divide(np.subtract(preds, labels).astype('float32'), labels)) 36 | mape = np.nan_to_num(mask * mape) 37 | mape = mape * zero_masking 38 | return np.sum(mape) / valid_num * 100 39 | 40 | 41 | def masked_crps_samples(pred_samples, labels, mask): 42 | mask = mask.astype('float32') 43 | mask /= np.mean(mask) 44 | 45 | dim1, dim2, nParticle = np.shape(pred_samples) 46 | pred_samples = np.reshape(pred_samples, [-1, nParticle]) 47 | labels = np.reshape(labels, [-1]) 48 | mask = np.reshape(mask, [-1]) 49 | 50 | crps = np.zeros(np.shape(mask)) 51 | for idx in range(len(mask)): 52 | crps[idx] = crps_ensemble(labels[idx], pred_samples[idx, :]) 53 | 54 | crps = crps.astype('float32') 55 | crps = np.nan_to_num(crps * mask) 56 | return np.mean(crps) 57 | 58 | 59 | def empirical_coverage_samples(pred_samples, labels, mask, interval=0.95): 60 | mask = mask.astype('float32') 61 | mask /= np.mean(mask) 62 | 63 | dim1, dim2, nParticle = np.shape(pred_samples) 64 | pred_samples = np.reshape(pred_samples, [-1, nParticle]) 65 | labels = np.reshape(labels, [-1]) 66 | mask = np.reshape(mask, [-1]) 67 | 68 | lower_quantile = (1-interval)/2 69 | upper_quantile = 1 - (1-interval)/2 70 | 71 | coverage = np.zeros(np.shape(mask)) 72 | for idx in range(len(mask)): 73 | 74 | lb = np.quantile(pred_samples[idx, :], lower_quantile) 75 | ub = np.quantile(pred_samples[idx, :], upper_quantile) 76 | 77 | if lb <= labels[idx] <= ub: 78 | coverage[idx] = 1.0 79 | 80 | coverage = coverage.astype('float32') 81 | coverage = np.nan_to_num(coverage * mask) 82 | return np.mean(coverage) * 100 83 | 84 | 85 | def masked_crps_dist(mu, sigma, labels, mask): 86 | mask = mask.astype('float32') 87 | mask /= np.mean(mask) 88 | 89 | mu = np.reshape(mu, [-1]) 90 | sigma = np.reshape(sigma, [-1]) 91 | labels = np.reshape(labels, [-1]) 92 | mask = np.reshape(mask, [-1]) 93 | 94 | crps = np.zeros(np.shape(mask)) 95 | for idx in range(len(mask)): 96 | crps[idx] = crps_gaussian(labels[idx], mu[idx], sigma[idx]) 97 | 98 | crps = crps.astype('float32') 99 | crps = np.nan_to_num(crps * mask) 100 | return np.mean(crps) 101 | 102 | 103 | def empirical_coverage_dist(mu, sigma, labels, mask, interval=0.95): 104 | mask = mask.astype('float32') 105 | mask /= np.mean(mask) 106 | 107 | mu = np.reshape(mu, [-1]) 108 | sigma = np.reshape(sigma, [-1]) 109 | labels = np.reshape(labels, [-1]) 110 | mask = np.reshape(mask, [-1]) 111 | 112 | lower_quantile = (1-interval)/2 113 | upper_quantile = 1 - (1-interval)/2 114 | 115 | lb = mu + scipy.stats.norm.ppf(lower_quantile) * sigma 116 | ub = mu + scipy.stats.norm.ppf(upper_quantile) * sigma 117 | 118 | coverage = np.zeros(np.shape(mask)) 119 | for idx in range(len(mask)): 120 | 121 | if lb[idx] <= labels[idx] <= ub[idx]: 122 | coverage[idx] = 1.0 123 | 124 | coverage = coverage.astype('float32') 125 | coverage = np.nan_to_num(coverage * mask) 126 | return np.mean(coverage) * 100 127 | 128 | 129 | def percentage_quantile_loss_samples(pred_samples, labels, mask, q=0.9): 130 | mask = mask.astype('float32') 131 | mask /= np.mean(mask) 132 | 133 | dim1, dim2, nParticle = np.shape(pred_samples) 134 | pred_samples = np.reshape(pred_samples, [-1, nParticle]) 135 | labels = np.reshape(labels, [-1]) 136 | mask = np.reshape(mask, [-1]) 137 | 138 | pred_q = np.zeros(np.shape(mask)) 139 | 140 | for idx in range(len(mask)): 141 | pred_q[idx] = np.quantile(pred_samples[idx, :], q) 142 | 143 | loss = 2 * np.maximum(q * (labels - pred_q), (q - 1) * (labels - pred_q)) 144 | 145 | loss *= mask 146 | labels = np.abs(labels) * mask 147 | loss = loss.astype('float32') 148 | labels = labels.astype('float32') 149 | loss = np.nan_to_num(loss) 150 | labels = np.nan_to_num(labels) 151 | return np.mean(loss) / np.mean(labels) * 100 152 | 153 | 154 | def percentage_quantile_loss_dist(mu, sigma, labels, mask, q=0.9): 155 | mask = mask.astype('float32') 156 | mask /= np.mean(mask) 157 | 158 | mu = np.reshape(mu, [-1]) 159 | sigma = np.reshape(sigma, [-1]) 160 | labels = np.reshape(labels, [-1]) 161 | mask = np.reshape(mask, [-1]) 162 | 163 | pred_q = mu + scipy.stats.norm.ppf(q) * sigma 164 | loss = 2 * np.maximum(q * (labels - pred_q), (q - 1) * (labels - pred_q)) 165 | 166 | loss *= mask 167 | labels = np.abs(labels) * mask 168 | loss = loss.astype('float32') 169 | labels = labels.astype('float32') 170 | loss = np.nan_to_num(loss) 171 | labels = np.nan_to_num(labels) 172 | return np.mean(loss) / np.mean(labels) * 100 173 | 174 | 175 | def percentage_quantile_loss_mqrnn(percentile, labels, mask, q=0.9): 176 | mask = mask.astype('float32') 177 | mask /= np.mean(mask) 178 | 179 | percentile = np.reshape(percentile, [-1]) 180 | labels = np.reshape(labels, [-1]) 181 | mask = np.reshape(mask, [-1]) 182 | 183 | pred_q = percentile 184 | 185 | loss = 2 * np.maximum(q * (labels - pred_q), (q - 1) * (labels - pred_q)) 186 | 187 | loss *= mask 188 | labels = np.abs(labels) * mask 189 | loss = loss.astype('float32') 190 | labels = labels.astype('float32') 191 | loss = np.nan_to_num(loss) 192 | labels = np.nan_to_num(labels) 193 | return np.mean(loss) / np.mean(labels) * 100 194 | -------------------------------------------------------------------------------- /lib/metrics.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | 5 | def masked_mse_tf(preds, labels, null_val=np.nan): 6 | """ 7 | Accuracy with masking. 8 | :param preds: 9 | :param labels: 10 | :param null_val: 11 | :return: 12 | """ 13 | if np.isnan(null_val): 14 | mask = ~tf.is_nan(labels) 15 | else: 16 | mask = tf.not_equal(labels, null_val) 17 | mask = tf.cast(mask, tf.float32) 18 | mask /= tf.reduce_mean(mask) 19 | mask = tf.where(tf.is_nan(mask), tf.zeros_like(mask), mask) 20 | loss = tf.square(tf.subtract(preds, labels)) 21 | loss = loss * mask 22 | loss = tf.where(tf.is_nan(loss), tf.zeros_like(loss), loss) 23 | return tf.reduce_mean(loss) 24 | 25 | 26 | def masked_mae_tf(preds, labels, null_val=np.nan): 27 | """ 28 | Accuracy with masking. 29 | :param preds: 30 | :param labels: 31 | :param null_val: 32 | :return: 33 | """ 34 | if np.isnan(null_val): 35 | mask = ~tf.is_nan(labels) 36 | else: 37 | mask = tf.not_equal(labels, null_val) 38 | mask = tf.cast(mask, tf.float32) 39 | mask /= tf.reduce_mean(mask) 40 | mask = tf.where(tf.is_nan(mask), tf.zeros_like(mask), mask) 41 | mask = tf.where(tf.is_inf(mask), tf.zeros_like(mask), mask) 42 | labels = tf.where(tf.is_nan(labels), tf.zeros_like(labels), labels) 43 | labels = tf.where(tf.is_inf(labels), tf.zeros_like(labels), labels) 44 | preds = tf.where(tf.is_nan(preds), tf.zeros_like(preds), preds) 45 | preds = tf.where(tf.is_inf(preds), tf.zeros_like(preds), preds) 46 | loss = tf.abs(tf.subtract(preds, labels)) 47 | loss = loss * mask 48 | loss = tf.where(tf.is_nan(loss), tf.zeros_like(loss), loss) 49 | loss = tf.where(tf.is_inf(loss), tf.zeros_like(loss), loss) 50 | return tf.reduce_mean(loss) 51 | 52 | 53 | def neg_log_gauss_tf(preds_mu, preds_sigma, labels): 54 | preds_sigma += 1e-8 55 | nParticle = preds_mu.get_shape()[3].value 56 | loss = 0.5 * tf.divide(tf.subtract(preds_mu, tf.tile(labels, [1, 1, 1, nParticle])), preds_sigma) ** 2 + tf.log(preds_sigma) 57 | loss = tf.where(tf.is_nan(loss), tf.zeros_like(loss), loss) 58 | loss = tf.where(tf.is_inf(loss), tf.zeros_like(loss), loss) 59 | loss_max = tf.reduce_max(loss, axis=3)[:, :, :, tf.newaxis] 60 | loss = tf.exp(loss-tf.tile(loss_max, [1, 1, 1, nParticle])) 61 | loss = tf.where(tf.is_nan(loss), tf.zeros_like(loss), loss) 62 | loss = tf.where(tf.is_inf(loss), tf.zeros_like(loss), loss) 63 | loss = tf.log(tf.reduce_mean(loss, axis=3)) 64 | loss = tf.where(tf.is_nan(loss), tf.zeros_like(loss), loss) 65 | loss = tf.where(tf.is_inf(loss), tf.zeros_like(loss), loss) 66 | loss += tf.squeeze(loss_max) 67 | return tf.reduce_mean(loss) 68 | 69 | 70 | def masked_rmse_tf(preds, labels, null_val=np.nan): 71 | """ 72 | Accuracy with masking. 73 | :param preds: 74 | :param labels: 75 | :param null_val: 76 | :return: 77 | """ 78 | return tf.sqrt(masked_mse_tf(preds=preds, labels=labels, null_val=null_val)) 79 | 80 | 81 | def masked_rmse_np(preds, labels, null_val=np.nan): 82 | return np.sqrt(masked_mse_np(preds=preds, labels=labels, null_val=null_val)) 83 | 84 | 85 | def masked_mse_np(preds, labels, null_val=np.nan): 86 | with np.errstate(divide='ignore', invalid='ignore'): 87 | if np.isnan(null_val): 88 | mask = ~np.isnan(labels) 89 | else: 90 | mask = np.not_equal(labels, null_val) 91 | mask = mask.astype('float32') 92 | mask /= np.mean(mask) 93 | rmse = np.square(np.subtract(preds, labels)).astype('float32') 94 | rmse = np.nan_to_num(rmse * mask) 95 | return np.mean(rmse) 96 | 97 | 98 | def masked_mae_np(preds, labels, null_val=np.nan): 99 | with np.errstate(divide='ignore', invalid='ignore'): 100 | if np.isnan(null_val): 101 | mask = ~np.isnan(labels) 102 | else: 103 | mask = np.not_equal(labels, null_val) 104 | mask = mask.astype('float32') 105 | mask /= np.mean(mask) 106 | mae = np.abs(np.subtract(preds, labels)).astype('float32') 107 | mae = np.nan_to_num(mae * mask) 108 | return np.mean(mae) 109 | 110 | 111 | def masked_mape_np(preds, labels, null_val=np.nan): 112 | with np.errstate(divide='ignore', invalid='ignore'): 113 | if np.isnan(null_val): 114 | mask = ~np.isnan(labels) 115 | else: 116 | mask = np.not_equal(labels, null_val) 117 | mask = mask.astype('float32') 118 | mask /= np.mean(mask) 119 | mape = np.abs(np.divide(np.subtract(preds, labels).astype('float32'), labels)) 120 | mape = np.nan_to_num(mask * mape) 121 | return np.mean(mape) 122 | 123 | 124 | # Builds loss function. 125 | def masked_mse_loss(scaler, null_val): 126 | def loss(preds, labels): 127 | if scaler: 128 | preds = scaler.inverse_transform(preds) 129 | labels = scaler.inverse_transform(labels) 130 | return masked_mse_tf(preds=preds, labels=labels, null_val=null_val) 131 | 132 | return loss 133 | 134 | 135 | def masked_rmse_loss(scaler, null_val): 136 | def loss(preds, labels): 137 | if scaler: 138 | preds = scaler.inverse_transform(preds) 139 | labels = scaler.inverse_transform(labels) 140 | return masked_rmse_tf(preds=preds, labels=labels, null_val=null_val) 141 | 142 | return loss 143 | 144 | 145 | def masked_mae_loss(scaler, null_val): 146 | def loss(preds, labels): 147 | if scaler: 148 | preds = scaler.inverse_transform(preds) 149 | labels = scaler.inverse_transform(labels) 150 | mae = masked_mae_tf(preds=preds, labels=labels, null_val=null_val) 151 | return mae 152 | 153 | return loss 154 | 155 | 156 | def neg_log_gauss(): 157 | def loss(preds_mu, preds_sigma, labels): 158 | neg_gauss_ll = neg_log_gauss_tf(preds_mu=preds_mu, preds_sigma=preds_sigma, labels=labels) 159 | return neg_gauss_ll 160 | 161 | return loss 162 | 163 | 164 | def calculate_metrics(df_pred, df_test, null_val): 165 | """ 166 | Calculate the MAE, MAPE, RMSE 167 | :param df_pred: 168 | :param df_test: 169 | :param null_val: 170 | :return: 171 | """ 172 | mape = masked_mape_np(preds=df_pred.as_matrix(), labels=df_test.as_matrix(), null_val=null_val) 173 | mae = masked_mae_np(preds=df_pred.as_matrix(), labels=df_test.as_matrix(), null_val=null_val) 174 | rmse = masked_rmse_np(preds=df_pred.as_matrix(), labels=df_test.as_matrix(), null_val=null_val) 175 | return mae, mape, rmse -------------------------------------------------------------------------------- /lib/metrics_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import numpy as np 4 | import tensorflow as tf 5 | 6 | from lib import metrics 7 | 8 | 9 | class MyTestCase(unittest.TestCase): 10 | def test_masked_mape_np(self): 11 | preds = np.array([ 12 | [1, 2, 2], 13 | [3, 4, 5], 14 | ], dtype=np.float32) 15 | labels = np.array([ 16 | [1, 2, 2], 17 | [3, 4, 4] 18 | ], dtype=np.float32) 19 | mape = metrics.masked_mape_np(preds=preds, labels=labels) 20 | self.assertAlmostEqual(1 / 24.0, mape, delta=1e-5) 21 | 22 | def test_masked_mape_np2(self): 23 | preds = np.array([ 24 | [1, 2, 2], 25 | [3, 4, 5], 26 | ], dtype=np.float32) 27 | labels = np.array([ 28 | [1, 2, 2], 29 | [3, 4, 4] 30 | ], dtype=np.float32) 31 | mape = metrics.masked_mape_np(preds=preds, labels=labels, null_val=4) 32 | self.assertEqual(0., mape) 33 | 34 | def test_masked_mape_np_all_zero(self): 35 | preds = np.array([ 36 | [1, 2], 37 | [3, 4], 38 | ], dtype=np.float32) 39 | labels = np.array([ 40 | [0, 0], 41 | [0, 0] 42 | ], dtype=np.float32) 43 | mape = metrics.masked_mape_np(preds=preds, labels=labels, null_val=0) 44 | self.assertEqual(0., mape) 45 | 46 | def test_masked_mape_np_all_nan(self): 47 | preds = np.array([ 48 | [1, 2], 49 | [3, 4], 50 | ], dtype=np.float32) 51 | labels = np.array([ 52 | [np.nan, np.nan], 53 | [np.nan, np.nan] 54 | ], dtype=np.float32) 55 | mape = metrics.masked_mape_np(preds=preds, labels=labels) 56 | self.assertEqual(0., mape) 57 | 58 | def test_masked_mape_np_nan(self): 59 | preds = np.array([ 60 | [1, 2], 61 | [3, 4], 62 | ], dtype=np.float32) 63 | labels = np.array([ 64 | [np.nan, np.nan], 65 | [np.nan, 3] 66 | ], dtype=np.float32) 67 | mape = metrics.masked_mape_np(preds=preds, labels=labels) 68 | self.assertAlmostEqual(1 / 3., mape, delta=1e-5) 69 | 70 | def test_masked_rmse_np_vanilla(self): 71 | preds = np.array([ 72 | [1, 2], 73 | [3, 4], 74 | ], dtype=np.float32) 75 | labels = np.array([ 76 | [1, 4], 77 | [3, 4] 78 | ], dtype=np.float32) 79 | mape = metrics.masked_rmse_np(preds=preds, labels=labels, null_val=0) 80 | self.assertEqual(1., mape) 81 | 82 | def test_masked_rmse_np_nan(self): 83 | preds = np.array([ 84 | [1, 2], 85 | [3, 4], 86 | ], dtype=np.float32) 87 | labels = np.array([ 88 | [1, np.nan], 89 | [3, 4] 90 | ], dtype=np.float32) 91 | rmse = metrics.masked_rmse_np(preds=preds, labels=labels) 92 | self.assertEqual(0., rmse) 93 | 94 | def test_masked_rmse_np_all_zero(self): 95 | preds = np.array([ 96 | [1, 2], 97 | [3, 4], 98 | ], dtype=np.float32) 99 | labels = np.array([ 100 | [0, 0], 101 | [0, 0] 102 | ], dtype=np.float32) 103 | mape = metrics.masked_rmse_np(preds=preds, labels=labels, null_val=0) 104 | self.assertEqual(0., mape) 105 | 106 | def test_masked_rmse_np_missing(self): 107 | preds = np.array([ 108 | [1, 2], 109 | [3, 4], 110 | ], dtype=np.float32) 111 | labels = np.array([ 112 | [1, 0], 113 | [3, 4] 114 | ], dtype=np.float32) 115 | mape = metrics.masked_rmse_np(preds=preds, labels=labels, null_val=0) 116 | self.assertEqual(0., mape) 117 | 118 | def test_masked_rmse_np2(self): 119 | preds = np.array([ 120 | [1, 2], 121 | [3, 4], 122 | ], dtype=np.float32) 123 | labels = np.array([ 124 | [1, 0], 125 | [3, 3] 126 | ], dtype=np.float32) 127 | rmse = metrics.masked_rmse_np(preds=preds, labels=labels, null_val=0) 128 | self.assertAlmostEqual(np.sqrt(1 / 3.), rmse, delta=1e-5) 129 | 130 | 131 | class TFRMSETestCase(unittest.TestCase): 132 | def test_masked_mse_null(self): 133 | with tf.Session() as sess: 134 | preds = tf.constant(np.array([ 135 | [1, 2], 136 | [3, 4], 137 | ], dtype=np.float32)) 138 | labels = tf.constant(np.array([ 139 | [1, 0], 140 | [3, 3] 141 | ], dtype=np.float32)) 142 | rmse = metrics.masked_mse_tf(preds=preds, labels=labels, null_val=0) 143 | self.assertAlmostEqual(1 / 3.0, sess.run(rmse), delta=1e-5) 144 | 145 | def test_masked_mse_vanilla(self): 146 | with tf.Session() as sess: 147 | preds = tf.constant(np.array([ 148 | [1, 2], 149 | [3, 4], 150 | ], dtype=np.float32)) 151 | labels = tf.constant(np.array([ 152 | [1, 0], 153 | [3, 3] 154 | ], dtype=np.float32)) 155 | rmse = metrics.masked_mse_tf(preds=preds, labels=labels) 156 | self.assertAlmostEqual(1.25, sess.run(rmse), delta=1e-5) 157 | 158 | def test_masked_mse_all_zero(self): 159 | with tf.Session() as sess: 160 | preds = tf.constant(np.array([ 161 | [1, 2], 162 | [3, 4], 163 | ], dtype=np.float32)) 164 | labels = tf.constant(np.array([ 165 | [0, 0], 166 | [0, 0] 167 | ], dtype=np.float32)) 168 | rmse = metrics.masked_mse_tf(preds=preds, labels=labels, null_val=0) 169 | self.assertAlmostEqual(0., sess.run(rmse), delta=1e-5) 170 | 171 | def test_masked_mse_nan(self): 172 | with tf.Session() as sess: 173 | preds = tf.constant(np.array([ 174 | [1, 2], 175 | [3, 4], 176 | ], dtype=np.float32)) 177 | labels = tf.constant(np.array([ 178 | [1, 2], 179 | [3, np.nan] 180 | ], dtype=np.float32)) 181 | rmse = metrics.masked_mse_tf(preds=preds, labels=labels) 182 | self.assertAlmostEqual(0., sess.run(rmse), delta=1e-5) 183 | 184 | def test_masked_mse_all_nan(self): 185 | with tf.Session() as sess: 186 | preds = tf.constant(np.array([ 187 | [1, 2], 188 | [3, 4], 189 | ], dtype=np.float32)) 190 | labels = tf.constant(np.array([ 191 | [np.nan, np.nan], 192 | [np.nan, np.nan] 193 | ], dtype=np.float32)) 194 | rmse = metrics.masked_mse_tf(preds=preds, labels=labels, null_val=0) 195 | self.assertAlmostEqual(0., sess.run(rmse), delta=1e-5) 196 | 197 | if __name__ == '__main__': 198 | unittest.main() 199 | -------------------------------------------------------------------------------- /inference.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import os 4 | from os.path import join, exists 5 | import sys 6 | import glob 7 | from eval_metrics.metrics import eval_deter 8 | from eval_metrics.metrics_sample import eval_samples 9 | import re 10 | 11 | if __name__ == '__main__': 12 | sys.path.append(os.getcwd()) 13 | parser = argparse.ArgumentParser() 14 | # ----------- args for load checkpoint of trained models 15 | parser.add_argument('--pretrained_dir', default=None, type=str, 16 | help='the config dir for trained model; if set as None, will be automatically set based on {ckpt_dir},{dataset},{trial_id}') 17 | parser.add_argument("--config_filename", default=None, type=str, help="The config file to used; if None, automatically find the latest checkpoint under {config_dir}") 18 | # ----------- if config_filename and pretrained_dir are not set; please ensure the following setting the same as the 'training.py' in order to evaluate the model on test set. 19 | parser.add_argument("--rnn_type", type=str, default=None, help="The type of rnn architecture of rnn_flow; if None, following the setting in the config file") 20 | parser.add_argument("--cost", type=str, default=None, help="The type of loss function (e.g., [mae], [nll]); if None, following the setting of the config file") 21 | parser.add_argument("--dataset", default="PEMSD8", help="name of datasets") 22 | parser.add_argument("--ckpt_dir", default="./ckpt", help="the dir to store checkpoints") 23 | parser.add_argument("--trial_id", type=int, default=1, help="id of the trial. Used as the random seed in multiple trials training") 24 | # ----------- the args to load data, please keep consistent with 'training.py' 25 | parser.add_argument("--data_dir", default="./data", help="the dir storing dataset") 26 | parser.add_argument("--graph_dir", default=None, help="the dir storing the graph information; if None, will be set as the '{data_dir}/sensor_graph'.") 27 | parser.add_argument("--output_dir", type=str, default="./result", help="The dir to store the output result") 28 | parser.add_argument('--output_filename', default=None, help="the name of output file; if None, automatically set as p{rnn_type}_{cost}_prediction_{dataset}_trial_{trial_id}.npz ") 29 | # ----------- 30 | parser.add_argument('--gpu_id', default=0, type=int, help='GPU id to use; by default using 0') 31 | parser.add_argument('--use_cpu_only', action="store_true", help='Add this if want to train in cpu only') 32 | args = parser.parse_args() 33 | 34 | if not args.use_cpu_only: 35 | os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu_id)# the GPU number 36 | else: 37 | os.environ["CUDA_VISIBLE_DEVICES"] = str(-1)# the GPU number 38 | 39 | if args.graph_dir is None: 40 | args.graph_dir = join(args.data_dir, "sensor_graph") 41 | 42 | args.dataset = args.dataset.lower() 43 | 44 | 45 | if args.pretrained_dir is None: 46 | args.pre_set = False 47 | if args.rnn_type is None or args.cost is None: 48 | raise NotImplementedError("[pretrained_dir] and [rnn_type & cost] cannot be None at the same time") 49 | args.pretrained_dir = join(args.ckpt_dir, args.dataset, args.rnn_type + "_" + args.cost, "trial_{}".format(args.trial_id)) 50 | else: 51 | args.pre_set = True 52 | 53 | import tensorflow as tf 54 | import yaml, os 55 | 56 | from lib.utils import load_graph_data 57 | from model.prnn_supervisor import PRNNSupervisor 58 | 59 | def run_inference(args): 60 | tf.reset_default_graph() 61 | if not os.path.exists(args.output_dir): 62 | os.mkdir(args.output_dir) 63 | 64 | 65 | 66 | if args.config_filename is None: 67 | args.config_file = config_finder(args.pretrained_dir) 68 | else: 69 | args.config_file = join(args.pretrained_dir, args.config_filename) 70 | 71 | 72 | 73 | with open(args.config_file) as f: 74 | config = yaml.load(f) 75 | 76 | 77 | tf_config = tf.ConfigProto() 78 | if args.use_cpu_only: 79 | tf_config = tf.ConfigProto(device_count={'GPU': 0}) 80 | tf_config.gpu_options.allow_growth = True 81 | 82 | args.cost = config["train"]["cost"] 83 | args.rnn_type = config["model"]["rnn_type"] 84 | try: 85 | dataset = config["data"]["dataset_name"] 86 | args.dataset = dataset 87 | except Exception: 88 | print("") 89 | 90 | 91 | graph_pkl_filename = join(args.graph_dir, "adj_" + args.dataset + ".pkl") 92 | config['data']["graph_pkl_filename"] = graph_pkl_filename 93 | 94 | adj_mx = load_graph_data(graph_pkl_filename).astype('float32') 95 | 96 | 97 | if args.output_filename is None: 98 | args.output_filename = get_output_filename(config, args) 99 | 100 | args.output_filename = join(args.output_dir, args.output_filename) 101 | 102 | 103 | 104 | print("Start evaluation on Test set") 105 | with tf.Session(config=tf_config) as sess: 106 | supervisor = PRNNSupervisor(adj_mx=adj_mx, args=args, inference=True, pretrained_dir=args.pretrained_dir, **config) 107 | supervisor.load(sess, config['train']['model_filename']) 108 | outputs = supervisor.evaluate(sess) 109 | np.savez_compressed(args.output_filename, **outputs) 110 | print('Predictions saved as {}.'.format(args.output_filename)) 111 | 112 | deter_metrics = eval_deter(args) 113 | sample_metrics = eval_samples(args) 114 | deter_lines = "\n".join(deter_metrics) 115 | sample_lines = "\n".join(sample_metrics) 116 | lines = "Rnn_type: {} Cost: {} Dataset: {}\n".format(args.rnn_type, args.cost, args.dataset) 117 | lines = lines + "\n-------------------------------\n" + deter_lines 118 | lines = lines + "\n-------------------------------\n" + sample_lines 119 | 120 | with open(args.output_filename.replace(".npz", "_result.txt"), "w") as f: 121 | f.writelines(lines) 122 | 123 | 124 | 125 | def config_finder(path): 126 | files = glob.glob(join(path, "**.yaml")) 127 | # print(files) 128 | if len(files) == 0: 129 | raise NotImplementedError("cannot find checkpoint files") 130 | 131 | index = [int(i.replace(join(path, "config_"), "").replace(".yaml","")) for i in files] 132 | max_id = np.argmax(index) 133 | config_file = files[max_id] 134 | return config_file 135 | 136 | def get_output_filename(config, args): 137 | rnn_type = config["model"]["rnn_type"] 138 | loss_type = config["train"]["cost"] 139 | dataset = args.dataset 140 | 141 | if args.pre_set: 142 | pre_set_dir = args.pretrained_dir.replace("/","_").replace("\\", "_").replace(".","").replace("ckpt","") 143 | pre_set_dir = re.sub("__*", "_", pre_set_dir) 144 | filename = "{}_{}_predictions_{}_{}.npz".format(rnn_type, loss_type, dataset, pre_set_dir) 145 | else: 146 | filename = "{}_{}_predictions_{}_trial_{}.npz".format(rnn_type, loss_type, dataset, args.trial_id) 147 | 148 | return filename 149 | 150 | 151 | 152 | 153 | if __name__ == '__main__': 154 | sys.path.append(os.getcwd()) 155 | 156 | run_inference(args) 157 | 158 | 159 | -------------------------------------------------------------------------------- /lib/AMSGrad.py: -------------------------------------------------------------------------------- 1 | """AMSGrad for TensorFlow. 2 | From: https://github.com/taki0112/AMSGrad-Tensorflow 3 | """ 4 | 5 | from tensorflow.python.eager import context 6 | from tensorflow.python.framework import ops 7 | from tensorflow.python.ops import control_flow_ops 8 | from tensorflow.python.ops import math_ops 9 | from tensorflow.python.ops import resource_variable_ops 10 | from tensorflow.python.ops import state_ops 11 | from tensorflow.python.ops import variable_scope 12 | from tensorflow.python.training import optimizer 13 | 14 | 15 | class AMSGrad(optimizer.Optimizer): 16 | def __init__(self, learning_rate=0.01, beta1=0.9, beta2=0.99, epsilon=1e-8, use_locking=False, name="AMSGrad"): 17 | super(AMSGrad, self).__init__(use_locking, name) 18 | self._lr = learning_rate 19 | self._beta1 = beta1 20 | self._beta2 = beta2 21 | self._epsilon = epsilon 22 | 23 | self._lr_t = None 24 | self._beta1_t = None 25 | self._beta2_t = None 26 | self._epsilon_t = None 27 | 28 | self._beta1_power = None 29 | self._beta2_power = None 30 | 31 | def _create_slots(self, var_list): 32 | first_var = min(var_list, key=lambda x: x.name) 33 | 34 | create_new = self._beta1_power is None 35 | if not create_new and context.in_graph_mode(): 36 | create_new = (self._beta1_power.graph is not first_var.graph) 37 | 38 | if create_new: 39 | with ops.colocate_with(first_var): 40 | self._beta1_power = variable_scope.variable(self._beta1, name="beta1_power", trainable=False) 41 | self._beta2_power = variable_scope.variable(self._beta2, name="beta2_power", trainable=False) 42 | # Create slots for the first and second moments. 43 | for v in var_list: 44 | self._zeros_slot(v, "m", self._name) 45 | self._zeros_slot(v, "v", self._name) 46 | self._zeros_slot(v, "vhat", self._name) 47 | 48 | def _prepare(self): 49 | self._lr_t = ops.convert_to_tensor(self._lr) 50 | self._beta1_t = ops.convert_to_tensor(self._beta1) 51 | self._beta2_t = ops.convert_to_tensor(self._beta2) 52 | self._epsilon_t = ops.convert_to_tensor(self._epsilon) 53 | 54 | def _apply_dense(self, grad, var): 55 | beta1_power = math_ops.cast(self._beta1_power, var.dtype.base_dtype) 56 | beta2_power = math_ops.cast(self._beta2_power, var.dtype.base_dtype) 57 | lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype) 58 | beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype) 59 | beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype) 60 | epsilon_t = math_ops.cast(self._epsilon_t, var.dtype.base_dtype) 61 | 62 | lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power)) 63 | 64 | # m_t = beta1 * m + (1 - beta1) * g_t 65 | m = self.get_slot(var, "m") 66 | m_scaled_g_values = grad * (1 - beta1_t) 67 | m_t = state_ops.assign(m, beta1_t * m + m_scaled_g_values, use_locking=self._use_locking) 68 | 69 | # v_t = beta2 * v + (1 - beta2) * (g_t * g_t) 70 | v = self.get_slot(var, "v") 71 | v_scaled_g_values = (grad * grad) * (1 - beta2_t) 72 | v_t = state_ops.assign(v, beta2_t * v + v_scaled_g_values, use_locking=self._use_locking) 73 | 74 | # amsgrad 75 | vhat = self.get_slot(var, "vhat") 76 | vhat_t = state_ops.assign(vhat, math_ops.maximum(v_t, vhat)) 77 | v_sqrt = math_ops.sqrt(vhat_t) 78 | 79 | var_update = state_ops.assign_sub(var, lr * m_t / (v_sqrt + epsilon_t), use_locking=self._use_locking) 80 | return control_flow_ops.group(*[var_update, m_t, v_t, vhat_t]) 81 | 82 | def _resource_apply_dense(self, grad, var): 83 | var = var.handle 84 | beta1_power = math_ops.cast(self._beta1_power, grad.dtype.base_dtype) 85 | beta2_power = math_ops.cast(self._beta2_power, grad.dtype.base_dtype) 86 | lr_t = math_ops.cast(self._lr_t, grad.dtype.base_dtype) 87 | beta1_t = math_ops.cast(self._beta1_t, grad.dtype.base_dtype) 88 | beta2_t = math_ops.cast(self._beta2_t, grad.dtype.base_dtype) 89 | epsilon_t = math_ops.cast(self._epsilon_t, grad.dtype.base_dtype) 90 | 91 | lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power)) 92 | 93 | # m_t = beta1 * m + (1 - beta1) * g_t 94 | m = self.get_slot(var, "m").handle 95 | m_scaled_g_values = grad * (1 - beta1_t) 96 | m_t = state_ops.assign(m, beta1_t * m + m_scaled_g_values, use_locking=self._use_locking) 97 | 98 | # v_t = beta2 * v + (1 - beta2) * (g_t * g_t) 99 | v = self.get_slot(var, "v").handle 100 | v_scaled_g_values = (grad * grad) * (1 - beta2_t) 101 | v_t = state_ops.assign(v, beta2_t * v + v_scaled_g_values, use_locking=self._use_locking) 102 | 103 | # amsgrad 104 | vhat = self.get_slot(var, "vhat").handle 105 | vhat_t = state_ops.assign(vhat, math_ops.maximum(v_t, vhat)) 106 | v_sqrt = math_ops.sqrt(vhat_t) 107 | 108 | var_update = state_ops.assign_sub(var, lr * m_t / (v_sqrt + epsilon_t), use_locking=self._use_locking) 109 | return control_flow_ops.group(*[var_update, m_t, v_t, vhat_t]) 110 | 111 | def _apply_sparse_shared(self, grad, var, indices, scatter_add): 112 | beta1_power = math_ops.cast(self._beta1_power, var.dtype.base_dtype) 113 | beta2_power = math_ops.cast(self._beta2_power, var.dtype.base_dtype) 114 | lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype) 115 | beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype) 116 | beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype) 117 | epsilon_t = math_ops.cast(self._epsilon_t, var.dtype.base_dtype) 118 | 119 | lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power)) 120 | 121 | # m_t = beta1 * m + (1 - beta1) * g_t 122 | m = self.get_slot(var, "m") 123 | m_scaled_g_values = grad * (1 - beta1_t) 124 | m_t = state_ops.assign(m, m * beta1_t, use_locking=self._use_locking) 125 | with ops.control_dependencies([m_t]): 126 | m_t = scatter_add(m, indices, m_scaled_g_values) 127 | 128 | # v_t = beta2 * v + (1 - beta2) * (g_t * g_t) 129 | v = self.get_slot(var, "v") 130 | v_scaled_g_values = (grad * grad) * (1 - beta2_t) 131 | v_t = state_ops.assign(v, v * beta2_t, use_locking=self._use_locking) 132 | with ops.control_dependencies([v_t]): 133 | v_t = scatter_add(v, indices, v_scaled_g_values) 134 | 135 | # amsgrad 136 | vhat = self.get_slot(var, "vhat") 137 | vhat_t = state_ops.assign(vhat, math_ops.maximum(v_t, vhat)) 138 | v_sqrt = math_ops.sqrt(vhat_t) 139 | var_update = state_ops.assign_sub(var, lr * m_t / (v_sqrt + epsilon_t), use_locking=self._use_locking) 140 | return control_flow_ops.group(*[var_update, m_t, v_t, vhat_t]) 141 | 142 | def _apply_sparse(self, grad, var): 143 | return self._apply_sparse_shared( 144 | grad.values, var, grad.indices, 145 | lambda x, i, v: state_ops.scatter_add( # pylint: disable=g-long-lambda 146 | x, i, v, use_locking=self._use_locking)) 147 | 148 | def _resource_scatter_add(self, x, i, v): 149 | with ops.control_dependencies( 150 | [resource_variable_ops.resource_scatter_add(x.handle, i, v)]): 151 | return x.value() 152 | 153 | def _resource_apply_sparse(self, grad, var, indices): 154 | return self._apply_sparse_shared( 155 | grad, var, indices, self._resource_scatter_add) 156 | 157 | def _finish(self, update_ops, name_scope): 158 | # Update the power accumulators. 159 | with ops.control_dependencies(update_ops): 160 | with ops.colocate_with(self._beta1_power): 161 | update_beta1 = self._beta1_power.assign( 162 | self._beta1_power * self._beta1_t, 163 | use_locking=self._use_locking) 164 | update_beta2 = self._beta2_power.assign( 165 | self._beta2_power * self._beta2_t, 166 | use_locking=self._use_locking) 167 | return control_flow_ops.group(*update_ops + [update_beta1, update_beta2], 168 | name=name_scope) 169 | -------------------------------------------------------------------------------- /model/prnn_cell.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import numpy as np 6 | import tensorflow as tf 7 | 8 | from tensorflow.contrib.rnn import RNNCell 9 | 10 | from lib import utils 11 | 12 | 13 | class DCGRUCell(RNNCell): 14 | """Graph Convolution Gated Recurrent Unit cell. 15 | """ 16 | 17 | def call(self, inputs, **kwargs): 18 | pass 19 | 20 | def compute_output_shape(self, input_shape): 21 | pass 22 | 23 | def __init__(self, num_units, adj_mx, max_diffusion_step, num_nodes, num_proj=None, 24 | activation=tf.nn.tanh, reuse=None, filter_type="laplacian", use_gc_for_ru=True): 25 | """ 26 | 27 | :param num_units: 28 | :param adj_mx: 29 | :param max_diffusion_step: 30 | :param num_nodes: 31 | :param input_size: 32 | :param num_proj: 33 | :param activation: 34 | :param reuse: 35 | :param filter_type: "laplacian", "random_walk", "dual_random_walk". 36 | :param use_gc_for_ru: whether to use Graph convolution to calculate the reset and update gates. 37 | """ 38 | super(DCGRUCell, self).__init__(_reuse=reuse) 39 | self._activation = activation 40 | self._num_nodes = num_nodes 41 | self._num_proj = num_proj 42 | self._num_units = num_units 43 | self._max_diffusion_step = max_diffusion_step 44 | self._supports = [] 45 | self._use_gc_for_ru = use_gc_for_ru 46 | supports = [] 47 | if filter_type == "laplacian": 48 | supports.append(utils.calculate_scaled_laplacian(adj_mx, lambda_max=None)) 49 | elif filter_type == "random_walk": 50 | supports.append(utils.calculate_random_walk_matrix(adj_mx).T) 51 | elif filter_type == "dual_random_walk": 52 | supports.append(utils.calculate_random_walk_matrix(adj_mx).T) 53 | supports.append(utils.calculate_random_walk_matrix(adj_mx.T).T) 54 | else: 55 | supports.append(utils.calculate_scaled_laplacian(adj_mx)) 56 | for support in supports: 57 | self._supports.append(self._build_sparse_matrix(support)) 58 | 59 | @staticmethod 60 | def _build_sparse_matrix(L): 61 | L = L.tocoo() 62 | indices = np.column_stack((L.row, L.col)) 63 | L = tf.SparseTensor(indices, L.data, L.shape) 64 | return tf.sparse_reorder(L) 65 | 66 | @property 67 | def state_size(self): 68 | return self._num_nodes * self._num_units 69 | 70 | @property 71 | def output_size(self): 72 | output_size = self._num_nodes * self._num_units 73 | if self._num_proj is not None: 74 | output_size = self._num_nodes * self._num_proj 75 | return output_size 76 | 77 | def __call__(self, inputs, state, scope=None): 78 | """Gated recurrent unit (GRU) with Graph Convolution. 79 | :param inputs: (B, num_nodes * input_dim) 80 | 81 | :return 82 | - Output: A `2-D` tensor with shape `[batch_size x self.output_size]`. 83 | - New state: Either a single `2-D` tensor, or a tuple of tensors matching 84 | the arity and shapes of `state` 85 | """ 86 | with tf.variable_scope(scope or "dcgru_cell"): 87 | with tf.variable_scope("gates"): # Reset gate and update gate. 88 | output_size = 2 * self._num_units 89 | # We start with bias of 1.0 to not reset and not update. 90 | if self._use_gc_for_ru: 91 | fn = self._gconv 92 | else: 93 | fn = self._fc 94 | value = tf.nn.sigmoid(fn(inputs, state, output_size, bias_start=1.0)) 95 | value = tf.reshape(value, (-1, self._num_nodes, output_size)) 96 | r, u = tf.split(value=value, num_or_size_splits=2, axis=-1) 97 | r = tf.reshape(r, (-1, self._num_nodes * self._num_units)) 98 | u = tf.reshape(u, (-1, self._num_nodes * self._num_units)) 99 | with tf.variable_scope("candidate"): 100 | c = self._gconv(inputs, r * state, self._num_units) 101 | if self._activation is not None: 102 | c = self._activation(c) 103 | output = new_state = u * state + (1 - u) * c 104 | if self._num_proj is not None: 105 | with tf.variable_scope("projection"): 106 | w = tf.get_variable('w', shape=(self._num_units, self._num_proj)) 107 | batch_size = inputs.get_shape()[0].value 108 | output = tf.reshape(new_state, shape=(-1, self._num_units)) 109 | output = tf.reshape(tf.matmul(output, w), shape=(batch_size, self.output_size)) 110 | return output, new_state 111 | 112 | @staticmethod 113 | def _concat(x, x_): 114 | x_ = tf.expand_dims(x_, 0) 115 | return tf.concat([x, x_], axis=0) 116 | 117 | def _fc(self, inputs, state, output_size, bias_start=0.0): 118 | dtype = inputs.dtype 119 | batch_size = inputs.get_shape()[0].value 120 | inputs = tf.reshape(inputs, (batch_size * self._num_nodes, -1)) 121 | state = tf.reshape(state, (batch_size * self._num_nodes, -1)) 122 | inputs_and_state = tf.concat([inputs, state], axis=-1) 123 | input_size = inputs_and_state.get_shape()[-1].value 124 | weights = tf.get_variable( 125 | 'weights', [input_size, output_size], dtype=dtype, 126 | initializer=tf.contrib.layers.xavier_initializer()) 127 | value = tf.nn.sigmoid(tf.matmul(inputs_and_state, weights)) 128 | biases = tf.get_variable("biases", [output_size], dtype=dtype, 129 | initializer=tf.constant_initializer(bias_start, dtype=dtype)) 130 | value = tf.nn.bias_add(value, biases) 131 | return value 132 | 133 | def _gconv(self, inputs, state, output_size, bias_start=0.0): 134 | """Graph convolution between input and the graph matrix. 135 | 136 | :param args: a 2D Tensor or a list of 2D, batch x n, Tensors. 137 | :param output_size: 138 | :param bias: 139 | :param bias_start: 140 | :param scope: 141 | :return: 142 | """ 143 | # Reshape input and state to (batch_size, num_nodes, input_dim/state_dim) 144 | batch_size = inputs.get_shape()[0].value 145 | inputs = tf.reshape(inputs, (batch_size, self._num_nodes, -1)) 146 | state = tf.reshape(state, (batch_size, self._num_nodes, -1)) 147 | inputs_and_state = tf.concat([inputs, state], axis=2) 148 | input_size = inputs_and_state.get_shape()[2].value 149 | dtype = inputs.dtype 150 | 151 | x = inputs_and_state 152 | x0 = tf.transpose(x, perm=[1, 2, 0]) # (num_nodes, total_arg_size, batch_size) 153 | x0 = tf.reshape(x0, shape=[self._num_nodes, input_size * batch_size]) 154 | x = tf.expand_dims(x0, axis=0) 155 | 156 | scope = tf.get_variable_scope() 157 | with tf.variable_scope(scope): 158 | if self._max_diffusion_step == 0: 159 | pass 160 | else: 161 | for support in self._supports: 162 | x1 = tf.sparse_tensor_dense_matmul(support, x0) 163 | x = self._concat(x, x1) 164 | 165 | for k in range(2, self._max_diffusion_step + 1): 166 | x2 = 2 * tf.sparse_tensor_dense_matmul(support, x1) - x0 167 | x = self._concat(x, x2) 168 | x1, x0 = x2, x1 169 | 170 | num_matrices = len(self._supports) * self._max_diffusion_step + 1 # Adds for x itself. 171 | x = tf.reshape(x, shape=[num_matrices, self._num_nodes, input_size, batch_size]) 172 | x = tf.transpose(x, perm=[3, 1, 2, 0]) # (batch_size, num_nodes, input_size, order) 173 | x = tf.reshape(x, shape=[batch_size * self._num_nodes, input_size * num_matrices]) 174 | 175 | weights = tf.get_variable( 176 | 'weights', [input_size * num_matrices, output_size], dtype=dtype, 177 | initializer=tf.contrib.layers.xavier_initializer()) 178 | x = tf.matmul(x, weights) # (batch_size * self._num_nodes, output_size) 179 | 180 | biases = tf.get_variable("biases", [output_size], dtype=dtype, 181 | initializer=tf.constant_initializer(bias_start, dtype=dtype)) 182 | x = tf.nn.bias_add(x, biases) 183 | # Reshape res back to 2D: (batch_size, num_node, state_dim) -> (batch_size, num_node * state_dim) 184 | return tf.reshape(x, [batch_size, self._num_nodes * output_size]) 185 | -------------------------------------------------------------------------------- /lib/utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import numpy as np 3 | import os 4 | import pickle 5 | import scipy.sparse as sp 6 | import sys 7 | import tensorflow as tf 8 | 9 | from scipy.sparse import linalg 10 | 11 | 12 | class DataLoader(object): 13 | def __init__(self, xs, ys, batch_size, pad_with_last_sample=True, shuffle=False): 14 | """ 15 | 16 | :param xs: 17 | :param ys: 18 | :param batch_size: 19 | :param pad_with_last_sample: pad with the last sample to make number of samples divisible to batch_size. 20 | """ 21 | self.batch_size = batch_size 22 | self.current_ind = 0 23 | if pad_with_last_sample: 24 | num_padding = (batch_size - (len(xs) % batch_size)) % batch_size 25 | x_padding = np.repeat(xs[-1:], num_padding, axis=0) 26 | y_padding = np.repeat(ys[-1:], num_padding, axis=0) 27 | xs = np.concatenate([xs, x_padding], axis=0) 28 | ys = np.concatenate([ys, y_padding], axis=0) 29 | self.size = len(xs) 30 | self.num_batch = int(self.size // self.batch_size) 31 | if shuffle: 32 | permutation = np.random.permutation(self.size) 33 | xs, ys = xs[permutation], ys[permutation] 34 | self.xs = xs 35 | self.ys = ys 36 | 37 | def get_iterator(self): 38 | self.current_ind = 0 39 | 40 | def _wrapper(): 41 | while self.current_ind < self.num_batch: 42 | start_ind = self.batch_size * self.current_ind 43 | end_ind = min(self.size, self.batch_size * (self.current_ind + 1)) 44 | x_i = self.xs[start_ind: end_ind, ...] 45 | y_i = self.ys[start_ind: end_ind, ...] 46 | yield (x_i, y_i) 47 | self.current_ind += 1 48 | 49 | return _wrapper() 50 | 51 | 52 | class StandardScaler: 53 | """ 54 | Standard the input 55 | """ 56 | 57 | def __init__(self, mean, std): 58 | self.mean = mean 59 | self.std = std 60 | 61 | def transform(self, data): 62 | return (data - self.mean) / self.std 63 | 64 | def inverse_transform(self, data): 65 | return (data * self.std) + self.mean 66 | 67 | 68 | class StandardScalerColumn: 69 | """ 70 | Standard the input 71 | """ 72 | 73 | def __init__(self, mean, std): 74 | self.mean = mean 75 | self.std = std 76 | 77 | def transform(self, data): 78 | mu = self.mean[np.newaxis, np.newaxis, :] 79 | sigma = self.std[np.newaxis, np.newaxis, :] 80 | return (data - mu) / sigma 81 | 82 | def inverse_transform(self, data): 83 | mu = self.mean[np.newaxis, np.newaxis, :] 84 | sigma = self.std[np.newaxis, np.newaxis, :] 85 | return (data * sigma) + mu 86 | 87 | 88 | def add_simple_summary(writer, names, values, global_step): 89 | """ 90 | Writes summary for a list of scalars. 91 | :param writer: 92 | :param names: 93 | :param values: 94 | :param global_step: 95 | :return: 96 | """ 97 | for name, value in zip(names, values): 98 | summary = tf.Summary() 99 | summary_value = summary.value.add() 100 | summary_value.simple_value = value 101 | summary_value.tag = name 102 | writer.add_summary(summary, global_step) 103 | 104 | 105 | def calculate_normalized_laplacian(adj): 106 | """ 107 | # L = D^-1/2 (D-A) D^-1/2 = I - D^-1/2 A D^-1/2 108 | # D = diag(A 1) 109 | :param adj: 110 | :return: 111 | """ 112 | adj = sp.coo_matrix(adj) 113 | d = np.array(adj.sum(1)) 114 | d_inv_sqrt = np.power(d, -0.5).flatten() 115 | d_inv_sqrt[np.isinf(d_inv_sqrt)] = 0. 116 | d_mat_inv_sqrt = sp.diags(d_inv_sqrt) 117 | normalized_laplacian = sp.eye(adj.shape[0]) - adj.dot(d_mat_inv_sqrt).transpose().dot(d_mat_inv_sqrt).tocoo() 118 | return normalized_laplacian 119 | 120 | 121 | def calculate_random_walk_matrix(adj_mx): 122 | adj_mx = sp.coo_matrix(adj_mx) 123 | d = np.array(adj_mx.sum(1)) 124 | d_inv = np.power(d, -1).flatten() 125 | d_inv[np.isinf(d_inv)] = 0. 126 | d_mat_inv = sp.diags(d_inv) 127 | random_walk_mx = d_mat_inv.dot(adj_mx).tocoo() 128 | return random_walk_mx 129 | 130 | 131 | def calculate_reverse_random_walk_matrix(adj_mx): 132 | return calculate_random_walk_matrix(np.transpose(adj_mx)) 133 | 134 | 135 | def calculate_scaled_laplacian(adj_mx, lambda_max=2, undirected=True): 136 | if undirected: 137 | adj_mx = np.maximum.reduce([adj_mx, adj_mx.T]) 138 | L = calculate_normalized_laplacian(adj_mx) 139 | if lambda_max is None: 140 | lambda_max, _ = linalg.eigsh(L, 1, which='LM') 141 | lambda_max = lambda_max[0] 142 | L = sp.csr_matrix(L) 143 | M, _ = L.shape 144 | I = sp.identity(M, format='csr', dtype=L.dtype) 145 | L = (2 / lambda_max * L) - I 146 | return L.astype(np.float32) 147 | 148 | 149 | def config_logging(log_dir, log_filename='info.log', level=logging.INFO): 150 | # Add file handler and stdout handler 151 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') 152 | # Create the log directory if necessary. 153 | try: 154 | os.makedirs(log_dir) 155 | except OSError: 156 | pass 157 | file_handler = logging.FileHandler(os.path.join(log_dir, log_filename)) 158 | file_handler.setFormatter(formatter) 159 | file_handler.setLevel(level=level) 160 | # Add console handler. 161 | console_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') 162 | console_handler = logging.StreamHandler(sys.stdout) 163 | console_handler.setFormatter(console_formatter) 164 | console_handler.setLevel(level=level) 165 | logging.basicConfig(handlers=[file_handler, console_handler], level=level) 166 | 167 | 168 | def get_logger(log_dir, name, log_filename='info.log', level=logging.INFO): 169 | logger = logging.getLogger(name) 170 | logger.setLevel(level) 171 | # Add file handler and stdout handler 172 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') 173 | file_handler = logging.FileHandler(os.path.join(log_dir, log_filename)) 174 | file_handler.setFormatter(formatter) 175 | # Add console handler. 176 | console_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') 177 | console_handler = logging.StreamHandler(sys.stdout) 178 | console_handler.setFormatter(console_formatter) 179 | logger.addHandler(file_handler) 180 | logger.addHandler(console_handler) 181 | # Add google cloud log handler 182 | logger.info('Log directory: %s', log_dir) 183 | return logger 184 | 185 | 186 | def get_total_trainable_parameter_size(): 187 | """ 188 | Calculates the total number of trainable parameters in the current graph. 189 | :return: 190 | """ 191 | total_parameters = 0 192 | for variable in tf.trainable_variables(): 193 | # shape is an array of tf.Dimension 194 | total_parameters += np.product([x.value for x in variable.get_shape()]) 195 | return int(total_parameters) 196 | 197 | 198 | def load_dataset(dataset_dir, batch_size, test_batch_size=None, **kwargs): 199 | data = {} 200 | for category in ['train', 'val', 'test']: 201 | cat_data = np.load(os.path.join(dataset_dir, category + '.npz')) 202 | data['x_' + category] = cat_data['x'] 203 | data['y_' + category] = cat_data['y'] 204 | 205 | scaler = StandardScaler(mean=data['x_train'][..., 0].mean(), std=data['x_train'][..., 0].std()) 206 | 207 | for category in ['train', 'val', 'test']: 208 | data['x_' + category][..., 0] = scaler.transform(data['x_' + category][..., 0]) 209 | data['y_' + category][..., 0] = scaler.transform(data['y_' + category][..., 0]) 210 | print(np.mean(data['x_' + category][..., 0])) 211 | print(np.std(data['x_' + category][..., 0])) 212 | print(np.mean(data['y_' + category][..., 0])) 213 | print(np.std(data['y_' + category][..., 0])) 214 | print(np.sum(1 * np.isnan(data['x_' + category]))) 215 | print(np.sum(1 * np.isinf(data['x_' + category]))) 216 | print(np.sum(1 * np.isnan(data['y_' + category]))) 217 | print(np.sum(1 * np.isinf(data['y_' + category]))) 218 | data['train_loader'] = DataLoader(data['x_train'], data['y_train'], batch_size, shuffle=True) 219 | data['val_loader'] = DataLoader(data['x_val'], data['y_val'], test_batch_size, shuffle=False) 220 | data['test_loader'] = DataLoader(data['x_test'], data['y_test'], test_batch_size, shuffle=False) 221 | data['scaler'] = scaler 222 | 223 | return data 224 | 225 | 226 | def load_graph_data(pkl_filename): 227 | # sensor_ids, sensor_id_to_ind, adj_mx = load_pickle(pkl_filename) 228 | adj_mx = load_pickle(pkl_filename) 229 | 230 | return adj_mx 231 | 232 | 233 | def load_pickle(pickle_file): 234 | try: 235 | with open(pickle_file, 'rb') as f: 236 | pickle_data = pickle.load(f) 237 | except UnicodeDecodeError as e: 238 | with open(pickle_file, 'rb') as f: 239 | pickle_data = pickle.load(f, encoding='latin1') 240 | except Exception as e: 241 | print('Unable to load data ', pickle_file, ':', e) 242 | raise 243 | return pickle_data 244 | -------------------------------------------------------------------------------- /eval_metrics/crps.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import warnings 4 | import contextlib 5 | 6 | from scipy import special, integrate, stats 7 | 8 | # The normalization constant for the univariate standard Gaussian pdf 9 | _normconst = 1.0 / np.sqrt(2.0 * np.pi) 10 | 11 | 12 | def _normpdf(x): 13 | """Probability density function of a univariate standard Gaussian 14 | distribution with zero mean and unit variance. 15 | """ 16 | return _normconst * np.exp(-(x * x) / 2.0) 17 | 18 | 19 | # Cumulative distribution function of a univariate standard Gaussian 20 | # distribution with zero mean and unit variance. 21 | _normcdf = special.ndtr 22 | 23 | 24 | def crps_gaussian(x, mu, sig, grad=False): 25 | """ 26 | Computes the CRPS of observations x relative to normally distributed 27 | forecasts with mean, mu, and standard deviation, sig. 28 | 29 | CRPS(N(mu, sig^2); x) 30 | 31 | Formula taken from Equation (5): 32 | 33 | Calibrated Probablistic Forecasting Using Ensemble Model Output 34 | Statistics and Minimum CRPS Estimation. Gneiting, Raftery, 35 | Westveld, Goldman. Monthly Weather Review 2004 36 | 37 | http://journals.ametsoc.org/doi/pdf/10.1175/MWR2904.1 38 | 39 | Parameters 40 | ---------- 41 | x : scalar or np.ndarray 42 | The observation or set of observations. 43 | mu : scalar or np.ndarray 44 | The mean of the forecast normal distribution 45 | sig : scalar or np.ndarray 46 | The standard deviation of the forecast distribution 47 | grad : boolean 48 | If True the gradient of the CRPS w.r.t. mu and sig 49 | is returned along with the CRPS. 50 | 51 | Returns 52 | ------- 53 | crps : scalar or np.ndarray or tuple of 54 | The CRPS of each observation x relative to mu and sig. 55 | The shape of the output array is determined by numpy 56 | broadcasting rules. 57 | crps_grad : np.ndarray (optional) 58 | If grad=True the gradient of the crps is returned as 59 | a numpy array [grad_wrt_mu, grad_wrt_sig]. The 60 | same broadcasting rules apply. 61 | """ 62 | x = np.asarray(x) 63 | mu = np.asarray(mu) 64 | sig = np.asarray(sig) 65 | # standadized x 66 | sx = (x - mu) / sig 67 | # some precomputations to speed up the gradient 68 | pdf = _normpdf(sx) 69 | cdf = _normcdf(sx) 70 | pi_inv = 1. / np.sqrt(np.pi) 71 | # the actual crps 72 | crps = sig * (sx * (2 * cdf - 1) + 2 * pdf - pi_inv) 73 | if grad: 74 | dmu = 1 - 2 * cdf 75 | dsig = 2 * pdf - pi_inv 76 | return crps, np.array([dmu, dsig]) 77 | else: 78 | return crps 79 | 80 | 81 | def _discover_bounds(cdf, tol=1e-7): 82 | """ 83 | Uses scipy's general continuous distribution methods 84 | which compute the ppf from the cdf, then use the ppf 85 | to find the lower and upper limits of the distribution. 86 | """ 87 | class DistFromCDF(stats.distributions.rv_continuous): 88 | def cdf(self, x): 89 | return cdf(x) 90 | dist = DistFromCDF() 91 | # the ppf is the inverse cdf 92 | lower = dist.ppf(tol) 93 | upper = dist.ppf(1. - tol) 94 | return lower, upper 95 | 96 | 97 | def _crps_cdf_single(x, cdf_or_dist, xmin=None, xmax=None, tol=1e-6): 98 | """ 99 | See crps_cdf for docs. 100 | """ 101 | # TODO: this function is pretty slow. Look for clever ways to speed it up. 102 | 103 | # allow for directly passing in scipy.stats distribution objects. 104 | cdf = getattr(cdf_or_dist, 'cdf', cdf_or_dist) 105 | assert callable(cdf) 106 | 107 | # if bounds aren't given, discover them 108 | if xmin is None or xmax is None: 109 | # Note that infinite values for xmin and xmax are valid, but 110 | # it slows down the resulting quadrature significantly. 111 | xmin, xmax = _discover_bounds(cdf) 112 | 113 | # make sure the bounds haven't clipped the cdf. 114 | if (tol is not None) and (cdf(xmin) >= tol) or (cdf(xmax) <= (1. - tol)): 115 | raise ValueError('CDF does not meet tolerance requirements at %s ' 116 | 'extreme(s)! Consider using function defaults ' 117 | 'or using infinities at the bounds. ' 118 | % ('lower' if cdf(xmin) >= tol else 'upper')) 119 | 120 | # CRPS = int_-inf^inf (F(y) - H(x))**2 dy 121 | # = int_-inf^x F(y)**2 dy + int_x^inf (1 - F(y))**2 dy 122 | def lhs(y): 123 | # left hand side of CRPS integral 124 | return np.square(cdf(y)) 125 | # use quadrature to integrate the lhs 126 | lhs_int, lhs_tol = integrate.quad(lhs, xmin, x) 127 | # make sure the resulting CRPS will be with tolerance 128 | if (tol is not None) and (lhs_tol >= 0.5 * tol): 129 | raise ValueError('Lower integral did not evaluate to within tolerance! ' 130 | 'Tolerance achieved: %f , Value of integral: %f \n' 131 | 'Consider setting the lower bound to -np.inf.' % 132 | (lhs_tol, lhs_int)) 133 | 134 | def rhs(y): 135 | # right hand side of CRPS integral 136 | return np.square(1. - cdf(y)) 137 | rhs_int, rhs_tol = integrate.quad(rhs, x, xmax) 138 | # make sure the resulting CRPS will be with tolerance 139 | if (tol is not None) and (rhs_tol >= 0.5 * tol): 140 | raise ValueError('Upper integral did not evaluate to within tolerance! \n' 141 | 'Tolerance achieved: %f , Value of integral: %f \n' 142 | 'Consider setting the upper bound to np.inf or if ' 143 | 'you already have, set warn_level to `ignore`.' % 144 | (rhs_tol, rhs_int)) 145 | 146 | return lhs_int + rhs_int 147 | 148 | _crps_cdf = np.vectorize(_crps_cdf_single) 149 | 150 | 151 | def crps_quadrature(x, cdf_or_dist, xmin=None, xmax=None, tol=1e-6): 152 | """ 153 | Compute the continuously ranked probability score (CPRS) for a given 154 | forecast distribution (cdf) and observation (x) using numerical quadrature. 155 | 156 | This implementation allows the computation of CRPS for arbitrary forecast 157 | distributions. If gaussianity can be assumed ``crps_gaussian`` is faster. 158 | 159 | Parameters 160 | ---------- 161 | x : np.ndarray 162 | Observations associated with the forecast distribution cdf_or_dist 163 | cdf_or_dist : callable or scipy.stats.distribution 164 | Function which returns the the cumulative density of the 165 | forecast distribution at value x. This can also be an object with 166 | a callable cdf() method such as a scipy.stats.distribution object. 167 | xmin : np.ndarray or scalar 168 | The lower bounds for integration, this is required to perform 169 | quadrature. 170 | xmax : np.ndarray or scalar 171 | The upper bounds for integration, this is required to perform 172 | quadrature. 173 | tol : float , optional 174 | The desired accuracy of the CRPS, larger values will speed 175 | up integration. If tol is set to None, bounds errors or integration 176 | tolerance errors will be ignored. 177 | 178 | Returns 179 | ------- 180 | crps : np.ndarray 181 | The continuously ranked probability score of an observation x 182 | given forecast distribution. 183 | """ 184 | return _crps_cdf(x, cdf_or_dist, xmin, xmax, tol) 185 | 186 | 187 | def _crps_ensemble_vectorized(observations, forecasts, weights=1): 188 | """ 189 | An alternative but simpler implementation of CRPS for testing purposes 190 | 191 | This implementation is based on the identity: 192 | 193 | .. math:: 194 | CRPS(F, x) = E_F|X - x| - 1/2 * E_F|X - X'| 195 | 196 | where X and X' denote independent random variables drawn from the forecast 197 | distribution F, and E_F denotes the expectation value under F. 198 | 199 | Hence it has runtime O(n^2) instead of O(n log(n)) where n is the number of 200 | ensemble members. 201 | 202 | Reference 203 | --------- 204 | Tilmann Gneiting and Adrian E. Raftery. Strictly proper scoring rules, 205 | prediction, and estimation, 2005. University of Washington Department of 206 | Statistics Technical Report no. 463R. 207 | https://www.stat.washington.edu/research/reports/2004/tr463R.pdf 208 | """ 209 | observations = np.asarray(observations) 210 | forecasts = np.asarray(forecasts) 211 | weights = np.asarray(weights) 212 | if weights.ndim > 0: 213 | weights = np.where(~np.isnan(forecasts), weights, np.nan) 214 | weights = weights / np.nanmean(weights, axis=-1, keepdims=True) 215 | 216 | if observations.ndim == forecasts.ndim - 1: 217 | # sum over the last axis 218 | assert observations.shape == forecasts.shape[:-1] 219 | observations = observations[..., np.newaxis] 220 | with suppress_warnings('Mean of empty slice'): 221 | score = np.nanmean(weights * abs(forecasts - observations), -1) 222 | # insert new axes along last and second to last forecast dimensions so 223 | # forecasts_diff expands with the array broadcasting 224 | forecasts_diff = (np.expand_dims(forecasts, -1) - 225 | np.expand_dims(forecasts, -2)) 226 | weights_matrix = (np.expand_dims(weights, -1) * 227 | np.expand_dims(weights, -2)) 228 | with suppress_warnings('Mean of empty slice'): 229 | score += -0.5 * np.nanmean(weights_matrix * abs(forecasts_diff), 230 | axis=(-2, -1)) 231 | return score 232 | elif observations.ndim == forecasts.ndim: 233 | # there is no 'realization' axis to sum over (this is a deterministic 234 | # forecast) 235 | return abs(observations - forecasts) 236 | 237 | 238 | try: 239 | from ._gufuncs import _crps_ensemble_gufunc as _crps_ensemble_core 240 | except ImportError: 241 | _crps_ensemble_core = _crps_ensemble_vectorized 242 | 243 | 244 | def crps_ensemble(observations, forecasts, weights=None, issorted=False, 245 | axis=-1): 246 | """ 247 | Calculate the continuous ranked probability score (CRPS) for a set of 248 | explicit forecast realizations. 249 | 250 | The CRPS compares the empirical distribution of an ensemble forecast 251 | to a scalar observation. Smaller scores indicate better skill. 252 | 253 | CRPS is defined for one-dimensional random variables with a probability 254 | density $p(x)$, 255 | 256 | .. math:: 257 | CRPS(F, x) = \int_z (F(z) - H(z - x))^2 dz 258 | 259 | where $F(x) = \int_{z \leq x} p(z) dz$ is the cumulative distribution 260 | function (CDF) of the forecast distribution $F$ and $H(x)$ denotes the 261 | Heaviside step function, where $x$ is a point estimate of the true 262 | observation (observational error is neglected). 263 | 264 | This function calculates CRPS efficiently using the empirical CDF: 265 | http://en.wikipedia.org/wiki/Empirical_distribution_function 266 | 267 | The Numba accelerated version of this function requires time 268 | O(N * E * log(E)) and space O(N * E) where N is the number of observations 269 | and E is the size of the forecast ensemble. 270 | 271 | The non-Numba accelerated version much slower for large ensembles: it 272 | requires both time and space O(N * E ** 2). 273 | 274 | Parameters 275 | ---------- 276 | observations : float or array_like 277 | Observations float or array. Missing values (NaN) are given scores of 278 | NaN. 279 | forecasts : float or array_like 280 | Array of forecasts ensemble members, of the same shape as observations 281 | except for the axis along which CRPS is calculated (which should be the 282 | axis corresponding to the ensemble). If forecasts has the same shape as 283 | observations, the forecasts are treated as deterministic. Missing 284 | values (NaN) are ignored. 285 | weights : array_like, optional 286 | If provided, the CRPS is calculated exactly with the assigned 287 | probability weights to each forecast. Weights should be positive, but 288 | do not need to be normalized. By default, each forecast is weighted 289 | equally. 290 | issorted : bool, optional 291 | Optimization flag to indicate that the elements of `ensemble` are 292 | already sorted along `axis`. 293 | axis : int, optional 294 | Axis in forecasts and weights which corresponds to different ensemble 295 | members, along which to calculate CRPS. 296 | 297 | Returns 298 | ------- 299 | out : np.ndarray 300 | CRPS for each ensemble forecast against the observations. 301 | 302 | References 303 | ---------- 304 | Jochen Broecker. Chapter 7 in Forecast Verification: A Practitioner's Guide 305 | in Atmospheric Science. John Wiley & Sons, Ltd, Chichester, UK, 2nd 306 | edition, 2012. 307 | Tilmann Gneiting and Adrian E. Raftery. Strictly proper scoring rules, 308 | prediction, and estimation, 2005. University of Washington Department of 309 | Statistics Technical Report no. 463R. 310 | https://www.stat.washington.edu/research/reports/2004/tr463R.pdf 311 | Wilks D.S. (1995) Chapter 8 in _Statistical Methods in the 312 | Atmospheric Sciences_. Academic Press. 313 | """ 314 | observations = np.asarray(observations) 315 | forecasts = np.asarray(forecasts) 316 | if axis != -1: 317 | forecasts = move_axis_to_end(forecasts, axis) 318 | 319 | if weights is not None: 320 | weights = move_axis_to_end(weights, axis) 321 | if weights.shape != forecasts.shape: 322 | raise ValueError('forecasts and weights must have the same shape') 323 | 324 | if observations.shape not in [forecasts.shape, forecasts.shape[:-1]]: 325 | raise ValueError('observations and forecasts must have matching ' 326 | 'shapes or matching shapes except along `axis=%s`' 327 | % axis) 328 | 329 | if observations.shape == forecasts.shape: 330 | if weights is not None: 331 | raise ValueError('cannot supply weights unless you also supply ' 332 | 'an ensemble forecast') 333 | return abs(observations - forecasts) 334 | 335 | if not issorted: 336 | if weights is None: 337 | forecasts = np.sort(forecasts, axis=-1) 338 | else: 339 | idx = argsort_indices(forecasts, axis=-1) 340 | forecasts = forecasts[idx] 341 | weights = weights[idx] 342 | 343 | if weights is None: 344 | weights = np.ones_like(forecasts) 345 | 346 | return _crps_ensemble_core(observations, forecasts, weights) 347 | 348 | 349 | def move_axis_to_end(array, axis): 350 | array = np.asarray(array) 351 | return np.rollaxis(array, axis, start=array.ndim) 352 | 353 | 354 | def argsort_indices(a, axis=-1): 355 | """Like argsort, but returns an index suitable for sorting the 356 | the original array even if that array is multidimensional 357 | """ 358 | a = np.asarray(a) 359 | ind = list(np.ix_(*[np.arange(d) for d in a.shape])) 360 | ind[axis] = a.argsort(axis) 361 | return tuple(ind) 362 | 363 | 364 | @contextlib.contextmanager 365 | def suppress_warnings(msg=None): 366 | with warnings.catch_warnings(): 367 | warnings.filterwarnings('ignore', msg) 368 | yield -------------------------------------------------------------------------------- /model/prnn_supervisor.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import numpy as np 6 | import os 7 | from os.path import join, exists 8 | import shutil 9 | import sys 10 | import tensorflow as tf 11 | import time 12 | import yaml 13 | import random 14 | 15 | from lib import utils, metrics 16 | from lib.AMSGrad import AMSGrad 17 | from lib.metrics import neg_log_gauss, masked_mae_loss 18 | 19 | from model.prnn_model import PRNNModel 20 | 21 | 22 | class PRNNSupervisor(object): 23 | """ 24 | Do experiments using Graph Random Walk RNN model. 25 | """ 26 | 27 | def __init__(self, adj_mx, args, inference=False, pretrained_dir=None, **kwargs): 28 | 29 | SEED = args.trial_id 30 | np.random.seed(SEED) 31 | tf.set_random_seed(SEED) 32 | random.seed(SEED) 33 | 34 | self._dataset_name = args.dataset 35 | self.pretrained_dir = pretrained_dir 36 | 37 | kwargs["data"]["dataset_dir"] = join(args.data_dir, args.dataset) 38 | kwargs["data"]["dataset_name"] = args.dataset 39 | self._kwargs = kwargs 40 | self._data_kwargs = kwargs.get('data') 41 | self._model_kwargs = kwargs.get('model') 42 | self._train_kwargs = kwargs.get('train') 43 | 44 | self._rnn_type = kwargs.get("model")["rnn_type"] 45 | self._cost = kwargs.get("train")["cost"] 46 | 47 | # original log_dir setting 48 | # self._log_dir = self._get_log_dir(kwargs) 49 | # new log_dir setting: 50 | self._log_dir = self._get_log_dir_by_trial(args, inference=inference, kwargs=kwargs) 51 | 52 | log_level = self._kwargs.get('log_level', 'INFO') 53 | self._logger = utils.get_logger(self._log_dir, __name__, 'info.log', level=log_level) 54 | self._writer = tf.summary.FileWriter(self._log_dir) 55 | self._logger.info(kwargs) 56 | 57 | # Data preparation 58 | self._data = utils.load_dataset(**self._data_kwargs) 59 | for k, v in self._data.items(): 60 | if hasattr(v, 'shape'): 61 | self._logger.info((k, v.shape)) 62 | 63 | # Build models. 64 | scaler = self._data['scaler'] 65 | with tf.name_scope('Train'): 66 | with tf.variable_scope('DCRNN', reuse=False): 67 | self._train_model = PRNNModel(is_training=True, scaler=scaler, 68 | batch_size=self._data_kwargs['batch_size'], 69 | adj_mx=adj_mx, **self._model_kwargs) 70 | 71 | 72 | with tf.name_scope('Test'): 73 | with tf.variable_scope('DCRNN', reuse=True): 74 | self._test_model = PRNNModel(is_training=False, scaler=scaler, 75 | batch_size=self._data_kwargs['test_batch_size'], 76 | adj_mx=adj_mx, **self._model_kwargs) 77 | 78 | # learning 79 | self._lr = tf.get_variable('learning_rate', shape=(), initializer=tf.constant_initializer(0.01), 80 | trainable=False) 81 | self._new_lr = tf.placeholder(tf.float32, shape=(), name='new_learning_rate') 82 | self._lr_update = tf.assign(self._lr, self._new_lr, name='lr_update') 83 | 84 | 85 | # Configure optimizer 86 | optimizer_name = self._train_kwargs.get('optimizer', 'adam').lower() 87 | cost = self._train_kwargs.get('cost', 'mae').lower() 88 | epsilon = float(self._train_kwargs.get('epsilon', 1e-3)) 89 | optimizer = tf.train.AdamOptimizer(self._lr, epsilon=epsilon) 90 | if optimizer_name == 'sgd': 91 | optimizer = tf.train.GradientDescentOptimizer(self._lr, ) 92 | elif optimizer_name == 'amsgrad': 93 | optimizer = AMSGrad(self._lr, epsilon=epsilon) 94 | 95 | # Calculate loss 96 | output_dim = self._model_kwargs.get('output_dim') 97 | preds = self._train_model.outputs 98 | preds_samples = self._train_model.outputs_samples 99 | preds_mu = self._train_model.outputs_mu 100 | preds_sigma = self._train_model.outputs_sigma 101 | labels = self._train_model.labels[..., :output_dim] 102 | 103 | null_val = 0. 104 | if cost == 'mae': 105 | self._loss_fn = masked_mae_loss(scaler, null_val) 106 | self._train_loss = self._loss_fn(preds=preds, labels=labels) 107 | else: 108 | self._loss_fn = neg_log_gauss() 109 | self._train_loss = self._loss_fn(preds_mu=preds_mu, preds_sigma=preds_sigma, labels=labels) 110 | 111 | tvars = tf.trainable_variables() 112 | grads = tf.gradients(self._train_loss, tvars) 113 | max_grad_norm = kwargs['train'].get('max_grad_norm', 1.) 114 | grads, _ = tf.clip_by_global_norm(grads, max_grad_norm) 115 | global_step = tf.train.get_or_create_global_step() 116 | self._train_op = optimizer.apply_gradients(zip(grads, tvars), global_step=global_step, name='train_op') 117 | 118 | max_to_keep = self._train_kwargs.get('max_to_keep', 100) 119 | self._epoch = 0 120 | self._saver = tf.train.Saver(tf.global_variables(), max_to_keep=max_to_keep) 121 | 122 | # Log model statistics. 123 | total_trainable_parameter = utils.get_total_trainable_parameter_size() 124 | self._logger.info('Total number of trainable parameters: {:d}'.format(total_trainable_parameter)) 125 | for var in tf.global_variables(): 126 | self._logger.debug('{}, {}'.format(var.name, var.get_shape())) 127 | 128 | # @staticmethod 129 | # def _get_log_dir(kwargs): 130 | # log_dir = kwargs['train'].get('log_dir') 131 | # if log_dir is None: 132 | # batch_size = kwargs['data'].get('batch_size') 133 | # learning_rate = kwargs['train'].get('base_lr') 134 | # max_diffusion_step = kwargs['model'].get('max_diffusion_step') 135 | # num_rnn_layers = kwargs['model'].get('num_rnn_layers') 136 | # rnn_units = kwargs['model'].get('rnn_units') 137 | # structure = '-'.join( 138 | # ['%d' % rnn_units for _ in range(num_rnn_layers)]) 139 | # horizon = kwargs['model'].get('horizon') 140 | # filter_type = kwargs['model'].get('filter_type') 141 | # filter_type_abbr = 'L' 142 | # if filter_type == 'random_walk': 143 | # filter_type_abbr = 'R' 144 | # elif filter_type == 'dual_random_walk': 145 | # filter_type_abbr = 'DR' 146 | # run_id = 'dcrnn_%s_%d_h_%d_%s_lr_%g_bs_%d_%s/' % ( 147 | # filter_type_abbr, max_diffusion_step, horizon, 148 | # structure, learning_rate, batch_size, 149 | # time.strftime('%m%d%H%M%S')) 150 | # base_dir = kwargs.get('base_dir') 151 | # log_dir = os.path.join(base_dir, run_id) 152 | # if not os.path.exists(log_dir): 153 | # os.makedirs(log_dir) 154 | # return log_dir 155 | 156 | # refined version function to get log dir 157 | 158 | def _get_log_dir_by_trial(self, args, inference, kwargs): 159 | # log_dir = kwargs['train'].get('log_dir') 160 | if not inference: 161 | self.pretrained_dir = None 162 | if self.pretrained_dir is None: 163 | if not exists(args.ckpt_dir): 164 | os.mkdir(args.ckpt_dir) 165 | log_dir = join(args.ckpt_dir, args.dataset) 166 | if not exists(log_dir): 167 | os.mkdir(log_dir) 168 | 169 | log_dir = join(log_dir, self._rnn_type + "_" + self._cost) 170 | if not exists(log_dir): 171 | os.mkdir(log_dir) 172 | 173 | log_dir = join(log_dir, "trial_{}".format(args.trial_id)) 174 | 175 | if inference: 176 | log_dir = log_dir + "_infer" 177 | if not exists(log_dir): 178 | os.mkdir(log_dir) 179 | else: 180 | if exists(log_dir): 181 | shutil.rmtree(log_dir) 182 | os.mkdir(log_dir) 183 | else: 184 | log_dir = self.pretrained_dir 185 | log_dir = log_dir + "_infer" 186 | if not exists(log_dir): 187 | os.mkdir(log_dir) 188 | 189 | kwargs['train']['log_dir'] = log_dir 190 | 191 | return log_dir 192 | 193 | 194 | def run_epoch_generator(self, sess, model, data_generator, return_output=False, training=False, writer=None): 195 | losses = [] 196 | maes = [] 197 | outputs = [] 198 | outputs_samples = [] 199 | output_dim = self._model_kwargs.get('output_dim') 200 | preds = model.outputs 201 | preds_samples = model.outputs_samples 202 | preds_mu = model.outputs_mu 203 | preds_sigma = model.outputs_sigma 204 | labels = model.labels[..., :output_dim] 205 | cost = self._train_kwargs.get('cost', 'mae').lower() 206 | scaler = self._data['scaler'] 207 | null_val = 0. 208 | if cost == 'mae': 209 | loss = self._loss_fn(preds=preds, labels=labels) 210 | mae = loss 211 | else: 212 | loss = self._loss_fn(preds_mu=preds_mu, preds_sigma=preds_sigma, labels=labels) 213 | mae_fun = masked_mae_loss(scaler, null_val) 214 | mae = mae_fun(preds=preds, labels=labels) 215 | 216 | fetches = { 217 | 'loss': loss, 218 | 'mae': mae, 219 | 'global_step': tf.train.get_or_create_global_step() 220 | } 221 | if training: 222 | fetches.update({ 223 | 'train_op': self._train_op 224 | }) 225 | merged = model.merged 226 | if merged is not None: 227 | fetches.update({'merged': merged}) 228 | 229 | if return_output: 230 | fetches.update({ 231 | 'outputs': model.outputs, 232 | 'outputs_samples': model.outputs_samples 233 | }) 234 | 235 | for _, (x, y) in enumerate(data_generator): 236 | feed_dict = { 237 | model.inputs: x, 238 | model.labels: y, 239 | } 240 | 241 | vals = sess.run(fetches, feed_dict=feed_dict) 242 | 243 | losses.append(vals['loss']) 244 | maes.append(vals['mae']) 245 | if writer is not None and 'merged' in vals: 246 | writer.add_summary(vals['merged'], global_step=vals['global_step']) 247 | if return_output: 248 | outputs.append(vals['outputs']) 249 | outputs_samples.append(vals['outputs_samples']) 250 | 251 | results = { 252 | 'loss': np.mean(losses), 253 | 'mae': np.mean(maes) 254 | } 255 | if return_output: 256 | results['outputs'] = outputs 257 | results['outputs_samples'] = outputs_samples 258 | return results 259 | 260 | def get_lr(self, sess): 261 | return np.asscalar(sess.run(self._lr)) 262 | 263 | def set_lr(self, sess, lr): 264 | sess.run(self._lr_update, feed_dict={ 265 | self._new_lr: lr 266 | }) 267 | 268 | def train(self, sess, **kwargs): 269 | kwargs.update(self._train_kwargs) 270 | return self._train(sess, **kwargs) 271 | 272 | def _train(self, sess, base_lr, epoch, steps, patience=50, epochs=100, 273 | min_learning_rate=2e-6, lr_decay_ratio=0.1, save_model=1, 274 | test_every_n_epochs=10, **train_kwargs): 275 | history = [] 276 | min_val_loss = float('inf') 277 | wait = 0 278 | 279 | max_to_keep = train_kwargs.get('max_to_keep', 100) 280 | saver = tf.train.Saver(tf.global_variables(), max_to_keep=max_to_keep) 281 | model_filename = train_kwargs.get('model_filename') 282 | if model_filename is not None: 283 | saver.restore(sess, model_filename) 284 | self._epoch = epoch + 1 285 | else: 286 | sess.run(tf.global_variables_initializer()) 287 | self._logger.info('Start training {} on {}...'.format(self._rnn_type, self._dataset_name)) 288 | 289 | while self._epoch <= epochs: 290 | # Learning rate schedule. 291 | new_lr = max(min_learning_rate, base_lr * (lr_decay_ratio ** np.sum(self._epoch >= np.array(steps)))) 292 | self.set_lr(sess=sess, lr=new_lr) 293 | 294 | start_time = time.time() 295 | train_results = self.run_epoch_generator(sess, self._train_model, 296 | self._data['train_loader'].get_iterator(), 297 | training=True, 298 | writer=self._writer) 299 | train_loss, train_mae = train_results['loss'], train_results['mae'] 300 | if train_loss > 1e5: 301 | self._logger.warning('Gradient explosion detected. Ending...') 302 | break 303 | 304 | global_step = sess.run(tf.train.get_or_create_global_step()) 305 | # Compute validation error. 306 | val_results = self.run_epoch_generator(sess, self._test_model, 307 | self._data['val_loader'].get_iterator(), 308 | training=False) 309 | val_loss, val_mae = np.asscalar(val_results['loss']), np.asscalar(val_results['mae']) 310 | 311 | utils.add_simple_summary(self._writer, 312 | ['loss/train_loss', 'metric/train_mae', 'loss/val_loss', 'metric/val_mae'], 313 | [train_loss, train_mae, val_loss, val_mae], global_step=global_step) 314 | end_time = time.time() 315 | message = 'Epoch [{}/{}] ({}) train_loss: {:.4f}, val_mae: {:.4f} lr:{:.6f} {:.1f}s'.format( 316 | self._epoch, epochs, global_step, train_loss, val_mae, new_lr, (end_time - start_time)) 317 | self._logger.info(message) 318 | if self._epoch % test_every_n_epochs == test_every_n_epochs - 1: 319 | self.evaluate(sess) 320 | if val_mae <= min_val_loss: 321 | wait = 0 322 | if save_model > 0: 323 | model_filename = self.save(sess, val_loss) 324 | self._logger.info( 325 | 'Val loss decrease from %.4f to %.4f, saving to %s' % (min_val_loss, val_mae, model_filename)) 326 | min_val_loss = val_mae 327 | else: 328 | wait += 1 329 | if wait > patience: 330 | self._logger.warning('Early stopping at epoch: %d' % self._epoch) 331 | break 332 | 333 | history.append(val_mae) 334 | # Increases epoch. 335 | self._epoch += 1 336 | 337 | sys.stdout.flush() 338 | return np.min(history) 339 | 340 | def evaluate(self, sess, **kwargs): 341 | global_step = sess.run(tf.train.get_or_create_global_step()) 342 | test_results = self.run_epoch_generator(sess, self._test_model, 343 | self._data['test_loader'].get_iterator(), 344 | return_output=True, 345 | training=False) 346 | 347 | # y_preds: a list of (batch_size, horizon, num_nodes, output_dim) 348 | test_loss, y_preds, y_preds_samples = test_results['loss'], test_results['outputs'], test_results['outputs_samples'] 349 | utils.add_simple_summary(self._writer, ['loss/test_loss'], [test_loss], global_step=global_step) 350 | 351 | y_preds = np.concatenate(y_preds, axis=0) 352 | y_preds_samples = np.concatenate(y_preds_samples, axis=0) 353 | scaler = self._data['scaler'] 354 | predictions = [] 355 | predictions_samples = [] 356 | y_truths = [] 357 | for horizon_i in range(self._data['y_test'].shape[1]): 358 | y_truth = scaler.inverse_transform(self._data['y_test'][:, horizon_i, :, 0]) 359 | y_truths.append(y_truth) 360 | 361 | y_pred = scaler.inverse_transform(y_preds[:y_truth.shape[0], horizon_i, :, 0]) 362 | predictions.append(y_pred) 363 | 364 | y_pred_samples = scaler.inverse_transform(y_preds_samples[:y_truth.shape[0], horizon_i, :, :]) 365 | predictions_samples.append(y_pred_samples) 366 | 367 | mae = metrics.masked_mae_np(y_pred, y_truth, null_val=0) 368 | mape = metrics.masked_mape_np(y_pred, y_truth, null_val=0) 369 | rmse = metrics.masked_rmse_np(y_pred, y_truth, null_val=0) 370 | # self._logger.info( 371 | # "Horizon {:02d}, MAE: {:.2f}, MAPE: {:.4f}, RMSE: {:.2f}".format( 372 | # horizon_i + 1, mae, mape, rmse 373 | # ) 374 | # ) 375 | utils.add_simple_summary(self._writer, 376 | ['%s_%d' % (item, horizon_i + 1) for item in 377 | ['metric/rmse', 'metric/mape', 'metric/mae']], 378 | [rmse, mape, mae], 379 | global_step=global_step) 380 | outputs = { 381 | 'predictions': predictions, 382 | 'groundtruth': y_truths, 383 | 'predictions_samples': predictions_samples, 384 | } 385 | return outputs 386 | 387 | def load(self, sess, model_filename): 388 | """ 389 | Restore from saved model. 390 | :param sess: 391 | :param model_filename: 392 | :return: 393 | """ 394 | self._saver.restore(sess, model_filename) 395 | 396 | def save(self, sess, val_loss): 397 | config = dict(self._kwargs) 398 | global_step = np.asscalar(sess.run(tf.train.get_or_create_global_step())) 399 | prefix = os.path.join(self._log_dir, 'models-{:.4f}'.format(val_loss)) 400 | config['train']['epoch'] = self._epoch 401 | config['train']['global_step'] = global_step 402 | config['train']['log_dir'] = self._log_dir 403 | config['train']['model_filename'] = self._saver.save(sess, prefix, global_step=global_step, 404 | write_meta_graph=False) 405 | config_filename = 'config_{}.yaml'.format(self._epoch) 406 | with open(os.path.join(self._log_dir, config_filename), 'w') as f: 407 | yaml.dump(config, f, default_flow_style=False) 408 | return config['train']['model_filename'] 409 | 410 | 411 | 412 | 413 | 414 | 415 | 416 | 417 | 418 | 419 | -------------------------------------------------------------------------------- /model/utils_tf.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import numpy as np 6 | import tensorflow as tf 7 | import scipy.sparse as sp 8 | 9 | 10 | class AGCGRU: 11 | 12 | def __init__(self, adj_mx, input_dim, embed_dim, num_units, max_diffusion_step, scope): 13 | 14 | self._num_nodes = adj_mx.shape[0] 15 | self._num_units = num_units 16 | self._embed_dim = embed_dim 17 | self._max_diffusion_step = max_diffusion_step 18 | 19 | self.num_matrices = self._max_diffusion_step + 1 20 | self.input_size = input_dim + num_units 21 | 22 | with tf.variable_scope(scope): 23 | 24 | self.Wr = tf.get_variable('Wr', [self._embed_dim, self.num_matrices, self.input_size, self._num_units], 25 | dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer()) 26 | 27 | self.Wu = tf.get_variable('Wu', [self._embed_dim, self.num_matrices, self.input_size, self._num_units], 28 | dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer()) 29 | 30 | self.Wc = tf.get_variable('Wc', [self._embed_dim, self.num_matrices, self.input_size, self._num_units], 31 | dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer()) 32 | 33 | self.br = tf.get_variable("br", [self._embed_dim, self._num_units], dtype=tf.float32, 34 | initializer=tf.constant_initializer(1.0, dtype=tf.float32)) 35 | 36 | self.bu = tf.get_variable("bu", [self._embed_dim, self._num_units], dtype=tf.float32, 37 | initializer=tf.constant_initializer(1.0, dtype=tf.float32)) 38 | 39 | self.bc = tf.get_variable("bc", [self._embed_dim, self._num_units], dtype=tf.float32, 40 | initializer=tf.constant_initializer(0.0, dtype=tf.float32)) 41 | 42 | @staticmethod 43 | def _concat(x, x_): 44 | x_ = tf.expand_dims(x_, 0) 45 | return tf.concat([x, x_], axis=0) 46 | 47 | def _replicate_input(self, x, nParticle): 48 | if len(x.get_shape()) < 4: 49 | return tf.tile(x[:, :, :, tf.newaxis], [1, 1, 1, nParticle]) 50 | else: 51 | return x 52 | 53 | def forward(self, inputs, state, node_embedding): 54 | graph_processed_input = self._graph_filtering(inputs, state, node_embedding) 55 | 56 | r = tf.nn.sigmoid(self._gconv(graph_processed_input, node_embedding, self.Wr, self.br)) 57 | u = tf.nn.sigmoid(self._gconv(graph_processed_input, node_embedding, self.Wu, self.bu)) 58 | 59 | graph_processed_input_times_r = self._graph_filtering(inputs, r * state, node_embedding) 60 | 61 | c = tf.nn.tanh(self._gconv(graph_processed_input_times_r, node_embedding, self.Wc, self.bc)) 62 | output = u * state + (1 - u) * c 63 | return output 64 | 65 | def _gconv(self, graph_processed_input, node_embedding, weights, biases): 66 | 67 | batch_size = graph_processed_input.get_shape()[0].value 68 | nParticle = graph_processed_input.get_shape()[1].value 69 | 70 | weights_ = tf.einsum('nd, dkio -> nkio', node_embedding, weights) 71 | biases_ = tf.matmul(node_embedding, biases) 72 | 73 | x = tf.einsum('bpnik, nkio -> bpno', graph_processed_input, weights_) # (batch_size, nParticle, self._num_nodes, output_size) 74 | biases_ = tf.tile(biases_[tf.newaxis, tf.newaxis, :, :], [batch_size, nParticle, 1, 1]) 75 | x += biases_ 76 | # Reshape res back to 2D: (batch_size, num_node, state_dim) -> (batch_size, num_node, output_size, nParticle) 77 | return tf.transpose(x, perm=[0, 2, 3, 1]) 78 | 79 | def _graph_filtering(self, inputs, state, node_embedding): 80 | 81 | adj_learned = tf.nn.softmax(tf.nn.relu(tf.matmul(node_embedding, tf.transpose(node_embedding, perm=[1, 0])))) 82 | supports = [adj_learned] 83 | 84 | # Reshape input and state to (batch_size, num_nodes, input_dim/state_dim, nParticle) 85 | batch_size = inputs.get_shape()[0].value 86 | nParticle = state.get_shape()[3].value 87 | inputs = self._replicate_input(inputs, nParticle) 88 | 89 | inputs = tf.reshape(inputs, shape=[batch_size, self._num_nodes, -1, nParticle]) 90 | state = tf.reshape(state, shape=[batch_size, self._num_nodes, -1, nParticle]) 91 | inputs_and_state = tf.concat([inputs, state], axis=2) 92 | input_size = inputs_and_state.get_shape()[2].value 93 | 94 | x = inputs_and_state 95 | x0 = tf.transpose(x, perm=[1, 2, 0, 3]) # (num_nodes, total_arg_size, batch_size, nParticle) 96 | x0 = tf.reshape(x0, shape=[self._num_nodes, input_size * batch_size * nParticle]) 97 | x = tf.expand_dims(x0, axis=0) 98 | 99 | for support in supports: 100 | x1 = tf.matmul(support, x0) 101 | x = self._concat(x, x1) 102 | 103 | for k in range(2, self._max_diffusion_step + 1): 104 | x2 = 2 * tf.matmul(support, x1) - x0 105 | x = self._concat(x, x2) 106 | x1, x0 = x2, x1 107 | 108 | x = tf.reshape(x, shape=[self.num_matrices, self._num_nodes, input_size, batch_size, nParticle]) 109 | x = tf.transpose(x, perm=[3, 4, 1, 2, 0]) # (batch_size, nParticle, num_nodes, input_size, order) 110 | return x 111 | 112 | 113 | class DCGRU: 114 | 115 | def __init__(self, adj_mx, input_dim, num_units, max_diffusion_step, scope): 116 | 117 | self._num_nodes = adj_mx.shape[0] 118 | self._num_units = num_units 119 | self._max_diffusion_step = max_diffusion_step 120 | self._supports = [] 121 | 122 | supports = [] 123 | 124 | supports.append(calculate_random_walk_matrix(adj_mx).T) 125 | supports.append(calculate_random_walk_matrix(adj_mx.T).T) 126 | 127 | for support in supports: 128 | self._supports.append(self._build_sparse_matrix(support)) 129 | 130 | self.num_matrices = len(self._supports) * self._max_diffusion_step + 1 131 | self.input_size = input_dim + num_units 132 | 133 | with tf.variable_scope(scope): 134 | 135 | self.Wr = tf.get_variable('Wr', [self.input_size * self.num_matrices, self._num_units], dtype=tf.float32, 136 | initializer=tf.contrib.layers.xavier_initializer()) 137 | 138 | self.Wu = tf.get_variable('Wu', [self.input_size * self.num_matrices, self._num_units], dtype=tf.float32, 139 | initializer=tf.contrib.layers.xavier_initializer()) 140 | 141 | self.Wc = tf.get_variable('Wc', [self.input_size * self.num_matrices, self._num_units], dtype=tf.float32, 142 | initializer=tf.contrib.layers.xavier_initializer()) 143 | 144 | self.br = tf.get_variable("br", [self._num_units], dtype=tf.float32, 145 | initializer=tf.constant_initializer(1.0, dtype=tf.float32)) 146 | 147 | self.bu = tf.get_variable("bu", [self._num_units], dtype=tf.float32, 148 | initializer=tf.constant_initializer(1.0, dtype=tf.float32)) 149 | self.bc = tf.get_variable("bc", [self._num_units], dtype=tf.float32, 150 | initializer=tf.constant_initializer(0.0, dtype=tf.float32)) 151 | 152 | @staticmethod 153 | def _build_sparse_matrix(L): 154 | L = L.tocoo() 155 | indices = np.column_stack((L.row, L.col)) 156 | L = tf.SparseTensor(indices, L.data, L.shape) 157 | return tf.sparse_reorder(L) 158 | 159 | @staticmethod 160 | def _concat(x, x_): 161 | x_ = tf.expand_dims(x_, 0) 162 | return tf.concat([x, x_], axis=0) 163 | 164 | def _replicate_input(self, x, nParticle): 165 | if len(x.get_shape()) < 4: 166 | return tf.tile(x[:, :, :, tf.newaxis], [1, 1, 1, nParticle]) 167 | else: 168 | return x 169 | 170 | def forward(self, inputs, state): 171 | r = tf.nn.sigmoid(self._gconv(inputs, state, self._num_units, self.Wr, self.br)) 172 | u = tf.nn.sigmoid(self._gconv(inputs, state, self._num_units, self.Wu, self.bu)) 173 | c = tf.nn.tanh(self._gconv(inputs, r * state, self._num_units, self.Wc, self.bc)) 174 | output = u * state + (1 - u) * c 175 | return output 176 | 177 | def _gconv(self, inputs, state, output_size, weights, biases): 178 | # Reshape input and state to (batch_size, num_nodes, input_dim/state_dim, nParticle) 179 | batch_size = inputs.get_shape()[0].value 180 | nParticle = state.get_shape()[3].value 181 | inputs = self._replicate_input(inputs, nParticle) 182 | 183 | inputs = tf.reshape(inputs, shape=[batch_size, self._num_nodes, -1, nParticle]) 184 | state = tf.reshape(state, shape=[batch_size, self._num_nodes, -1, nParticle]) 185 | inputs_and_state = tf.concat([inputs, state], axis=2) 186 | input_size = inputs_and_state.get_shape()[2].value 187 | 188 | x = inputs_and_state 189 | x0 = tf.transpose(x, perm=[1, 2, 0, 3]) # (num_nodes, total_arg_size, batch_size, nParticle) 190 | x0 = tf.reshape(x0, shape=[self._num_nodes, input_size * batch_size * nParticle]) 191 | x = tf.expand_dims(x0, axis=0) 192 | 193 | for support in self._supports: 194 | x1 = tf.sparse_tensor_dense_matmul(support, x0) 195 | x = self._concat(x, x1) 196 | 197 | for k in range(2, self._max_diffusion_step + 1): 198 | x2 = 2 * tf.sparse_tensor_dense_matmul(support, x1) - x0 199 | x = self._concat(x, x2) 200 | x1, x0 = x2, x1 201 | 202 | x = tf.reshape(x, shape=[self.num_matrices, self._num_nodes, input_size, batch_size, nParticle]) 203 | x = tf.transpose(x, perm=[3, 4, 1, 2, 0]) # (batch_size, nParticle, num_nodes, input_size, order) 204 | x = tf.reshape(x, shape=[batch_size * nParticle * self._num_nodes, input_size * self.num_matrices]) 205 | 206 | x = tf.matmul(x, weights) # (batch_size * nParticle * self._num_nodes, output_size) 207 | 208 | x = tf.nn.bias_add(x, biases) 209 | # Reshape res back to 2D: (batch_size, num_node, state_dim) -> (batch_size, num_node, output_size, nParticle) 210 | return tf.transpose(tf.reshape(x, [batch_size, nParticle, self._num_nodes, output_size]), perm=[0, 2, 3, 1]) 211 | 212 | 213 | class GRU: 214 | 215 | def __init__(self, adj_mx, input_dim, num_units, scope): 216 | 217 | self._num_nodes = adj_mx.shape[0] 218 | self._num_units = num_units 219 | self.input_size = input_dim + num_units 220 | 221 | with tf.variable_scope(scope): 222 | 223 | self.Wr = tf.get_variable('Wr', [self.input_size, self._num_units], dtype=tf.float32, 224 | initializer=tf.contrib.layers.xavier_initializer()) 225 | 226 | self.Wu = tf.get_variable('Wu', [self.input_size, self._num_units], dtype=tf.float32, 227 | initializer=tf.contrib.layers.xavier_initializer()) 228 | 229 | self.Wc = tf.get_variable('Wc', [self.input_size, self._num_units], dtype=tf.float32, 230 | initializer=tf.contrib.layers.xavier_initializer()) 231 | 232 | self.br = tf.get_variable("br", [self._num_units], dtype=tf.float32, 233 | initializer=tf.constant_initializer(1.0, dtype=tf.float32)) 234 | 235 | self.bu = tf.get_variable("bu", [self._num_units], dtype=tf.float32, 236 | initializer=tf.constant_initializer(1.0, dtype=tf.float32)) 237 | self.bc = tf.get_variable("bc", [self._num_units], dtype=tf.float32, 238 | initializer=tf.constant_initializer(0.0, dtype=tf.float32)) 239 | 240 | @staticmethod 241 | def _concat(x, x_): 242 | x_ = tf.expand_dims(x_, 0) 243 | return tf.concat([x, x_], axis=0) 244 | 245 | def _replicate_input(self, x, nParticle): 246 | if len(x.get_shape()) < 4: 247 | return tf.tile(x[:, :, :, tf.newaxis], [1, 1, 1, nParticle]) 248 | else: 249 | return x 250 | 251 | def forward(self, inputs, state): 252 | r = tf.nn.sigmoid(self._conv(inputs, state, self._num_units, self.Wr, self.br)) 253 | u = tf.nn.sigmoid(self._conv(inputs, state, self._num_units, self.Wu, self.bu)) 254 | c = tf.nn.tanh(self._conv(inputs, r * state, self._num_units, self.Wc, self.bc)) 255 | output = u * state + (1 - u) * c 256 | return output 257 | 258 | def _conv(self, inputs, state, output_size, weights, biases): 259 | # Reshape input and state to (batch_size, num_nodes, input_dim/state_dim, nParticle) 260 | batch_size = inputs.get_shape()[0].value 261 | nParticle = state.get_shape()[3].value 262 | inputs = self._replicate_input(inputs, nParticle) 263 | 264 | inputs = tf.reshape(inputs, shape=[batch_size, self._num_nodes, -1, nParticle]) 265 | state = tf.reshape(state, shape=[batch_size, self._num_nodes, -1, nParticle]) 266 | inputs_and_state = tf.concat([inputs, state], axis=2) 267 | input_size = inputs_and_state.get_shape()[2].value 268 | 269 | x = inputs_and_state 270 | x = tf.transpose(x, perm=[0, 3, 1, 2]) # (batch_size, nParticle, num_nodes, input_size) 271 | x = tf.reshape(x, shape=[batch_size * nParticle * self._num_nodes, input_size]) 272 | 273 | x = tf.matmul(x, weights) # (batch_size * nParticle * self._num_nodes, output_size) 274 | 275 | x = tf.nn.bias_add(x, biases) 276 | # Reshape res back to 2D: (batch_size, num_node, state_dim) -> (batch_size, num_node, output_size, nParticle) 277 | return tf.transpose(tf.reshape(x, [batch_size, nParticle, self._num_nodes, output_size]), perm=[0, 2, 3, 1]) 278 | 279 | 280 | def calculate_random_walk_matrix(adj_mx): 281 | adj_mx = sp.coo_matrix(adj_mx) 282 | d = np.array(adj_mx.sum(1)) 283 | d_inv = np.power(d, -1).flatten() 284 | d_inv[np.isinf(d_inv)] = 0. 285 | d_mat_inv = sp.diags(d_inv) 286 | random_walk_mx = d_mat_inv.dot(adj_mx).tocoo() 287 | return random_walk_mx 288 | 289 | 290 | def linear_projection(state, weight, weight_delta, noise=True): 291 | 292 | num_nodes = state.get_shape()[1].value 293 | num_units = state.get_shape()[2].value 294 | nParticle = state.get_shape()[3].value 295 | nBatch = state.get_shape()[0].value 296 | 297 | state = tf.reshape(tf.transpose(state, perm=(0, 1, 3, 2)), shape=(-1, num_units)) 298 | measurement_mean = tf.reshape(tf.matmul(state, weight), shape=(nBatch, num_nodes, nParticle)) 299 | noise_std_dev = tf.reshape(tf.math.softplus(tf.matmul(state, weight_delta)), shape=(nBatch, num_nodes, nParticle)) 300 | 301 | measurement_mean = tf.where(tf.is_nan(measurement_mean), tf.zeros_like(measurement_mean), measurement_mean) 302 | measurement_mean = tf.where(tf.is_inf(measurement_mean), tf.zeros_like(measurement_mean), measurement_mean) 303 | 304 | noise_std_dev = tf.where(tf.is_nan(noise_std_dev), tf.zeros_like(noise_std_dev), noise_std_dev) 305 | noise_std_dev = tf.where(tf.is_inf(noise_std_dev), tf.zeros_like(noise_std_dev), noise_std_dev) 306 | 307 | if noise: 308 | measurement = measurement_mean + noise_std_dev * tf.random.normal(shape=(tf.shape(measurement_mean)), mean=0, stddev=1) 309 | 310 | return measurement, measurement_mean, noise_std_dev 311 | 312 | 313 | def particle_flow(particles, measurement, weight, weight_delta): 314 | num_nodes = tf.shape(particles)[1] 315 | num_units = tf.shape(particles)[2] 316 | nParticle = tf.shape(particles)[3] 317 | nBatch = tf.shape(particles)[0] 318 | 319 | particles_temp = particles 320 | 321 | lambda_seq = generate_exponential_lambda(29, 1.2) 322 | 323 | mu_0_all = tf.reduce_mean(particles, axis=3) 324 | P_all = tf_cov(tf.reshape(tf.transpose(particles, perm=(0, 1, 3, 2)), shape=[nBatch, -1, num_units])) 325 | 326 | delta = tf.squeeze(tf.log(1 + tf.exp(tf.matmul(tf.reduce_mean(mu_0_all, axis=1), weight_delta)))) 327 | R = delta * delta 328 | R_inv = 1.0 / R 329 | 330 | PH_t = tf.einsum('ijk,kl->ijl', P_all, weight) 331 | PH_tH = tf.einsum('ijk,kl->ijl', PH_t, tf.transpose(weight)) 332 | HPH_t = tf.squeeze(tf.einsum('ij,kjl->ikl', tf.transpose(weight), PH_t)) 333 | 334 | PH_t_R_inv_z = tf.tile(R_inv[:, tf.newaxis, tf.newaxis], [1, num_units, num_nodes]) * tf.einsum('ijk,ikl->ijl', PH_t, measurement) 335 | 336 | C = tf.tile(tf.eye(num_units)[tf.newaxis, :, :], [nBatch, 1, 1]) 337 | D = tf.zeros([nBatch, num_units, num_nodes]) 338 | 339 | # particle flow 340 | lmbd_prev = 0 341 | 342 | for lmbd in lambda_seq: 343 | step_size = lmbd - lmbd_prev 344 | 345 | A_denom = lmbd*HPH_t + R 346 | A = -0.5 * PH_tH / tf.tile(A_denom[:, tf.newaxis, tf.newaxis], [1, num_units, num_units]) 347 | 348 | b_term1 = tf.tile(tf.eye(num_units)[tf.newaxis, :, :], [nBatch, 1, 1]) + lmbd * A 349 | b_term2 = tf.tile(tf.eye(num_units)[tf.newaxis, :, :], [nBatch, 1, 1]) + 2.0 * lmbd * A 350 | b_term3 = tf.einsum('ijk,ikl->ijl', b_term1, PH_t_R_inv_z) + tf.einsum('ijk,ilk->ijl', A, mu_0_all) 351 | b = tf.einsum('ijk,ikl->ijl', b_term2, b_term3) 352 | 353 | C_term1 = tf.tile(tf.eye(num_units)[tf.newaxis, :, :], [nBatch, 1, 1]) + step_size * A 354 | C = tf.einsum('ijk,ikl->ijl', C_term1, C) 355 | D = tf.einsum('ijk,ikl->ijl', C_term1, D) + step_size * b 356 | 357 | lmbd_prev = lmbd 358 | 359 | particles = tf.einsum('ijk,ilkp->iljp', C, particles) + \ 360 | tf.transpose(tf.tile(D[:, :, :, tf.newaxis], [1, 1, 1, nParticle]), perm=(0, 2, 1, 3)) 361 | 362 | particles = tf.where(tf.is_nan(particles), particles_temp, particles) 363 | particles = tf.where(tf.is_inf(particles), particles_temp, particles) 364 | return particles 365 | 366 | 367 | def tf_cov(x): 368 | # x: nBatch x nSamples x dim, cov_xx: n_batch x dim x dim 369 | mean_x = tf.reduce_mean(x, axis=1, keep_dims=True) 370 | mx = tf.einsum('ijk,ikl->ijl', tf.transpose(mean_x, perm=(0, 2, 1)), mean_x) 371 | vx = tf.einsum('ijk,ikl->ijl', tf.transpose(x, perm=(0, 2, 1)), x)/tf.cast(tf.shape(x)[1], tf.float32) 372 | cov_xx = vx - mx 373 | return cov_xx 374 | 375 | 376 | def generate_exponential_lambda(num_steps, delta_lambda_ratio): 377 | lambda_1 = (1-delta_lambda_ratio) / (1-delta_lambda_ratio**num_steps) 378 | lambda_seq = np.cumsum(lambda_1 * (delta_lambda_ratio**np.arange(num_steps))).astype('float32') 379 | return lambda_seq 380 | 381 | 382 | def prediction_batch(particles_0, particles, init_measurement, target, rnn_0, rnn_1, weight, weight_delta, node_embedding, 383 | is_training, curriculam_prob, rnn_type): 384 | nParticle = particles.get_shape()[3].value 385 | pred_len = target.get_shape()[1].value 386 | 387 | input_dim = target.get_shape()[3].value 388 | 389 | for t in range(pred_len): 390 | if t == 0: 391 | if rnn_type == 'agcgru': 392 | particles_0 = rnn_0.forward(init_measurement, particles_0, node_embedding) 393 | else: 394 | particles_0 = rnn_0.forward(init_measurement, particles_0) 395 | else: 396 | c = tf.random_uniform((), minval=0, maxval=1.) 397 | curriculam = tf.cond(tf.less(c, curriculam_prob), lambda: 'yes', lambda: 'no') 398 | 399 | if is_training and curriculam == 'yes': 400 | if rnn_type == 'agcgru': 401 | particles_0 = rnn_0.forward(target[:, t-1, :, :input_dim], particles_0, node_embedding) 402 | else: 403 | particles_0 = rnn_0.forward(target[:, t - 1, :, :input_dim], particles_0) 404 | else: 405 | if input_dim == 1: 406 | obs_samples_aug = obs_samples_temp[:, :, tf.newaxis, :] 407 | else: 408 | obs_samples_aug = tf.concat([obs_samples_temp[:, :, tf.newaxis, :], 409 | tf.tile(target[:, t-1, :, input_dim-1:][:, :, :, tf.newaxis], [1, 1, 1, nParticle])], axis=2) 410 | if rnn_type == 'agcgru': 411 | particles_0 = rnn_0.forward(obs_samples_aug, particles_0, node_embedding) 412 | else: 413 | particles_0 = rnn_0.forward(obs_samples_aug, particles_0) 414 | if rnn_type == 'agcgru': 415 | particles = rnn_1.forward(particles_0, particles, node_embedding) 416 | else: 417 | particles = rnn_1.forward(particles_0, particles) 418 | obs_samples_temp, obs_mu_temp, obs_sigma_temp = linear_projection(particles, weight, weight_delta, noise=True) 419 | 420 | if t == 0: 421 | prediction_samples = obs_samples_temp[:, tf.newaxis, :, :] 422 | prediction_mu = obs_mu_temp[:, tf.newaxis, :, :] 423 | prediction_sigma = obs_sigma_temp[:, tf.newaxis, :, :] 424 | else: 425 | prediction_samples = tf.concat([prediction_samples, obs_samples_temp[:, tf.newaxis, :, :]], axis=1) 426 | prediction_mu = tf.concat([prediction_mu, obs_mu_temp[:, tf.newaxis, :, :]], axis=1) 427 | prediction_sigma = tf.concat([prediction_sigma, obs_sigma_temp[:, tf.newaxis, :, :]], axis=1) 428 | 429 | return prediction_samples, prediction_mu, prediction_sigma 430 | 431 | 432 | def training(inputs, targets, rnn_0, rnn_1, weight, weight_delta, 433 | rho, node_embedding, num_units, nParticle, is_training, curriculam_prob, rnn_type): 434 | 435 | batch = inputs.get_shape()[0].value 436 | seq_len = inputs.get_shape()[1].value 437 | num_nodes = inputs.get_shape()[2].value 438 | input_dim = inputs.get_shape()[3].value 439 | 440 | particles_0 = rho * tf.random.normal(shape=[batch, num_nodes, num_units, nParticle]) 441 | particles = rho * tf.random.normal(shape=[batch, num_nodes, num_units, nParticle]) 442 | 443 | for t in range(seq_len): 444 | if t == 0: 445 | if rnn_type == 'agcgru': 446 | particles_0 = rnn_0.forward(tf.zeros(shape=[batch, num_nodes, input_dim]), particles_0, node_embedding) 447 | else: 448 | particles_0 = rnn_0.forward(tf.zeros(shape=[batch, num_nodes, input_dim]), particles_0) 449 | else: 450 | if rnn_type == 'agcgru': 451 | particles_0 = rnn_0.forward(inputs[:, t-1, :, :input_dim], particles_0, node_embedding) 452 | else: 453 | particles_0 = rnn_0.forward(inputs[:, t - 1, :, :input_dim], particles_0) 454 | 455 | if rnn_type == 'agcgru': 456 | particles = rnn_1.forward(particles_0, particles, node_embedding) 457 | else: 458 | particles = rnn_1.forward(particles_0, particles) 459 | particles = particle_flow(particles, inputs[:, t:t+1, :, 0], weight, weight_delta) 460 | 461 | prediction_samples, prediction_mu, prediction_sigma = prediction_batch(particles_0, particles, inputs[:, -1, :, :], targets, rnn_0, rnn_1, weight, 462 | weight_delta, node_embedding, is_training, curriculam_prob, rnn_type) 463 | return prediction_samples, prediction_mu, prediction_sigma --------------------------------------------------------------------------------