├── __init__.py ├── model ├── __init__.py ├── FC_GRU.py ├── FC_LSTM.py ├── ConvLSTM.py ├── ConvGRU.py ├── GCN.py ├── Coupled_GCN.py └── Coupled_ConvGRU.py ├── baseline ├── STDN │ ├── __init__.py │ ├── minMax.py │ ├── utils.py │ ├── main.py │ └── model.py ├── DMVSTNet │ ├── __init__.py │ ├── minMax.py │ ├── get_embeddings.py │ ├── utils.py │ ├── main_2.py │ ├── main.py │ ├── model_2.py │ └── model.py ├── STGCN │ ├── models │ │ ├── __init__.py │ │ ├── base_model.py │ │ ├── trainer.py │ │ ├── tester.py │ │ └── layers.py │ ├── utils │ │ ├── __init__.py │ │ ├── math_utils.py │ │ └── math_graph.py │ ├── data_loader │ │ ├── __init__.py │ │ └── data_utils.py │ └── main.py ├── STResNet │ ├── op_utils.py │ ├── preprocessing.py │ ├── ResNet.py │ ├── main.py │ ├── solver.py │ └── utils.py └── ARIMA.py ├── preprocessing.py ├── citibike.py └── taxi_graph.py /__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import -------------------------------------------------------------------------------- /model/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import -------------------------------------------------------------------------------- /baseline/STDN/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import -------------------------------------------------------------------------------- /baseline/DMVSTNet/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import -------------------------------------------------------------------------------- /baseline/STGCN/models/__init__.py: -------------------------------------------------------------------------------- 1 | # @Time : Jan. 10, 2019 17:49 2 | # @Author : Veritas YIN 3 | # @FileName : __init__.py 4 | # @Version : 1.0 5 | # @IDE : PyCharm 6 | # @Github : https://github.com/VeritasYin/Project_Orion 7 | -------------------------------------------------------------------------------- /baseline/STGCN/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # @Time : Jan. 10, 2019 15:24 2 | # @Author : Veritas YIN 3 | # @FileName : __init__.py 4 | # @Version : 1.0 5 | # @IDE : PyCharm 6 | # @Github : https://github.com/VeritasYin/Project_Orion 7 | -------------------------------------------------------------------------------- /baseline/STGCN/data_loader/__init__.py: -------------------------------------------------------------------------------- 1 | # @Time : Jan. 10, 2019 15:24 2 | # @Author : Veritas YIN 3 | # @FileName : __init__.py 4 | # @Version : 1.0 5 | # @IDE : PyCharm 6 | # @Github : https://github.com/VeritasYin/Project_Orion 7 | -------------------------------------------------------------------------------- /baseline/STResNet/op_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pickle 3 | import scipy.io as sio 4 | import h5py 5 | import time 6 | import os 7 | 8 | def RMSE(x_pre, x_true): 9 | x_pre = np.array(x_pre) 10 | x_true = np.array(x_true) 11 | return np.sqrt(np.mean(np.square(x_pre - x_true))) 12 | 13 | def MAE(x_pre, x_true): 14 | x_pre = np.array(x_pre) 15 | x_true = np.array(x_true) 16 | return np.mean(np.abs(x_pre - x_true)) 17 | 18 | def MAPE(x_pre, x_true): 19 | x_pre = np.array(x_pre) 20 | x_true = np.array(x_true) 21 | return np.mean(np.abs((x_pre - x_true)/(x_true + 1))) 22 | -------------------------------------------------------------------------------- /baseline/STDN/minMax.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | class MinMax: 4 | def __init__(self, data): 5 | self.max_value = np.max(data) 6 | self.min_value = np.min(data) 7 | self.data = data 8 | 9 | def transform(self): 10 | return (self.data-self.min_value)/(self.max_value-self.min_value) 11 | 12 | def inverse(self, rmse): 13 | return rmse * (self.max_value - self.min_value) 14 | 15 | class MinMaxNormalization01(object): 16 | def __init__(self, ): 17 | pass 18 | 19 | def fit(self, data): 20 | self._min = np.amin(data) 21 | self._max = np.amax(data) 22 | print("min: ", self._min, "max:", self._max) 23 | 24 | def transform(self, data): 25 | norm_data = 1. * (data - self._min) / (self._max - self._min) 26 | return norm_data 27 | 28 | def fit_transform(self, data): 29 | self.fit(data) 30 | return self.transform(data) 31 | 32 | def inverse(self, data): 33 | inverse_norm_data = 1. * data * (self._max - self._min) + self._min 34 | return inverse_norm_data 35 | 36 | def real_loss(self, loss): 37 | # loss is rmse 38 | return loss*(self._max - self._min) 39 | #return real_loss -------------------------------------------------------------------------------- /baseline/DMVSTNet/minMax.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | class MinMax: 4 | def __init__(self, data): 5 | self.max_value = np.max(data) 6 | self.min_value = np.min(data) 7 | self.data = data 8 | 9 | def transform(self): 10 | return (self.data-self.min_value)/(self.max_value-self.min_value) 11 | 12 | def inverse(self, rmse): 13 | return rmse * (self.max_value - self.min_value) 14 | 15 | class MinMaxNormalization01(object): 16 | def __init__(self, ): 17 | pass 18 | 19 | def fit(self, data): 20 | self._min = np.amin(data) 21 | self._max = np.amax(data) 22 | print("min: ", self._min, "max:", self._max) 23 | 24 | def transform(self, data): 25 | norm_data = 1. * (data - self._min) / (self._max - self._min) 26 | return norm_data 27 | 28 | def fit_transform(self, data): 29 | self.fit(data) 30 | return self.transform(data) 31 | 32 | def inverse(self, data): 33 | inverse_norm_data = 1. * data * (self._max - self._min) + self._min 34 | return inverse_norm_data 35 | 36 | def real_loss(self, loss): 37 | # loss is rmse 38 | return loss*(self._max - self._min) 39 | #return real_loss -------------------------------------------------------------------------------- /baseline/STResNet/preprocessing.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class MinMaxNormalization01(object): 5 | def __init__(self, ): 6 | pass 7 | 8 | def fit(self, data): 9 | self._min = np.amin(data) 10 | self._max = np.amax(data) 11 | print("min: ", self._min, "max:", self._max) 12 | 13 | def transform(self, data): 14 | norm_data = 1. * (data - self._min) / (self._max - self._min) 15 | return norm_data 16 | 17 | def fit_transform(self, data): 18 | self.fit(data) 19 | return self.transform(data) 20 | 21 | def inverse_transform(self, data): 22 | inverse_norm_data = 1. * data * (self._max - self._min) + self._min 23 | return inverse_norm_data 24 | 25 | def real_loss(self, loss): 26 | # loss is rmse 27 | return loss * (self._max - self._min) 28 | # return real_loss 29 | 30 | 31 | class MinMaxNormalization_neg_1_pos_1(object): 32 | def __init__(self): 33 | pass 34 | 35 | def fit(self, X): 36 | self._min = X.min() 37 | self._max = X.max() 38 | print("min:", self._min, "max:", self._max) 39 | 40 | def transform(self, X): 41 | X = 1. * (X - self._min) / (self._max - self._min) 42 | X = X * 2. - 1. 43 | return X 44 | 45 | def fit_transform(self, X): 46 | self.fit(X) 47 | return self.transform(X) 48 | 49 | def inverse_transform(self, X): 50 | X = (X + 1.) / 2. 51 | X = 1. * X * (self._max - self._min) + self._min 52 | return X 53 | 54 | def real_loss(self, loss): 55 | # loss is rmse 56 | return loss * (self._max - self._min) / 2. 57 | # return real_loss 58 | -------------------------------------------------------------------------------- /baseline/STGCN/models/base_model.py: -------------------------------------------------------------------------------- 1 | # @Time : Jan. 12, 2019 19:01 2 | # @Author : Veritas YIN 3 | # @FileName : base_model.py 4 | # @Version : 1.0 5 | # @IDE : PyCharm 6 | # @Github : https://github.com/VeritasYin/Project_Orion 7 | 8 | from models.layers import * 9 | from os.path import join as pjoin 10 | import tensorflow as tf 11 | 12 | 13 | def build_model(inputs, n_his, Ks, Kt, blocks, keep_prob): 14 | ''' 15 | Build the base model. 16 | :param inputs: placeholder. 17 | :param n_his: int, size of historical records for training. 18 | :param Ks: int, kernel size of spatial convolution. 19 | :param Kt: int, kernel size of temporal convolution. 20 | :param blocks: list, channel configs of st_conv blocks. 21 | :param keep_prob: placeholder. 22 | ''' 23 | x = inputs[:, 0:n_his, :, :] 24 | 25 | # Ko>0: kernel size of temporal convolution in the output layer. 26 | Ko = n_his 27 | # ST-Block 28 | for i, channels in enumerate(blocks): 29 | x = st_conv_block(x, Ks, Kt, channels, i, keep_prob, act_func='GLU') 30 | Ko -= 2 * (Ks - 1) 31 | 32 | # Output Layer 33 | if Ko > 1: 34 | y = output_layer(x, Ko, 'output_layer') 35 | else: 36 | raise ValueError(f'ERROR: kernel size Ko must be greater than 1, but received "{Ko}".') 37 | 38 | tf.add_to_collection(name='copy_loss', 39 | value=tf.nn.l2_loss(inputs[:, n_his - 1:n_his, :, :] - inputs[:, n_his:n_his + 1, :, :])) 40 | train_loss = tf.nn.l2_loss(y - inputs[:, n_his:n_his + 1, :, :]) 41 | single_pred = y[:, 0, :, :] 42 | tf.add_to_collection(name='y_pred', value=single_pred) 43 | return train_loss, single_pred 44 | 45 | 46 | def model_save(sess, global_steps, model_name, save_path='./output/models/'): 47 | ''' 48 | Save the checkpoint of trained model. 49 | :param sess: tf.Session(). 50 | :param global_steps: tensor, record the global step of training in epochs. 51 | :param model_name: str, the name of saved model. 52 | :param save_path: str, the path of saved model. 53 | :return: 54 | ''' 55 | saver = tf.train.Saver(max_to_keep=3) 56 | prefix_path = saver.save(sess, pjoin(save_path, model_name), global_step=global_steps) 57 | print(f'<< Saving model to {prefix_path} ...') 58 | -------------------------------------------------------------------------------- /model/FC_GRU.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import numpy as np 3 | import pickle 4 | import tensorflow as tf 5 | from tensorflow.contrib import rnn 6 | sys.path.append('./util/') 7 | from utils import * 8 | 9 | 10 | class FC_GRU(): 11 | def __init__(self, num_station, input_steps, 12 | num_layers=2, num_units=64, 13 | max_diffusion_step=2, 14 | dy_adj=1, 15 | dy_filter=0, 16 | f_adj_mx=None, 17 | filter_type='dual_random_walk', 18 | batch_size=32): 19 | self.num_station = num_station 20 | self.input_steps = input_steps 21 | self.num_units = num_units 22 | 23 | # self.max_diffusion_step = max_diffusion_step 24 | # self.dy_adj = dy_adj 25 | # self.dy_filter = dy_filter 26 | # self.f_adj_mx = f_adj_mx 27 | # self.filter_type = filter_type 28 | 29 | self.batch_size = batch_size 30 | 31 | self.weight_initializer = tf.contrib.layers.xavier_initializer() 32 | self.const_initializer = tf.constant_initializer() 33 | 34 | 35 | cells = [tf.contrib.rnn.GRUCell(self.num_units, name='gru_{0}'.format(i)) for i in range(num_layers)] 36 | # cell_with_projection = tf.contrib.rnn.BasicLSTMCell(self.num_station*2, forget_bias=1.0) 37 | # cells = [cell] * num_layers 38 | self.cells = tf.contrib.rnn.MultiRNNCell(cells, state_is_tuple=True) 39 | 40 | 41 | self.x = tf.placeholder(tf.float32, [self.batch_size, self.input_steps, self.num_station, 2]) 42 | self.f = tf.placeholder(tf.float32, [self.batch_size, self.input_steps, self.num_station, self.num_station]) 43 | self.y = tf.placeholder(tf.float32, [self.batch_size, self.input_steps, self.num_station, 2]) 44 | 45 | 46 | 47 | def build_easy_model(self): 48 | x = tf.transpose(tf.reshape(self.x, (self.batch_size, self.input_steps, -1)), [1, 0, 2]) 49 | #f_all = tf.transpose(tf.reshape(self.f, (self.batch_size, self.input_steps, -1)), [1, 0, 2]) 50 | #inputs = tf.concat([x, f_all], axis=-1) 51 | #inputs = tf.unstack(inputs, axis=0) 52 | inputs = tf.unstack(x, axis=0) 53 | # 54 | outputs, _ = tf.contrib.rnn.static_rnn(self.cells, inputs, dtype=tf.float32) 55 | outputs = tf.stack(outputs) 56 | # projection 57 | outputs = tf.layers.dense(tf.reshape(outputs, (-1, self.num_units)), units=self.num_station*2, activation=None, kernel_initializer=self.weight_initializer) 58 | # 59 | outputs = tf.reshape(outputs, (self.input_steps, self.batch_size, self.num_station, -1)) 60 | outputs = tf.transpose(outputs, [1, 0, 2, 3]) 61 | # outputs = outputs + self.x 62 | loss = 2*tf.nn.l2_loss(self.y - outputs) 63 | return outputs, loss 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /model/FC_LSTM.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import numpy as np 3 | import pickle 4 | import tensorflow as tf 5 | from tensorflow.contrib import rnn 6 | sys.path.append('./util/') 7 | from utils import * 8 | 9 | 10 | class FC_LSTM(): 11 | def __init__(self, num_station, input_steps, 12 | num_layers=2, num_units=64, 13 | max_diffusion_step=2, 14 | dy_adj=1, 15 | dy_filter=0, 16 | f_adj_mx=None, 17 | filter_type='dual_random_walk', 18 | batch_size=32): 19 | self.num_station = num_station 20 | self.input_steps = input_steps 21 | self.num_units = num_units 22 | 23 | # self.max_diffusion_step = max_diffusion_step 24 | # self.dy_adj = dy_adj 25 | # self.dy_filter = dy_filter 26 | # self.f_adj_mx = f_adj_mx 27 | # self.filter_type = filter_type 28 | 29 | self.batch_size = batch_size 30 | 31 | self.weight_initializer = tf.contrib.layers.xavier_initializer() 32 | self.const_initializer = tf.constant_initializer() 33 | 34 | 35 | cells = [tf.contrib.rnn.BasicLSTMCell(self.num_units, forget_bias=1.0, name='lstm_{0}'.format(i)) for i in range(num_layers)] 36 | # cell_with_projection = tf.contrib.rnn.BasicLSTMCell(self.num_station*2, forget_bias=1.0) 37 | # cells = [cell] * num_layers 38 | self.cells = tf.contrib.rnn.MultiRNNCell(cells, state_is_tuple=True) 39 | 40 | 41 | self.x = tf.placeholder(tf.float32, [self.batch_size, self.input_steps, self.num_station, 2]) 42 | self.f = tf.placeholder(tf.float32, [self.batch_size, self.input_steps, self.num_station, self.num_station]) 43 | self.y = tf.placeholder(tf.float32, [self.batch_size, self.input_steps, self.num_station, 2]) 44 | 45 | 46 | 47 | def build_easy_model(self): 48 | x = tf.transpose(tf.reshape(self.x, (self.batch_size, self.input_steps, -1)), [1, 0, 2]) 49 | #f_all = tf.transpose(tf.reshape(self.f, (self.batch_size, self.input_steps, -1)), [1, 0, 2]) 50 | #inputs = tf.concat([x, f_all], axis=-1) 51 | #inputs = tf.unstack(inputs, axis=0) 52 | inputs = tf.unstack(x, axis=0) 53 | # 54 | outputs, _ = tf.contrib.rnn.static_rnn(self.cells, inputs, dtype=tf.float32) 55 | outputs = tf.stack(outputs) 56 | # projection 57 | outputs = tf.layers.dense(tf.reshape(outputs, (-1, self.num_units)), units=self.num_station*2, activation=None, kernel_initializer=self.weight_initializer) 58 | # 59 | outputs = tf.reshape(outputs, (self.input_steps, self.batch_size, self.num_station, -1)) 60 | outputs = tf.transpose(outputs, [1, 0, 2, 3]) 61 | # outputs = outputs + self.x 62 | loss = 2*tf.nn.l2_loss(self.y - outputs) 63 | return outputs, loss 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /baseline/STGCN/utils/math_utils.py: -------------------------------------------------------------------------------- 1 | # @Time : Jan. 10, 2019 15:15 2 | # @Author : Veritas YIN 3 | # @FileName : math_utils.py 4 | # @Version : 1.0 5 | # @IDE : PyCharm 6 | # @Github : https://github.com/VeritasYin/Project_Orion 7 | 8 | import numpy as np 9 | 10 | 11 | def z_score(x, mean, std): 12 | ''' 13 | Z-score normalization function: $z = (X - \mu) / \sigma $, 14 | where z is the z-score, X is the value of the element, 15 | $\mu$ is the population mean, and $\sigma$ is the standard deviation. 16 | :param x: np.ndarray, input array to be normalized. 17 | :param mean: float, the value of mean. 18 | :param std: float, the value of standard deviation. 19 | :return: np.ndarray, z-score normalized array. 20 | ''' 21 | return (x - mean) / std 22 | 23 | 24 | def z_inverse(x, mean, std): 25 | ''' 26 | The inverse of function z_score(). 27 | :param x: np.ndarray, input to be recovered. 28 | :param mean: float, the value of mean. 29 | :param std: float, the value of standard deviation. 30 | :return: np.ndarray, z-score inverse array. 31 | ''' 32 | return x * std + mean 33 | 34 | 35 | def MAPE(v, v_): 36 | ''' 37 | Mean absolute percentage error. 38 | :param v: np.ndarray or int, ground truth. 39 | :param v_: np.ndarray or int, prediction. 40 | :return: int, MAPE averages on all elements of input. 41 | ''' 42 | return np.mean(np.abs(v_ - v) / (v + 1e-5)) 43 | 44 | 45 | def RMSE(v, v_): 46 | ''' 47 | Mean squared error. 48 | :param v: np.ndarray or int, ground truth. 49 | :param v_: np.ndarray or int, prediction. 50 | :return: int, RMSE averages on all elements of input. 51 | ''' 52 | return np.sqrt(np.mean((v_ - v) ** 2)) 53 | 54 | 55 | def MAE(v, v_): 56 | ''' 57 | Mean absolute error. 58 | :param v: np.ndarray or int, ground truth. 59 | :param v_: np.ndarray or int, prediction. 60 | :return: int, MAE averages on all elements of input. 61 | ''' 62 | return np.mean(np.abs(v_ - v)) 63 | 64 | 65 | def evaluation(y, y_, x_stats): 66 | ''' 67 | Evaluation function: interface to calculate MAPE, MAE and RMSE between ground truth and prediction. 68 | Extended version: multi-step prediction can be calculated by self-calling. 69 | :param y: np.ndarray or int, ground truth. 70 | :param y_: np.ndarray or int, prediction. 71 | :param x_stats: dict, paras of z-scores (mean & std). 72 | :return: np.ndarray, averaged metric values. 73 | ''' 74 | dim = len(y_.shape) 75 | 76 | if dim == 3: 77 | # single_step case 78 | v = z_inverse(y, x_stats['mean'], x_stats['std']) 79 | v_ = z_inverse(y_, x_stats['mean'], x_stats['std']) 80 | return np.array([MAPE(v, v_), MAE(v, v_), RMSE(v, v_)]) 81 | else: 82 | # multi_step case 83 | tmp_list = [] 84 | # y -> [time_step, batch_size, n_route, 1] 85 | y = np.swapaxes(y, 0, 1) 86 | # recursively call 87 | print(y_.shape) 88 | for i in range(y_.shape[0]): 89 | tmp_res = evaluation(y[i], y_[i], x_stats) 90 | tmp_list.append(tmp_res) 91 | return np.concatenate(tmp_list, axis=-1) 92 | -------------------------------------------------------------------------------- /baseline/STGCN/utils/math_graph.py: -------------------------------------------------------------------------------- 1 | # @Time : Jan. 10, 2019 15:21 2 | # @Author : Veritas YIN 3 | # @FileName : math_graph.py 4 | # @Version : 1.0 5 | # @IDE : PyCharm 6 | # @Github : https://github.com/VeritasYin/Project_Orion 7 | 8 | import numpy as np 9 | import pandas as pd 10 | import tensorflow as tf 11 | from scipy.sparse.linalg import eigs 12 | 13 | 14 | def scaled_laplacian(W): 15 | ''' 16 | Normalized graph Laplacian function. 17 | :param W: np.ndarray, [n_route, n_route], weighted adjacency matrix of G. 18 | :return: np.matrix, [n_route, n_route]. 19 | ''' 20 | # d -> diagonal degree matrix 21 | n, d = np.shape(W)[0], np.sum(W, axis=1) 22 | # L -> graph Laplacian 23 | L = -W 24 | L[np.diag_indices_from(L)] = d 25 | for i in range(n): 26 | for j in range(n): 27 | if (d[i] > 0) and (d[j] > 0): 28 | L[i, j] = L[i, j] / np.sqrt(d[i] * d[j]) 29 | # lambda_max \approx 2.0, the largest eigenvalues of L. 30 | lambda_max = eigs(L, k=1, which='LR')[0][0].real 31 | return np.mat(2 * L / lambda_max - np.identity(n)) 32 | 33 | 34 | def cheb_poly_approx(L, Ks, n): 35 | ''' 36 | Chebyshev polynomials approximation function. 37 | :param L: np.matrix, [n_route, n_route], graph Laplacian. 38 | :param Ks: int, kernel size of spatial convolution. 39 | :param n: int, number of routes / size of graph. 40 | :return: np.ndarray, [n_route, Ks*n_route]. 41 | ''' 42 | L0, L1 = np.mat(np.identity(n)), np.mat(np.copy(L)) 43 | 44 | if Ks > 1: 45 | L_list = [np.copy(L0), np.copy(L1)] 46 | for i in range(Ks - 2): 47 | Ln = np.mat(2 * L * L1 - L0) 48 | L_list.append(np.copy(Ln)) 49 | L0, L1 = np.matrix(np.copy(L1)), np.matrix(np.copy(Ln)) 50 | # L_lsit [Ks, n*n], Lk [n, Ks*n] 51 | return np.concatenate(L_list, axis=-1) 52 | elif Ks == 1: 53 | return np.asarray(L0) 54 | else: 55 | raise ValueError('ERROR: the size of spatial kernel must be greater than 1, but received "{0}".'.format(Ks)) 56 | 57 | 58 | 59 | def first_approx(W, n): 60 | ''' 61 | 1st-order approximation function. 62 | :param W: np.ndarray, [n_route, n_route], weighted adjacency matrix of G. 63 | :param n: int, number of routes / size of graph. 64 | :return: np.ndarray, [n_route, n_route]. 65 | ''' 66 | A = W + np.identity(n) 67 | d = np.sum(A, axis=1) 68 | sinvD = np.sqrt(np.mat(np.diag(d)).I) 69 | # refer to Eq.5 70 | return np.mat(np.identity(n) + sinvD * A * sinvD) 71 | 72 | 73 | def weight_matrix(file_path, sigma2=0.1, epsilon=0.5, scaling=True): 74 | ''' 75 | Load weight matrix function. 76 | :param file_path: str, the path of saved weight matrix file. 77 | :param sigma2: float, scalar of matrix W. 78 | :param epsilon: float, thresholds to control the sparsity of matrix W. 79 | :param scaling: bool, whether applies numerical scaling on W. 80 | :return: np.ndarray, [n_route, n_route]. 81 | ''' 82 | try: 83 | W = pd.read_csv(file_path, header=None).values 84 | except FileNotFoundError: 85 | print('ERROR: input file was not found in {0}.'.format(file_path)) 86 | 87 | # check whether W is a 0/1 matrix. 88 | if set(np.unique(W)) == {0, 1}: 89 | print('The input graph is a 0/1 matrix; set "scaling" to False.') 90 | scaling = False 91 | 92 | if scaling: 93 | n = W.shape[0] 94 | W = W / 10000. 95 | W2, W_mask = W * W, np.ones([n, n]) - np.identity(n) 96 | # refer to Eq.10 97 | return np.exp(-W2 / sigma2) * (np.exp(-W2 / sigma2) >= epsilon) * W_mask 98 | else: 99 | return W 100 | -------------------------------------------------------------------------------- /baseline/DMVSTNet/get_embeddings.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import argparse 4 | from utils import * 5 | #from dtw import dtw 6 | from fastdtw import fastdtw 7 | from scipy.spatial.distance import euclidean 8 | import math 9 | import pandas as pd 10 | 11 | 12 | def getGraphEmbedding(data, week_length=24 * 7, output_folder='./'): 13 | """ 14 | :param data: t x w x h x c 15 | :return: 16 | """ 17 | #embedding = None 18 | #graph = np.zeros([data.shape[1] * data.shape[2], data.shape[1] * data.shape[2]], dtype=np.float32) 19 | s = np.zeros([week_length, data.shape[1], data.shape[2]]) 20 | count = 0 21 | for i in range(0, data.shape[0], week_length): 22 | if i + week_length > data.shape[0]: 23 | break 24 | s += data[i:i + week_length, :, :] 25 | count += 1 26 | s = s / count 27 | s = np.reshape(s, [s.shape[0], -1]) 28 | # for i in range(s.shape[1]): 29 | # for j in range(i, s.shape[1]): 30 | # graph[i, j], _, _, _ = dtw(s[:, i], s[:, j], dist=lambda x, y: abs(x - y)) 31 | # #print("at %d %d " % (i, j)) 32 | print('fastdtw...') 33 | dtw_output = [[fastdtw(s[:,i], s[:,j], dist=euclidean)[0] for j in range(i, s.shape[1])] for i in range(s.shape[1])] 34 | print('save graph data...') 35 | dump_pickle(dtw_output, os.path.join(output_folder, 'graph.pkl')) 36 | #np.save(os.path.join(output_folder, 'graph.npy'), graph) 37 | # graph = np.load("./graph.npy") 38 | print('generate graph_embedding_input.txt ') 39 | with open(os.path.join(output_folder, 'graph_embedding_input.txt'), 'w') as f: 40 | # for i in range(graph.shape[0]): 41 | # for j in range(i, graph.shape[1]): 42 | # f.write(str(i) + " " + str(j) + " " + str(graph[i, j]) + "\n") 43 | # f.write(str(j) + " " + str(i) + " " + str(graph[i, j]) + "\n") 44 | for i in range(s.shape[1]): 45 | for j in range(i, s.shape[1]): 46 | f.write(str(i) + " " + str(j) + " " + str(dtw_output[i][j-i]) + "\n") 47 | f.write(str(j) + " " + str(i) + " " + str(dtw_output[i][j-i]) + "\n") 48 | 49 | 50 | def main(): 51 | parse = argparse.ArgumentParser() 52 | parse.add_argument('-dataset', '--dataset', type=str, default='didi') 53 | # parse.add_argument('-dataset', '--dataset', type=str, default='citibike') 54 | # parse.add_argument('-dataset', '--dataset', type=str, default='taxi') 55 | parse.add_argument('-predict_steps', '--predict_steps', type=int, default=1, help='prediction steps') 56 | parse.add_argument('-input_steps', '--input_steps', type=int, default=6, help='number of input steps') 57 | parse.add_argument('-dim', '--dim', type=int, default=0, help='dim of data to be processed') 58 | # 59 | args = parse.parse_args() 60 | # 61 | data_folder = '../../datasets/' + args.dataset + '-data/data/' 62 | if args.dim > 0: 63 | output_folder = './data/' + args.dataset + '/dim1/' 64 | else: 65 | output_folder = './data/' + args.dataset 66 | if not os.path.exists(output_folder): 67 | os.makedirs(output_folder) 68 | print('load train, test data...') 69 | if 'taxi' in args.dataset: 70 | split = [11640, 744, 720] 71 | data_folder = '../../datasets/' + args.dataset + '-data/graph-data/' 72 | data, train_data, val_data, test_data = load_npy_data(filename=[data_folder + 'nyc_taxi_data.npy'], split=split) 73 | train_data = np.reshape(train_data, [train_data.shape[0], 20, 10, -1]) 74 | p = 24 * 7 75 | elif 'didi' in args.dataset: 76 | split = [2400, 192, 288] 77 | data, train_data, val_data, test_data = load_npy_data(filename=[data_folder + 'cd_didi_data.npy'], split=split) 78 | train_data = np.reshape(train_data, [train_data.shape[0], 20, 20, -1]) 79 | p = 24 * 7 * 4 80 | print(train_data.shape) 81 | train_emb_data = train_data[..., args.dim] 82 | getGraphEmbedding(train_emb_data, week_length=p, output_folder=output_folder) 83 | 84 | 85 | if __name__ == '__main__': 86 | main() -------------------------------------------------------------------------------- /baseline/STGCN/main.py: -------------------------------------------------------------------------------- 1 | # @Time : Jan. 02, 2019 22:17 2 | # @Author : Veritas YIN 3 | # @FileName : main.py 4 | # @Version : 1.0 5 | # @Project : Orion 6 | # @IDE : PyCharm 7 | # @Github : https://github.com/VeritasYin/Project_Orion 8 | 9 | import os 10 | 11 | #os.environ["CUDA_VISIBLE_DEVICES"] = "0" 12 | from os.path import join as pjoin 13 | 14 | import tensorflow as tf 15 | 16 | config = tf.ConfigProto() 17 | config.gpu_options.allow_growth = True 18 | tf.Session(config=config) 19 | 20 | from utils.math_graph import * 21 | from data_loader.data_utils import * 22 | from models.trainer import model_train 23 | from models.tester import model_test 24 | 25 | import argparse 26 | 27 | parser = argparse.ArgumentParser() 28 | parser.add_argument('-gpu', '--gpu', type=str, default='0', help='which gpu to use: 0 or 1') 29 | parser.add_argument('-dataset', '--dataset', type=str, default='citibike', help='datasets: citibike') 30 | parser.add_argument('-input_steps', '--input_steps', type=int, default=6, help='number of input steps') 31 | parser.add_argument('-model_save', '--model_save', type=str, default='', help='path to save model') 32 | parser.add_argument('-trained_adj_mx', '--trained_adj_mx', type=int, default=0, help='if training adjacent matrix') 33 | parser.add_argument('-delta', '--delta', type=int, default=1e7, help='delta to calculate rescaled weighted matrix') 34 | parser.add_argument('-epsilon', '--epsilon', type=float, default=0.8, help='epsilon to calculate rescaled weighted matrix') 35 | # 36 | parser.add_argument('--n_route', type=int, default=0) 37 | parser.add_argument('--n_his', type=int, default=6) 38 | parser.add_argument('--n_pred', type=int, default=1) 39 | parser.add_argument('-batch_size', '--batch_size', type=int, default=50) 40 | parser.add_argument('-epoch', '--epoch', type=int, default=50) 41 | parser.add_argument('--save', type=int, default=10) 42 | parser.add_argument('--ks', type=int, default=3) 43 | parser.add_argument('--kt', type=int, default=3) 44 | parser.add_argument('-lr', '--lr', type=float, default=1e-3) 45 | parser.add_argument('--opt', type=str, default='RMSProp') 46 | parser.add_argument('--graph', type=str, default='default') 47 | parser.add_argument('--inf_mode', type=str, default='sep') 48 | 49 | args = parser.parse_args() 50 | os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu 51 | # 52 | data_folder = '../../datasets/' + args.dataset + '-data/data/' 53 | output_folder = os.path.join('./data', args.dataset, 'model_save', args.model_save) 54 | if not os.path.exists(output_folder): 55 | os.makedirs(output_folder) 56 | 57 | print(f'Training configs: {args}') 58 | 59 | # Data Preprocessing 60 | if 'citibike' in args.dataset: 61 | split = [3672, 240, 480] 62 | data, n_route = data_gen_2(filename=[data_folder+'d_station.npy', data_folder+'p_station.npy'], split=split, 63 | n_frame=args.input_steps+1) 64 | args.n_route = n_route 65 | 66 | print(f'>> Loading dataset with Mean: {data.mean:.2f}, STD: {data.std:.2f}') 67 | 68 | n, n_his, n_pred = args.n_route, args.n_his, args.n_pred 69 | Ks, Kt = args.ks, args.kt 70 | # blocks: settings of channel size in st_conv_blocks / bottleneck design 71 | #blocks = [[2, 32, 64], [64, 32, 128]] 72 | blocks = [[2, 32, 64]] 73 | 74 | # Load wighted adjacency matrix W 75 | if args.trained_adj_mx: 76 | L = tf.get_variable('weight_matrix', shape=(n, n), dtype=tf.float32) 77 | Lk = cheb_poly_approx_tf(L, Ks, n) 78 | #W = weight_matrix(pjoin('./dataset', f'PeMSD7_W_{n}.csv')) 79 | tf.add_to_collection(name='graph_kernel', value=Lk) 80 | else: 81 | # load customized graph weight matrix 82 | #W = weight_matrix(pjoin('./dataset', args.graph)) 83 | w = np.load(data_folder + 'w.npy') 84 | #w = np.array(w, dtype=np.float32) 85 | W = get_rescaled_W(w, delta=args.delta, epsilon=args.epsilon) 86 | # Calculate graph kernel 87 | L = scaled_laplacian(W) 88 | # Alternative approximation method: 1st approx - first_approx(W, n). 89 | Lk = cheb_poly_approx(L, Ks, n) 90 | tf.add_to_collection(name='graph_kernel', value=tf.cast(tf.constant(Lk), tf.float32)) 91 | 92 | 93 | 94 | if __name__ == '__main__': 95 | model_train(data, blocks, args) 96 | model_test(data, data.get_len('test'), n_his, n_pred, args.inf_mode) 97 | -------------------------------------------------------------------------------- /model/ConvLSTM.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import numpy as np 3 | import pickle 4 | import tensorflow as tf 5 | 6 | sys.path.append('./util/') 7 | from utils import * 8 | from model.dcrnn_cell import DCGRUCell 9 | from model.convlstm_cell import Dy_Conv2DLSTMCell 10 | 11 | 12 | class ConvLSTM(): 13 | def __init__(self, input_shape=[20,10,2], input_steps=6, 14 | num_layers=3, num_units=64, kernel_shape=[3,3], 15 | dy_adj=0, 16 | dy_filter=0, 17 | batch_size=32): 18 | self.input_shape = input_shape 19 | self.input_steps = input_steps 20 | self.num_layers = num_layers 21 | self.num_units = num_units 22 | self.kernel_shape = kernel_shape 23 | self.dy_adj = dy_adj 24 | self.dy_filter = dy_filter 25 | 26 | self.batch_size = batch_size 27 | 28 | self.weight_initializer = tf.contrib.layers.xavier_initializer() 29 | self.const_initializer = tf.constant_initializer() 30 | 31 | # if self.dy_adj > 0: 32 | # output_dy_adj = True 33 | # else: 34 | # output_dy_adj = False 35 | 36 | first_cell = Dy_Conv2DLSTMCell(input_shape=self.input_shape, 37 | output_channels=self.num_units, 38 | kernel_shape=self.kernel_shape, 39 | input_dim=self.input_shape[-1], dy_adj=self.dy_adj, dy_filter=self.dy_filter, output_dy_adj=self.dy_adj) 40 | cell = Dy_Conv2DLSTMCell(input_shape=[self.input_shape[0], self.input_shape[1], self.num_units], 41 | output_channels=self.num_units, 42 | kernel_shape=self.kernel_shape, 43 | input_dim=self.num_units, dy_adj=self.dy_adj, dy_filter=self.dy_filter, output_dy_adj=self.dy_adj) 44 | last_cell = Dy_Conv2DLSTMCell(input_shape=[self.input_shape[0], self.input_shape[1], self.num_units], 45 | output_channels=self.num_units, 46 | kernel_shape=self.kernel_shape, 47 | input_dim=self.num_units, dy_adj=self.dy_adj, dy_filter=self.dy_filter, output_dy_adj=0) 48 | 49 | if num_layers > 2: 50 | cells = [first_cell] + [cell] * (num_layers-2) + [last_cell] 51 | else: 52 | cells = [first_cell, last_cell] 53 | 54 | self.cells = tf.contrib.rnn.MultiRNNCell(cells, state_is_tuple=True) 55 | 56 | self.x = tf.placeholder(tf.float32, [self.batch_size, self.input_steps, self.input_shape[0], self.input_shape[1], self.input_shape[2]]) 57 | self.f = tf.placeholder(tf.float32, 58 | [self.batch_size, self.input_steps, self.input_shape[0] * self.input_shape[1], 59 | self.input_shape[0] * self.input_shape[1]]) 60 | self.y = tf.placeholder(tf.float32, [self.batch_size, self.input_steps, self.input_shape[0], self.input_shape[1], self.input_shape[2]]) 61 | 62 | 63 | def build_easy_model(self): 64 | x = tf.transpose(tf.reshape(self.x, (self.batch_size, self.input_steps, self.input_shape[0], self.input_shape[1], -1)), [1, 0, 2, 3, 4]) 65 | inputs = tf.unstack(x, axis=0) 66 | #f_all = tf.transpose(tf.reshape(self.f, (self.batch_size, self.input_steps, self.input_shape[0], self.input_shape[1], self.input_shape[0] * self.input_shape[1])), [1, 0, 2, 3, 4]) 67 | #inputs = tf.concat([x, f_all], axis=-1) 68 | #inputs = tf.unstack(inputs, axis=0) 69 | # 70 | outputs, _ = tf.contrib.rnn.static_rnn(self.cells, inputs, dtype=tf.float32) 71 | outputs = tf.stack(outputs) 72 | # 73 | #print(outputs.get_shape().as_list()) 74 | # projection 75 | outputs = tf.layers.dense(tf.reshape(outputs, (-1, self.num_units)), units=self.input_shape[-1], 76 | activation=None, kernel_initializer=self.weight_initializer) 77 | # 78 | #print(outputs.get_shape().as_list()) 79 | outputs = tf.reshape(outputs, (self.input_steps, self.batch_size, self.input_shape[0], self.input_shape[1], -1)) 80 | outputs = tf.transpose(outputs, [1, 0, 2, 3, 4]) 81 | loss = 2 * tf.nn.l2_loss(self.y - outputs) 82 | return outputs, loss 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /model/ConvGRU.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import numpy as np 3 | import pickle 4 | import tensorflow as tf 5 | 6 | sys.path.append('./util/') 7 | from utils import * 8 | from model.dcrnn_cell import DCGRUCell 9 | from model.convgru_cell import Dy_Conv2DGRUCell 10 | 11 | 12 | class ConvGRU(): 13 | def __init__(self, input_shape=[20, 10, 2], input_steps=6, 14 | num_layers=2, num_units=64, kernel_shape=[3,3], 15 | dy_adj=0, 16 | dy_filter=0, 17 | batch_size=32): 18 | self.input_shape = input_shape 19 | self.input_steps = input_steps 20 | self.num_layers = num_layers 21 | self.num_units = num_units 22 | self.kernel_shape = kernel_shape 23 | self.dy_adj = dy_adj 24 | self.dy_filter = dy_filter 25 | 26 | self.batch_size = batch_size 27 | 28 | self.weight_initializer = tf.contrib.layers.xavier_initializer() 29 | self.const_initializer = tf.constant_initializer() 30 | 31 | # if self.dy_adj > 0: 32 | # output_dy_adj = True 33 | # else: 34 | # output_dy_adj = False 35 | 36 | first_cell = Dy_Conv2DGRUCell(input_shape=self.input_shape, 37 | output_channels=self.num_units, 38 | kernel_shape=self.kernel_shape, 39 | input_dim=self.input_shape[-1], dy_adj=self.dy_adj, dy_filter=self.dy_filter, output_dy_adj=self.dy_adj) 40 | cell = Dy_Conv2DGRUCell(input_shape=[self.input_shape[0], self.input_shape[1], self.num_units], 41 | output_channels=self.num_units, 42 | kernel_shape=self.kernel_shape, 43 | input_dim=self.num_units, dy_adj=self.dy_adj, dy_filter=self.dy_filter, output_dy_adj=self.dy_adj) 44 | last_cell = Dy_Conv2DGRUCell(input_shape=[self.input_shape[0], self.input_shape[1], self.num_units], 45 | output_channels=self.num_units, 46 | kernel_shape=self.kernel_shape, 47 | input_dim=self.num_units, dy_adj=self.dy_adj, dy_filter=self.dy_filter, output_dy_adj=0) 48 | 49 | if num_layers > 2: 50 | cells = [first_cell] + [cell] * (num_layers-2) + [last_cell] 51 | else: 52 | cells = [first_cell, last_cell] 53 | 54 | #cells = [first_cell] + [cell] * (num_layers - 1) 55 | 56 | self.cells = tf.contrib.rnn.MultiRNNCell(cells, state_is_tuple=True) 57 | 58 | self.x = tf.placeholder(tf.float32, 59 | [self.batch_size, self.input_steps, self.input_shape[0], self.input_shape[1], 60 | self.input_shape[2]]) 61 | self.f = tf.placeholder(tf.float32, 62 | [self.batch_size, self.input_steps, self.input_shape[0] * self.input_shape[1], 63 | self.input_shape[0] * self.input_shape[1]]) 64 | self.y = tf.placeholder(tf.float32, 65 | [self.batch_size, self.input_steps, self.input_shape[0], self.input_shape[1], 66 | self.input_shape[2]]) 67 | 68 | 69 | def build_easy_model(self): 70 | x = tf.transpose(tf.reshape(self.x, (self.batch_size, self.input_steps, self.input_shape[0], self.input_shape[1], -1)), [1, 0, 2, 3, 4]) 71 | inputs = tf.unstack(x, axis=0) 72 | # f_all = tf.transpose(tf.reshape(self.f, (self.batch_size, self.input_steps, self.input_shape[0], self.input_shape[1], -1)), [1, 0, 2, 3, 4]) 73 | # inputs = tf.concat([x, f_all], axis=-1) 74 | # inputs = tf.unstack(inputs, axis=0) 75 | # 76 | outputs, _ = tf.contrib.rnn.static_rnn(self.cells, inputs, dtype=tf.float32) 77 | outputs = tf.stack(outputs) 78 | # 79 | # projection 80 | outputs = tf.layers.dense(tf.reshape(outputs, (-1, self.num_units)), units=self.input_shape[-1], 81 | activation=None, kernel_initializer=self.weight_initializer) 82 | # 83 | outputs = tf.reshape(outputs, (self.input_steps, self.batch_size, self.input_shape[0], self.input_shape[1], -1)) 84 | outputs = tf.transpose(outputs, [1, 0, 2, 3, 4]) 85 | loss = 2 * tf.nn.l2_loss(self.y - outputs) 86 | return outputs, loss 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /preprocessing.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | class MinMaxNormalization01(object): 4 | def __init__(self, ): 5 | pass 6 | 7 | def fit(self, data): 8 | self._min = np.amin(data) 9 | self._max = np.amax(data) 10 | print("min: ", self._min, "max:", self._max) 11 | 12 | def transform(self, data): 13 | norm_data = 1. * (data - self._min) / (self._max - self._min) 14 | return norm_data 15 | 16 | def fit_transform(self, data): 17 | self.fit(data) 18 | return self.transform(data) 19 | 20 | def inverse_transform(self, data, mean_index=0, if_add_mean=False): 21 | inverse_norm_data = 1. * data * (self._max - self._min) + self._min 22 | return inverse_norm_data 23 | 24 | def real_loss(self, loss): 25 | # loss is rmse 26 | return loss*(self._max - self._min) 27 | #return real_loss 28 | 29 | class MinMaxNormalization01_minus_mean(object): 30 | def __init__(self, period=24*7): 31 | self.period = period 32 | pass 33 | 34 | def fit(self, data): 35 | self._min = np.amin(data) 36 | self._max = np.amax(data) 37 | print("min: ", self._min, "max:", self._max) 38 | # 39 | d_shape = np.array(data).shape 40 | #self._mean = np.zeros((self.period, d_shape[1], d_shape[2])) 41 | self._mean = np.zeros([self.period] + list(d_shape[1:])) 42 | index = np.arange(d_shape[0]//self.period) 43 | for i in range(self.period): 44 | self._mean[i] = np.mean(data[self.period*index+i], axis=0) 45 | self._mean = 1. * (self._mean - self._min) / (self._max - self._min) 46 | 47 | def transform(self, data, pre_index=0): 48 | norm_data = 1. * (data - self._min) / (self._max - self._min) 49 | mean_index = (np.arange(len(norm_data)) + pre_index)%self.period 50 | norm_minus_mean_data = norm_data - self._mean[mean_index] 51 | return norm_minus_mean_data 52 | 53 | def fit_transform(self, data): 54 | self.fit(data) 55 | return self.transform(data) 56 | 57 | def inverse_transform(self, data, mean_index=0, if_add_mean=True): 58 | if if_add_mean: 59 | index = mean_index%self.period 60 | inverse_norm_data = 1. * (data+self._mean[index]) * (self._max - self._min) + self._min 61 | else: 62 | inverse_norm_data = 1. * data * (self._max - self._min) + self._min 63 | return inverse_norm_data 64 | 65 | def real_loss(self, loss): 66 | # loss is rmse 67 | return loss*(self._max - self._min) 68 | #return real_loss 69 | 70 | class MinMaxNormalization01_by_axis(object): 71 | def __init__(self): 72 | pass 73 | def fit(self, data): 74 | self._min = np.amin(data, axis=0) 75 | self._max = np.amax(data, axis=0) 76 | def transform(self, data): 77 | norm_data = 1. * (data - self._min) / (self._max - self._min) 78 | return norm_data 79 | def fit_transform(self, data): 80 | self.fit(data) 81 | return self.transform(data) 82 | def inverse_transform(self, data): 83 | inverse_norm_data = 1. * data * (self._max - self._min) + self._min 84 | return inverse_norm_data 85 | #def real_loss(self, loss): 86 | 87 | class MinMaxNormalization_neg_1_pos_1(object): 88 | def __init__(self): 89 | pass 90 | 91 | def fit(self, X): 92 | self._min = X.min() 93 | self._max = X.max() 94 | print("min:", self._min, "max:", self._max) 95 | 96 | def transform(self, X): 97 | X = 1. * (X - self._min) / (self._max - self._min) 98 | X = X * 2. - 1. 99 | return X 100 | 101 | def fit_transform(self, X): 102 | self.fit(X) 103 | return self.transform(X) 104 | 105 | def inverse_transform(self, X): 106 | X = (X + 1.)/2. 107 | X = 1. * X * (self._max - self._min) + self._min 108 | return X 109 | 110 | def real_loss(self, loss): 111 | # loss is rmse 112 | return loss*(self._max - self._min)/2. 113 | #return real_loss 114 | 115 | class StandardScaler(object): 116 | def __init__(self): 117 | pass 118 | 119 | def fit(self, data): 120 | self.mean = np.mean(data) 121 | self.std = np.std(data) 122 | print("mean: ", self.mean, " std:", self.std) 123 | 124 | def transform(self, data): 125 | return (data - self.mean) / self.std 126 | 127 | def fit_transform(self, data): 128 | self.fit(data) 129 | return self.transform(data) 130 | 131 | def inverse_transform(self, data, mean_index=0, if_add_mean=False): 132 | return (data * self.std) + self.mean 133 | 134 | def real_loss(self, loss): 135 | return loss*self.std -------------------------------------------------------------------------------- /baseline/STGCN/models/trainer.py: -------------------------------------------------------------------------------- 1 | # @Time : Jan. 13, 2019 20:16 2 | # @Author : Veritas YIN 3 | # @FileName : trainer.py 4 | # @Version : 1.0 5 | # @IDE : PyCharm 6 | # @Github : https://github.com/VeritasYin/Project_Orion 7 | 8 | from data_loader.data_utils import gen_batch 9 | from models.tester import model_inference 10 | from models.base_model import build_model, model_save 11 | from os.path import join as pjoin 12 | 13 | import tensorflow as tf 14 | import numpy as np 15 | import time 16 | 17 | 18 | def model_train(inputs, blocks, args, sum_path='./output/tensorboard'): 19 | ''' 20 | Train the base model. 21 | :param inputs: instance of class Dataset, data source for training. 22 | :param blocks: list, channel configs of st_conv blocks. 23 | :param args: instance of class argparse, args for training. 24 | ''' 25 | n, n_his, n_pred = args.n_route, args.n_his, args.n_pred 26 | Ks, Kt = args.ks, args.kt 27 | batch_size, epoch, inf_mode, opt = args.batch_size, args.epoch, args.inf_mode, args.opt 28 | 29 | # Placeholder for model training 30 | x = tf.placeholder(tf.float32, [None, n_his + 1, n, 2], name='data_input') 31 | keep_prob = tf.placeholder(tf.float32, name='keep_prob') 32 | 33 | # Define model loss 34 | train_loss, pred = build_model(x, n_his, Ks, Kt, blocks, keep_prob) 35 | tf.summary.scalar('train_loss', train_loss) 36 | copy_loss = tf.add_n(tf.get_collection('copy_loss')) 37 | tf.summary.scalar('copy_loss', copy_loss) 38 | 39 | # Learning rate settings 40 | global_steps = tf.Variable(0, trainable=False) 41 | len_train = inputs.get_len('train') 42 | if len_train % batch_size == 0: 43 | epoch_step = len_train / batch_size 44 | else: 45 | epoch_step = int(len_train / batch_size) + 1 46 | # Learning rate decay with rate 0.7 every 5 epochs. 47 | lr = tf.train.exponential_decay(args.lr, global_steps, decay_steps=5 * epoch_step, decay_rate=0.7, staircase=True) 48 | tf.summary.scalar('learning_rate', lr) 49 | step_op = tf.assign_add(global_steps, 1) 50 | with tf.control_dependencies([step_op]): 51 | if opt == 'RMSProp': 52 | train_op = tf.train.RMSPropOptimizer(lr).minimize(train_loss) 53 | elif opt == 'ADAM': 54 | train_op = tf.train.AdamOptimizer(lr).minimize(train_loss) 55 | else: 56 | raise ValueError('ERROR: optimizer "{0}" is not defined.'.format(opt)) 57 | 58 | merged = tf.summary.merge_all() 59 | 60 | with tf.Session() as sess: 61 | writer = tf.summary.FileWriter(pjoin(sum_path, 'train'), sess.graph) 62 | sess.run(tf.global_variables_initializer()) 63 | 64 | if inf_mode == 'sep': 65 | # for inference mode 'sep', the type of step index is int. 66 | step_idx = n_pred - 1 67 | tmp_idx = [step_idx] 68 | min_val = min_va_val = np.array([4e1, 1e5, 1e5]) 69 | elif inf_mode == 'merge': 70 | # for inference mode 'merge', the type of step index is np.ndarray. 71 | step_idx = tmp_idx = np.arange(3, n_pred + 1, 3) - 1 72 | min_val = min_va_val = np.array([4e1, 1e5, 1e5] * len(step_idx)) 73 | else: 74 | raise ValueError('ERROR: test mode "{0}" is not defined.'.format(inf_mode)) 75 | 76 | for i in range(epoch): 77 | start_time = time.time() 78 | for j, x_batch in enumerate( 79 | gen_batch(inputs.get_data('train'), batch_size, dynamic_batch=True, shuffle=True)): 80 | summary, _ = sess.run([merged, train_op], feed_dict={x: x_batch[:, 0:n_his + 1, :, :], keep_prob: 1.0}) 81 | writer.add_summary(summary, i * epoch_step + j) 82 | if j % 50 == 0: 83 | loss_value = \ 84 | sess.run([train_loss, copy_loss], 85 | feed_dict={x: x_batch[:, 0:n_his + 1, :, :], keep_prob: 1.0}) 86 | print(f'Epoch {i:2d}, Step {j:3d}: [{loss_value[0]:.3f}, {loss_value[1]:.3f}]') 87 | print(f'Epoch {i:2d} Training Time {time.time() - start_time:.3f}s') 88 | 89 | start_time = time.time() 90 | min_va_val, min_val = \ 91 | model_inference(sess, pred, inputs, batch_size, n_his, n_pred, step_idx, min_va_val, min_val) 92 | 93 | for ix in tmp_idx: 94 | #va, te = min_va_val[ix - 2:ix + 1], min_val[ix - 2:ix + 1] 95 | va, te = min_va_val, min_val 96 | print(f'Time Step {ix + 1}: ' 97 | f'MAPE {va[0]:7.3%}, {te[0]:7.3%}; ' 98 | f'MAE {va[1]:4.3f}, {te[1]:4.3f}; ' 99 | f'RMSE {va[2]:6.3f}, {te[2]:6.3f}.') 100 | print(f'Epoch {i:2d} Inference Time {time.time() - start_time:.3f}s') 101 | 102 | if (i + 1) % args.save == 0: 103 | model_save(sess, global_steps, 'STGCN') 104 | writer.close() 105 | print('Training model finished!') 106 | -------------------------------------------------------------------------------- /baseline/STGCN/models/tester.py: -------------------------------------------------------------------------------- 1 | # @Time : Jan. 10, 2019 17:52 2 | # @Author : Veritas YIN 3 | # @FileName : tester.py 4 | # @Version : 1.0 5 | # @IDE : PyCharm 6 | # @Github : https://github.com/VeritasYin/Project_Orion 7 | 8 | from data_loader.data_utils import gen_batch 9 | from utils.math_utils import evaluation 10 | from os.path import join as pjoin 11 | 12 | import tensorflow as tf 13 | import numpy as np 14 | import time 15 | 16 | 17 | def multi_pred(sess, y_pred, seq, batch_size, n_his, n_pred, step_idx, dynamic_batch=True): 18 | ''' 19 | Multi_prediction function. 20 | :param sess: tf.Session(). 21 | :param y_pred: placeholder. 22 | :param seq: np.ndarray, [len_seq, n_frame, n_route, C_0]. 23 | :param batch_size: int, the size of batch. 24 | :param n_his: int, size of historical records for training. 25 | :param n_pred: int, the length of prediction. 26 | :param step_idx: int or list, index for prediction slice. 27 | :param dynamic_batch: bool, whether changes the batch size in the last one if its length is less than the default. 28 | :return y_ : tensor, 'sep' [len_inputs, n_route, 1]; 'merge' [step_idx, len_inputs, n_route, 1]. 29 | len_ : int, the length of prediction. 30 | ''' 31 | pred_list = [] 32 | for i in gen_batch(seq, min(batch_size, len(seq)), dynamic_batch=dynamic_batch): 33 | # Note: use np.copy() to avoid the modification of source data. 34 | test_seq = np.copy(i[:, 0:n_his + 1, :, :]) 35 | step_list = [] 36 | for j in range(n_pred): 37 | pred = sess.run(y_pred, 38 | feed_dict={'data_input:0': test_seq, 'keep_prob:0': 1.0}) 39 | if isinstance(pred, list): 40 | pred = np.array(pred[0]) 41 | test_seq[:, 0:n_his - 1, :, :] = test_seq[:, 1:n_his, :, :] 42 | test_seq[:, n_his - 1, :, :] = pred 43 | step_list.append(pred) 44 | pred_list.append(step_list) 45 | # pred_array -> [n_pred, batch_size, n_route, C_0) 46 | pred_array = np.concatenate(pred_list, axis=1) 47 | return pred_array[step_idx], pred_array.shape[1] 48 | 49 | 50 | def model_inference(sess, pred, inputs, batch_size, n_his, n_pred, step_idx, min_va_val, min_val): 51 | ''' 52 | Model inference function. 53 | :param sess: tf.Session(). 54 | :param pred: placeholder. 55 | :param inputs: instance of class Dataset, data source for inference. 56 | :param batch_size: int, the size of batch. 57 | :param n_his: int, the length of historical records for training. 58 | :param n_pred: int, the length of prediction. 59 | :param step_idx: int or list, index for prediction slice. 60 | :param min_va_val: np.ndarray, metric values on validation set. 61 | :param min_val: np.ndarray, metric values on test set. 62 | ''' 63 | x_val, x_test, x_stats = inputs.get_data('val'), inputs.get_data('test'), inputs.get_stats() 64 | 65 | if n_his + n_pred > x_val.shape[1]: 66 | raise ValueError('ERROR: the value of n_pred "{0}" exceeds the length limit.'.format(n_pred)) 67 | 68 | y_val, len_val = multi_pred(sess, pred, x_val, batch_size, n_his, n_pred, step_idx) 69 | evl_val = evaluation(x_val[0:len_val, step_idx + n_his, :, :], y_val, x_stats) 70 | 71 | # chks: indicator that reflects the relationship of values between evl_val and min_va_val. 72 | chks = evl_val < min_va_val 73 | # update the metric on test set, if model's performance got improved on the validation. 74 | if sum(chks): 75 | min_va_val[chks] = evl_val[chks] 76 | y_pred, len_pred = multi_pred(sess, pred, x_test, batch_size, n_his, n_pred, step_idx) 77 | evl_pred = evaluation(x_test[0:len_pred, step_idx + n_his, :, :], y_pred, x_stats) 78 | min_val = evl_pred 79 | return min_va_val, min_val 80 | 81 | 82 | def model_test(inputs, batch_size, n_his, n_pred, inf_mode, load_path='./output/models/'): 83 | ''' 84 | Load and test saved model from the checkpoint. 85 | :param inputs: instance of class Dataset, data source for test. 86 | :param batch_size: int, the size of batch. 87 | :param n_his: int, the length of historical records for training. 88 | :param n_pred: int, the length of prediction. 89 | :param inf_mode: str, test mode - 'merge / multi-step test' or 'separate / single-step test'. 90 | :param load_path: str, the path of loaded model. 91 | ''' 92 | start_time = time.time() 93 | model_path = tf.train.get_checkpoint_state(load_path).model_checkpoint_path 94 | 95 | test_graph = tf.Graph() 96 | 97 | with test_graph.as_default(): 98 | saver = tf.train.import_meta_graph(pjoin('{0}.meta'.format(model_path))) 99 | 100 | with tf.Session(graph=test_graph) as test_sess: 101 | saver.restore(test_sess, tf.train.latest_checkpoint(load_path)) 102 | print('>> Loading saved model from {0} ...'.format(model_path)) 103 | 104 | pred = test_graph.get_collection('y_pred') 105 | 106 | if inf_mode == 'sep': 107 | # for inference mode 'sep', the type of step index is int. 108 | step_idx = n_pred - 1 109 | tmp_idx = [step_idx] 110 | elif inf_mode == 'merge': 111 | # for inference mode 'merge', the type of step index is np.ndarray. 112 | step_idx = tmp_idx = np.arange(3, n_pred + 1, 3) - 1 113 | else: 114 | raise ValueError('ERROR: test mode "{0}" is not defined.'.format(inf_mode)) 115 | 116 | x_test, x_stats = inputs.get_data('test'), inputs.get_stats() 117 | 118 | y_test, len_test = multi_pred(test_sess, pred, x_test, batch_size, n_his, n_pred, step_idx) 119 | evl = evaluation(x_test[0:len_test, step_idx + n_his, :, :], y_test, x_stats) 120 | 121 | for ix in tmp_idx: 122 | #te = evl[ix - 2:ix + 1] 123 | te = evl 124 | print(f'Time Step {ix + 1}: MAPE {te[0]:7.3%}; MAE {te[1]:4.3f}; RMSE {te[2]:6.3f}.') 125 | print(f'Model Test Time {time.time() - start_time:.3f}s') 126 | print('Testing model finished!') 127 | -------------------------------------------------------------------------------- /baseline/STResNet/ResNet.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | import tensorflow as tf 4 | 5 | class ResNet(object): 6 | def __init__(self, input_conf=[[3,2,16,8],[4,2,16,8],[4,2,16,8],[8]], batch_size=32, layer={}, layer_param={}): 7 | # layer = ['conv', 'res_net', 'conv'] 8 | # layer_param = [ [[3,3], [1,1,1,1], 64], 9 | # [ 3, [ [[3,3], [1,1,1,1], 64], [[3,3], [1,1,1,1], 64] ] ], 10 | # [[3,3], [1,1,1,1], 2] ] 11 | self.input_conf = input_conf 12 | self.nb_flow = self.input_conf[0][1] 13 | self.row = self.input_conf[0][2] 14 | self.col = self.input_conf[0][3] 15 | self.batch_size = batch_size 16 | self.layer = layer 17 | self.layer_param = layer_param 18 | self.x_c = tf.placeholder(tf.float32, [None, self.row, self.col, self.input_conf[0][0]*self.nb_flow]) 19 | self.x_p = tf.placeholder(tf.float32, [None, self.row, self.col, self.input_conf[1][0]*self.nb_flow]) 20 | self.x_t = tf.placeholder(tf.float32, [None, self.row, self.col, self.input_conf[2][0]*self.nb_flow]) 21 | # for external input 22 | self.x_ext = tf.placeholder(tf.float32, [None, self.input_conf[-1][0]]) 23 | #conf = self.input_conf[0] 24 | self.y = tf.placeholder(tf.float32, [None, self.row, self.col, self.nb_flow]) 25 | 26 | self.weight_initializer = tf.contrib.layers.xavier_initializer() 27 | self.const_initializer = tf.constant_initializer() 28 | 29 | def conv(self, inputs, filter, strides, output_features, padding, idx): 30 | # param: filter, strides, output_features 31 | with tf.variable_scope('conv_{0}'.format(idx)) as scope: 32 | in_channels = inputs.get_shape().as_list()[3] 33 | w = tf.get_variable('w', [filter[0], filter[1], in_channels, output_features], initializer=self.weight_initializer) 34 | b = tf.get_variable('b', [output_features], initializer=self.const_initializer) 35 | y = tf.nn.conv2d(inputs, w, strides=strides, padding=padding) 36 | y_b = tf.nn.bias_add(y, b, name='wx_plus_b') 37 | y_relu = tf.nn.relu(y_b, name='out_conv_{0}'.format(idx)) 38 | return y_relu 39 | 40 | def conv_transpose(self, inputs, filter, strides, output_features, padding, idx): 41 | with tf.variable_scope('conv_transpose_{0}'.format(idx)) as scope: 42 | in_channels = inputs.get_shape().as_list()[3] 43 | w = tf.get_variable('w', [filter[0], filter[1], output_features, in_channels], initializer=self.weight_initializer) 44 | b = tf.get_variable('b', [output_features], initializer=self.const_initializer) 45 | output_shape = tf.stack([tf.shape(inputs)[0], tf.shape(inputs)[1]*strides[1], tf.shape(inputs)[2]*strides[2], output_features]) 46 | y = tf.nn.conv2d_transpose(inputs, w, output_shape, strides=strides, padding=padding) 47 | y_b = tf.nn.bias_add(y, b, name='wx_plus_b') 48 | y_relu = tf.nn.relu(y_b, name='out_conv_transpose_{0}'.format(idx)) 49 | return y_relu 50 | 51 | def res_unit(self, x, res_param, idx): 52 | res = x 53 | with tf.variable_scope('res_unit_{0}'.format(idx)) as scope: 54 | for i in range(len(res_param)): 55 | # TODO: 56 | # batch normalization 57 | res = self.conv(res, res_param[i][0], res_param[i][1], res_param[i][2], padding='SAME', idx=i) 58 | return x+res 59 | 60 | def res_net(self, x, unit_num, res_param, idx): 61 | y_ = x 62 | #unit_num = unit_num[0] 63 | with tf.variable_scope('res_net_{0}'.format(idx)) as scope: 64 | for i in range(unit_num): 65 | y_ = self.res_unit(y_, res_param=res_param, idx=i) 66 | return y_ 67 | 68 | 69 | def fusion(self, x, idx): 70 | # x: [batch_size, row, col, nb_flow] 71 | with tf.variable_scope('fusion_input_{0}'.format(idx)) as scope: 72 | shape = x.get_shape().as_list() 73 | w = tf.get_variable('w', [shape[1], shape[2]], initializer=self.weight_initializer) 74 | w_extend = tf.expand_dims(w,axis=-1) 75 | return tf.multiply(w_extend, x) 76 | 77 | def build_model(self): 78 | x = [self.x_c, self.x_p, self.x_t, self.x_ext] 79 | y = self.y 80 | #input_conf = self.input_conf 81 | layer = self.layer 82 | param = self.layer_param 83 | y_all = [] 84 | for i in range(len(x)-1): 85 | # i: inputs num 86 | with tf.variable_scope('input{0}'.format(i)): 87 | y_ = x[i] 88 | for l_i in range(len(layer)): 89 | # l_i: layers num of input i 90 | if layer[l_i]=='conv': 91 | y_ = self.conv(y_, param[l_i][0], param[l_i][1], param[l_i][2], padding='SAME', idx=l_i) 92 | if layer[l_i]=='res_net': 93 | y_ = self.res_net(y_, param[l_i][0], param[l_i][1], idx=l_i) 94 | # fusion 95 | # y_: [batch_size, row, col, channel] 96 | y_ = self.fusion(y_, idx=i) 97 | y_all.append(y_) 98 | # sum fusion 99 | y_all = tf.stack(y_all) 100 | #print(y_all.get_shape().as_list()) 101 | y_sum = tf.reduce_sum(y_all, axis=0, name='y_main') 102 | # external 103 | #print(i) 104 | y_ext = tf.layers.dense(x[-1], units=10, activation=tf.nn.relu, use_bias=True, 105 | kernel_initializer=self.weight_initializer, bias_initializer=self.const_initializer, 106 | name='external_dense_1') 107 | y_ext = tf.layers.dense(y_ext, units=self.nb_flow*self.row*self.col, 108 | activation=tf.nn.relu, use_bias=True, 109 | kernel_initializer=self.weight_initializer, bias_initializer=self.const_initializer, 110 | name='external_dense_2') 111 | y_ext = tf.reshape(y_ext, [-1, self.row, self.col, self.nb_flow], name='y_ext') 112 | # y_sum + y_ext 113 | y_out = tf.nn.relu(tf.add(y_sum, y_ext), name='y_out') 114 | # compute loss 115 | loss = 2*tf.nn.l2_loss(y-y_out) 116 | return y_out, loss 117 | 118 | 119 | 120 | -------------------------------------------------------------------------------- /baseline/ARIMA.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import numpy as np 3 | import warnings 4 | import os 5 | import argparse 6 | import sys 7 | import pickle 8 | #from statsmodels.tsa.api import VAR 9 | from statsmodels.tsa.api import ARIMA 10 | sys.path.append('../') 11 | from utils import * 12 | 13 | 14 | def predict_by_samples(data, test_data, train_length, num_sample, output_steps, lag_order=1, if_sample=True): 15 | warnings.filterwarnings("ignore") 16 | test_num, data_dim = test_data.shape 17 | if not if_sample: 18 | num_sample = data_dim 19 | #widgets = ['Train: ', Percentage(), ' ', Bar('-'), ' ', ETA()] 20 | #pbar = ProgressBar(widgets=widgets, maxval=test_data.shape[0]-output_steps).start() 21 | error_all = [] 22 | index_all = np.zeros([test_num - output_steps, num_sample], dtype=np.int32) 23 | valid_num = np.zeros(test_num - output_steps, dtype=np.int32) 24 | real = np.zeros([test_num - output_steps, num_sample]) 25 | predict = np.zeros([test_num - output_steps, num_sample]) 26 | # 27 | samples = np.arange(data_dim) 28 | # 29 | for t in range(test_num - output_steps): 30 | #pbar.update(t) 31 | if t%100 == 0: 32 | print(t) 33 | error_index = [] 34 | if if_sample: 35 | samples = np.random.randint(data_dim, size=num_sample) 36 | for r in range(num_sample): 37 | # t: which time slot 38 | # i: which station 39 | i = samples[r] 40 | train_df = data[t:train_length+t, i] 41 | try: 42 | results = ARIMA(train_df, order=(lag_order, 0, 1)).fit(trend='nc', disp=0) 43 | except: 44 | error_index.append(r) 45 | continue 46 | #pre, _, _ = results.forecast(output_steps) 47 | #pre = results.predict(train_length, train_length+output_steps) 48 | pre = results.predict(train_length, train_length) 49 | test_real = test_data[t:t + output_steps, i] 50 | real[t, r] = test_real 51 | #print(pre) 52 | predict[t, r] = pre 53 | index_all[t] = samples 54 | error_all.append(error_index) 55 | valid_num[t] = num_sample - len(error_index) 56 | #pbar.finish() 57 | return real, predict, index_all, error_all, valid_num 58 | 59 | 60 | if __name__ == '__main__': 61 | parse = argparse.ArgumentParser() 62 | # parse.add_argument('-dataset', '--dataset', type=str, default='didi') 63 | parse.add_argument('-dataset', '--dataset', type=str, default='citibike') 64 | # parse.add_argument('-dataset', '--dataset', type=str, default='taxi') 65 | parse.add_argument('-predict_steps', '--predict_steps', type=int, default=1, help='prediction steps') 66 | parse.add_argument('-lag_order', '--lag_order', type=int, default=5, help='lag order in VAR and ARIMA models') 67 | parse.add_argument('-num_samples', '--num_samples', type=int, default=3, help='number of samples for ARIMA model') 68 | # 69 | args = parse.parse_args() 70 | # 71 | data_folder = '../datasets/' + args.dataset + '-data/data/' 72 | output_folder = 'results/ARIMA/' + args.dataset 73 | if not os.path.exists(output_folder): 74 | os.makedirs(output_folder) 75 | prediction_file_name = os.path.join(output_folder,'ARIMA_prediction.npy') 76 | target_file_name = os.path.join(output_folder, 'ARIMA_target.npy') 77 | print('load train, test data...') 78 | if 'citibike' in args.dataset: 79 | split = [3672, 240, 480] 80 | data, train_data, _, test_data = load_npy_data(filename=[data_folder+'d_station.npy', data_folder+'p_station.npy'], split=split) 81 | elif 'taxi' in args.dataset: 82 | split = [11640, 744, 720] 83 | data_folder = '../datasets/' + args.dataset + '-data/graph-data/' 84 | data, train_data, val_data, test_data = load_npy_data(filename=[data_folder + 'nyc_taxi_data.npy'], split=split) 85 | elif 'didi' in args.dataset: 86 | split = [2400, 192, 288] 87 | data, train_data, val_data, test_data = load_npy_data(filename=[data_folder + 'cd_didi_data.npy'], split=split) 88 | s = data.shape 89 | print(s) 90 | #if 0: 91 | if os.path.exists(prediction_file_name): 92 | print('load pre-trained prediction file') 93 | test_predict = np.load(prediction_file_name) 94 | test_real = np.load(target_file_name) 95 | error_all = load_pickle(os.path.join(output_folder, 'arima_error.pkl')) 96 | valid_num = np.array([args.num_samples - len(e) for e in error_all]) 97 | else: 98 | data = np.reshape(data, (data.shape[0], -1)) 99 | test_data = np.reshape(test_data, (test_data.shape[0], -1)) 100 | # 101 | print('train ARIMA model...') 102 | #test_data_preindex = np.vstack((train_data[-args.lag_order:], test_data)) 103 | test_real, test_predict, index_all, error_all, valid_num = predict_by_samples(data, test_data, split[0]+split[1], args.num_samples, args.predict_steps, lag_order=args.lag_order) 104 | test_predict = np.clip(test_predict, 0, None) 105 | test_real = np.squeeze(test_real) 106 | test_predict = np.squeeze(test_predict) 107 | # 108 | test_predict = np.array(test_predict, dtype=np.float32) 109 | test_real = np.array(test_real, dtype=np.float32) 110 | np.save(prediction_file_name, test_predict) 111 | np.save(target_file_name, test_real) 112 | np.save(os.path.join(output_folder, 'index_all.npy'), index_all) 113 | dump_pickle(error_all, os.path.join(output_folder, 'arima_error.pkl')) 114 | # 115 | #rmse_test = np.sqrt(np.sum(np.square(test_real-test_predict))/np.prod(test_predict.shape)) 116 | rmse_test = np.sqrt(np.sum(np.square(test_real - test_predict)) / np.sum(valid_num)) 117 | mae_test = np.mean(np.abs(test_real-test_predict)) 118 | print('test in/out rmse is %.4f' % rmse_test) 119 | print('test in/out mae is %.4f' % mae_test) 120 | # 121 | # if not os.path.exists(prediction_file_name): 122 | # test_predict = np.array(test_predict, dtype=np.float32) 123 | # test_real = np.array(test_real, dtype=np.float32) 124 | # np.save(prediction_file_name, test_predict) 125 | # np.save(target_file_name, test_real) 126 | # np.save(output_folder + 'index_all.npy', index_all) 127 | # dump_pickle(error_all, output_folder + 'arima_error.pkl') 128 | 129 | -------------------------------------------------------------------------------- /model/GCN.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import numpy as np 3 | import pickle 4 | import tensorflow as tf 5 | sys.path.append('./util/') 6 | from utils import * 7 | from model.dcrnn_cell import DCGRUCell 8 | 9 | 10 | class GCN(): 11 | def __init__(self, num_station, input_steps, 12 | num_layers=2, num_units=64, 13 | max_diffusion_step=2, 14 | dy_adj=1, 15 | dy_filter=0, 16 | f_adj_mx=None, 17 | trained_adj_mx=False, 18 | filter_type='dual_random_walk', 19 | batch_size=32): 20 | self.num_station = num_station 21 | self.input_steps = input_steps 22 | self.num_units = num_units 23 | self.max_diffusion_step = max_diffusion_step 24 | 25 | self.dy_adj = dy_adj 26 | self.dy_filter = dy_filter 27 | self.f_adj_mx = f_adj_mx 28 | self.filter_type = filter_type 29 | 30 | self.batch_size = batch_size 31 | 32 | self.weight_initializer = tf.contrib.layers.xavier_initializer() 33 | self.const_initializer = tf.constant_initializer() 34 | 35 | 36 | if trained_adj_mx: 37 | with tf.variable_scope('trained_adj_mx', reuse=tf.AUTO_REUSE): 38 | adj_mx = tf.get_variable('adj_mx', [self.num_station, self.num_station], dtype=tf.float32, 39 | initializer=self.weight_initializer) 40 | else: 41 | adj_mx = self.f_adj_mx 42 | # 43 | first_cell = DCGRUCell(self.num_units, adj_mx=adj_mx, max_diffusion_step=self.max_diffusion_step, 44 | num_nodes=self.num_station, num_proj=None, 45 | input_dim=2, 46 | dy_adj=self.dy_adj, dy_filter=self.dy_filter, 47 | output_dy_adj=self.dy_adj, 48 | filter_type=self.filter_type) 49 | cell = DCGRUCell(self.num_units, adj_mx=adj_mx, max_diffusion_step=max_diffusion_step, 50 | num_nodes=self.num_station, num_proj=None, 51 | input_dim=self.num_units, 52 | dy_adj=self.dy_adj, dy_filter=0, 53 | output_dy_adj=self.dy_adj, 54 | filter_type=self.filter_type) 55 | cell_with_projection = DCGRUCell(self.num_units, adj_mx=adj_mx, max_diffusion_step=max_diffusion_step, 56 | num_nodes=self.num_station, num_proj=2, 57 | input_dim=self.num_units, 58 | dy_adj=self.dy_adj, dy_filter=0, 59 | output_dy_adj=False, 60 | filter_type=self.filter_type) 61 | if num_layers > 2: 62 | cells = [first_cell] + [cell] * (num_layers-2) + [cell_with_projection] 63 | else: 64 | cells = [first_cell, cell_with_projection] 65 | 66 | self.cells = tf.contrib.rnn.MultiRNNCell(cells, state_is_tuple=True) 67 | # 68 | self.x = tf.placeholder(tf.float32, [self.batch_size, self.input_steps, self.num_station, 2]) 69 | self.f = tf.placeholder(tf.float32, [self.batch_size, self.input_steps, self.num_station, self.num_station]) 70 | self.y = tf.placeholder(tf.float32, [self.batch_size, self.input_steps, self.num_station, 2]) 71 | 72 | 73 | def build_model(self): 74 | x = tf.unstack(tf.reshape(self.x, (self.batch_size, self.input_steps, self.num_station*2)), axis=1) 75 | f_all = tf.unstack(tf.reshape(self.f, (self.batch_size, self.input_steps, self.num_station*self.num_station)), axis=1) 76 | #x = tf.unstack(tf.reshape(self.x, (-1, self.input_steps, self.num_station * 2)), axis=1) 77 | #f_all = tf.unstack(tf.reshape(self.f, (-1, self.input_steps, self.num_station*self.num_station)), axis=1) 78 | 79 | y = self.y 80 | hidden_state = tf.zeros([self.batch_size, self.num_station*self.num_units]) 81 | #current_state = tf.zeros([self.batch_size, self.num_station*self.num_unists]) 82 | #state = hidden_state, current_state 83 | state_1 = hidden_state 84 | state_2 = hidden_state 85 | y_ = [] 86 | for i in range(self.input_steps): 87 | # for each step 88 | #print(i) 89 | f = f_all[i] 90 | current_step_batch = x[i] 91 | with tf.variable_scope('dcrnn', reuse=tf.AUTO_REUSE): 92 | output_1, state_1 = self.cell(tf.reshape(current_step_batch, (self.batch_size, -1)), f, state_1) 93 | with tf.variable_scope('output', reuse=tf.AUTO_REUSE): 94 | output_2, state_2 = self.cell_with_projection(tf.reshape(output_1, (self.batch_size, -1)), f, state_2) 95 | # output: [batch_size, state_size] 96 | output_2 = tf.reshape(output_2, (self.batch_size, self.num_station, -1)) 97 | y_.append(output_2) 98 | y_ = tf.stack(y_) 99 | y_ = tf.transpose(y_, [1, 0, 2, 3]) 100 | loss = 2*tf.nn.l2_loss(y-y_) 101 | return y_, loss 102 | 103 | 104 | def build_easy_model(self): 105 | #x = tf.unstack(tf.reshape(self.x, (self.batch_size, self.input_steps, self.num_station*2)), axis=1) 106 | #f_all = tf.unstack(tf.reshape(self.f, (self.batch_size, self.input_steps, self.num_station*self.num_station)), axis=1) 107 | x = tf.transpose(tf.reshape(self.x, (self.batch_size, self.input_steps, -1)), [1, 0, 2]) 108 | f_all = tf.transpose(tf.reshape(self.f, (self.batch_size, self.input_steps, -1)), [1, 0, 2]) 109 | # x: [input_steps, batch_size, num_station*2] 110 | # f_all: [input_steps, batch_size, num_station*num_station] 111 | inputs = tf.concat([x, f_all], axis=-1) 112 | inputs = tf.unstack(inputs, axis=0) 113 | #inputs = list(zip(*(x, f_all))) 114 | #elems = (x, f_all) 115 | #inputs = tf.map_fn(lambda x: tf.tuple([x[0], x[1]]), elems, dtype=[tf.float32, tf.float32]) 116 | outputs, _ = tf.contrib.rnn.static_rnn(self.cells, inputs, dtype=tf.float32) 117 | outputs = tf.stack(outputs) 118 | # 119 | #outputs = tf.nn.relu(outputs) 120 | # 121 | outputs = tf.reshape(outputs, (self.input_steps, self.batch_size, self.num_station, -1)) 122 | outputs = tf.transpose(outputs, [1, 0, 2, 3]) 123 | #outputs = outputs + self.x 124 | loss = 2*tf.nn.l2_loss(self.y - outputs) 125 | return outputs, loss 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | -------------------------------------------------------------------------------- /baseline/STDN/utils.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import numpy as np 3 | import scipy.io as sio 4 | import scipy.sparse as sp 5 | from scipy.sparse import linalg 6 | from sklearn import preprocessing 7 | 8 | class StrToBytes: 9 | def __init__(self, fileobj): 10 | self.fileobj = fileobj 11 | def read(self, size): 12 | return self.fileobj.read(size).encode() 13 | def readline(self, size=-1): 14 | return self.fileobj.readline(size).encode() 15 | 16 | def dump_pickle(data, file): 17 | try: 18 | with open(file, 'wb') as datafile: 19 | pickle.dump(data, datafile) 20 | except Exception as e: 21 | raise e 22 | 23 | def load_pickle_str2bytes(file): 24 | try: 25 | with open(file, 'r') as datafile: 26 | pickle.load(StrToBytes(datafile)) 27 | except Exception as e: 28 | raise e 29 | 30 | def load_pickle_rb(file): 31 | try: 32 | with open(file, 'rb') as datafile: 33 | data = pickle.load(datafile) 34 | except Exception as e: 35 | raise e 36 | return data 37 | 38 | def load_pickle(file): 39 | try: 40 | data = load_pickle_rb(file) 41 | except: 42 | data = load_pickle_str2bytes(file) 43 | return data 44 | 45 | 46 | def get_subarea_index(n1, n2): 47 | delta = n2 - n1 48 | indices = [] 49 | for i in range(delta, delta+n1): 50 | indices.append(np.arange(i*n2 + delta - 1, i*n2 + delta -1 + n1)) 51 | return np.concatenate(indices) 52 | 53 | 54 | 55 | def load_npy_data(filename, split): 56 | if len(filename) == 2: 57 | d1 = np.load(filename[0]) 58 | d2 = np.load(filename[1]) 59 | data = np.concatenate((np.expand_dims(d1, axis=-1), np.expand_dims(d2, axis=-1)), axis=-1) 60 | elif len(filename) == 1: 61 | data = np.load(filename[0]) 62 | train = data[0:split[0]] 63 | if len(split) > 2: 64 | validate = data[split[0]:(split[0] + split[1])] 65 | test = data[(split[0]+split[1]):(split[0]+split[1]+split[2])] 66 | else: 67 | validate = None 68 | test = data[split[0]:(split[0] + split[1])] 69 | return data, train, validate, test 70 | 71 | def load_npy_data_interval_split(filename, split): 72 | if len(filename) == 2: 73 | d1 = np.load(filename[0]) 74 | d2 = np.load(filename[1]) 75 | data = np.concatenate((np.expand_dims(d1, axis=-1), np.expand_dims(d2, axis=-1)), axis=-1) 76 | elif len(filename) == 1: 77 | data = np.load(filename[0]) 78 | train = data[split[0][0]:split[0][1]] 79 | if len(split) > 2: 80 | validate = data[split[1][0]:split[1][1]] 81 | test = data[split[2][0]:split[2][1]] 82 | else: 83 | validate = None 84 | test = data[split[1][0]:split[1][1]] 85 | return data, train, validate, test 86 | 87 | def load_pkl_data(filename, split): 88 | data = load_pickle_rb(filename) 89 | train = data[0:split[0]] 90 | if len(split) > 2: 91 | validate = data[split[0]:(split[0] + split[1])] 92 | test = data[(split[0]+split[1]):(split[0]+split[1]+split[2])] 93 | else: 94 | validate = None 95 | test = data[split[0]:(split[0]+split[1])] 96 | return data, train, validate, test 97 | 98 | def load_mat_data(filename, dataname, split): 99 | data = sio.loadmat(filename)[dataname] 100 | # 101 | #max_d = np.max(data[:, -2:], axis=0) 102 | #min_d = np.min(data[:, -2:], axis=0) 103 | #data[:, -2:] = (data[:, -2:] - min_d)/(max_d - min_d) 104 | data[:, -2:] = preprocessing.scale(data[:, -2:]) 105 | train = data[0:split[0]] 106 | if len(split) > 2: 107 | validate = data[split[0]:(split[0] + split[1])] 108 | test = data[(split[0]+split[1]):(split[0]+split[1]+split[2])] 109 | else: 110 | validate = None 111 | test = data[split[0]:(split[0] + split[1])] 112 | return data, train, validate, test 113 | 114 | def batch_data(d_data, f_data, batch_size=32, input_steps=10, output_steps=10): 115 | # d_data: [num, num_station, 2] 116 | # f_data: [num, {num_station, num_station}] 117 | num = d_data.shape[0] 118 | assert len(f_data) == num 119 | # x: [batches, batch_size, input_steps, num_station, 2] 120 | # y: [batches, batch_size, output_steps, num_station, 2] 121 | # f: [batches, batch_size, input_steps+output_steps, {num_station, num_station}] 122 | x = [] 123 | y = [] 124 | f = [] 125 | i = 0 126 | while i 2: 64 | validate = data[split[0]:(split[0] + split[1])] 65 | test = data[(split[0]+split[1]):(split[0]+split[1]+split[2])] 66 | else: 67 | validate = None 68 | test = data[split[0]:(split[0] + split[1])] 69 | return data, train, validate, test 70 | 71 | def load_npy_data_interval_split(filename, split): 72 | if len(filename) == 2: 73 | d1 = np.load(filename[0]) 74 | d2 = np.load(filename[1]) 75 | data = np.concatenate((np.expand_dims(d1, axis=-1), np.expand_dims(d2, axis=-1)), axis=-1) 76 | elif len(filename) == 1: 77 | data = np.load(filename[0]) 78 | train = data[split[0][0]:split[0][1]] 79 | if len(split) > 2: 80 | validate = data[split[1][0]:split[1][1]] 81 | test = data[split[2][0]:split[2][1]] 82 | else: 83 | validate = None 84 | test = data[split[1][0]:split[1][1]] 85 | return data, train, validate, test 86 | 87 | def load_pkl_data(filename, split): 88 | data = load_pickle_rb(filename) 89 | train = data[0:split[0]] 90 | if len(split) > 2: 91 | validate = data[split[0]:(split[0] + split[1])] 92 | test = data[(split[0]+split[1]):(split[0]+split[1]+split[2])] 93 | else: 94 | validate = None 95 | test = data[split[0]:(split[0]+split[1])] 96 | return data, train, validate, test 97 | 98 | def load_mat_data(filename, dataname, split): 99 | data = sio.loadmat(filename)[dataname] 100 | # 101 | #max_d = np.max(data[:, -2:], axis=0) 102 | #min_d = np.min(data[:, -2:], axis=0) 103 | #data[:, -2:] = (data[:, -2:] - min_d)/(max_d - min_d) 104 | data[:, -2:] = preprocessing.scale(data[:, -2:]) 105 | train = data[0:split[0]] 106 | if len(split) > 2: 107 | validate = data[split[0]:(split[0] + split[1])] 108 | test = data[(split[0]+split[1]):(split[0]+split[1]+split[2])] 109 | else: 110 | validate = None 111 | test = data[split[0]:(split[0] + split[1])] 112 | return data, train, validate, test 113 | 114 | def batch_data(d_data, f_data, batch_size=32, input_steps=10, output_steps=10): 115 | # d_data: [num, num_station, 2] 116 | # f_data: [num, {num_station, num_station}] 117 | num = d_data.shape[0] 118 | assert len(f_data) == num 119 | # x: [batches, batch_size, input_steps, num_station, 2] 120 | # y: [batches, batch_size, output_steps, num_station, 2] 121 | # f: [batches, batch_size, input_steps+output_steps, {num_station, num_station}] 122 | x = [] 123 | y = [] 124 | f = [] 125 | i = 0 126 | while i 0: 98 | embedding_file = os.path.join('./data', args.dataset, 'dim1', 'embedding.txt') 99 | output_folder = os.path.join('./data', args.dataset, 'dim1', 'model_save', args.model_save) 100 | else: 101 | embedding_file = os.path.join('./data', args.dataset, 'embedding.txt') 102 | output_folder = os.path.join('./data', args.dataset, 'model_save', args.model_save) 103 | # 104 | if not os.path.exists(output_folder): 105 | os.makedirs(output_folder) 106 | # 107 | print('load train, test data...') 108 | if 'taxi' in args.dataset: 109 | #split = [11640, 744, 720] 110 | split = [11640+744, 720] 111 | data_folder = '../../datasets/' + args.dataset + '-data/graph-data/' 112 | data, train_data, val_data, test_data = load_npy_data(filename=[data_folder + 'nyc_taxi_data.npy'], split=split) 113 | data = np.reshape(data, [data.shape[0], 20, 10, -1]) 114 | train_data = np.reshape(train_data, [train_data.shape[0], 20, 10, -1]) 115 | elif 'didi' in args.dataset: 116 | #split = [2400, 192, 288] 117 | split = [2400+192, 288] 118 | data, train_data, val_data, test_data = load_npy_data(filename=[data_folder + 'cd_didi_data.npy'], split=split) 119 | data = np.reshape(data, [data.shape[0], 20, 20, -1]) 120 | train_data = np.reshape(train_data, [train_data.shape[0], 20, 20, -1]) 121 | 122 | # 123 | print(data.shape) 124 | #data = data[..., args.dim] 125 | #data = np.expand_dims(data, axis=-1) 126 | print(data.shape) 127 | minMax = MinMaxNormalization01() 128 | minMax.fit(data[:split[0]]) 129 | data = minMax.transform(data) 130 | print(np.max(data[split[0]:])) 131 | #minMax = MinMax(data) 132 | #data = minMax.transform() 133 | embedding = load_embedding(embedding_file) 134 | # prepare data 135 | train_image, train_embedding, train_y, test_image, test_embedding, test_y, test_num = prepare_data_padding(args.input_steps, 136 | data, embedding, 137 | split, 9, if_padding=True) 138 | print(test_num) 139 | print(np.max(test_y)) 140 | # set gpu config 141 | config = tf.ConfigProto() 142 | config.gpu_options.allow_growth = True 143 | ktf.set_session(tf.Session(config=config)) 144 | # train model 145 | prediction = build_model(train_y, test_y, train_image, test_image, 64, minMax, 146 | seq_len=args.input_steps, 147 | batch_size=args.batch_size, 148 | trainable=args.trainable, 149 | model_path=output_folder) 150 | test_target = np.reshape(test_y, test_num+[-1]) 151 | test_prediction = np.reshape(prediction, test_num+[-1]) 152 | np.save(os.path.join(output_folder, 'test_target.npy'), test_target) 153 | np.save(os.path.join(output_folder, 'test_prediction.npy'), test_prediction) 154 | 155 | 156 | if __name__ == '__main__': 157 | main() 158 | -------------------------------------------------------------------------------- /baseline/DMVSTNet/main.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import sys 3 | import os 4 | import argparse 5 | import tensorflow as tf 6 | import keras.backend.tensorflow_backend as ktf 7 | from keras.callbacks import EarlyStopping, ModelCheckpoint 8 | currentPath = os.path.abspath(os.path.curdir) 9 | sys.path.append(currentPath) 10 | from model import build_model 11 | from utils import * 12 | from minMax import * 13 | 14 | 15 | def load_embedding(filename): 16 | with open(filename, 'r') as f: 17 | num, length = f.readline().strip().split(" ") 18 | embedding = {} 19 | for i in range(int(num)): 20 | line = f.readline() 21 | vector = line.strip().split(" ") 22 | id = int(vector[0]) 23 | vector = [float(item) for item in vector[1:]] 24 | embedding[id] = np.array(vector, dtype=np.float) 25 | return embedding 26 | 27 | 28 | def prepare_data_padding(input_length, map_data, embedding_data, split, image_size=9, if_padding=False): 29 | padding = int(image_size / 2) 30 | # 31 | if if_padding: 32 | raw_map_shape = map_data.shape 33 | padding_map_data = np.zeros((raw_map_shape[0], raw_map_shape[1]+2*padding, raw_map_shape[2]+2*padding, raw_map_shape[3]), dtype=np.float32) 34 | padding_map_data[:, padding:-padding, padding:-padding, :] = map_data 35 | map_data = padding_map_data 36 | # 37 | height = map_data.shape[1] 38 | width = map_data.shape[2] 39 | #T = map_data.shape[0] 40 | train_image_x = [] 41 | train_y = [] 42 | train_embedding = [] 43 | test_image_x = [] 44 | test_y = [] 45 | test_embedding_x = [] 46 | #for t in range(input_length, splits[0] - input_length): 47 | for t in range(input_length, split[0]): 48 | for i in range(padding, height - padding): 49 | for j in range(padding, width - padding): 50 | if not embedding_data.__contains__(i * width + j): 51 | continue 52 | train_image_x.append(map_data[t-input_length:t, i - padding:i + padding+1, j - padding:j + padding+1, :]) 53 | train_y.append(map_data[t, i, j, :]) 54 | train_embedding.append(embedding_data[i * width + j]) 55 | #for t in range(splits[0] - input_length, splits[0] + splits[1] - input_length): 56 | for t in range(np.sum(split[:-1]) + input_length, np.sum(split)): 57 | for i in range(padding, height - padding): 58 | for j in range(padding, width - padding): 59 | if not embedding_data.__contains__(i * width + j): 60 | continue 61 | test_image_x.append(map_data[t-input_length:t, i - padding:i + padding+1, j - padding:j + padding+1, :]) 62 | test_y.append(map_data[t, i, j, :]) 63 | test_embedding_x.append(embedding_data[i * width + j]) 64 | train_image_x = np.asarray(train_image_x) 65 | train_embedding = np.asarray(train_embedding) 66 | train_y = np.asarray(train_y) 67 | test_embedding_x = np.asarray(test_embedding_x) 68 | test_image_x = np.asarray(test_image_x) 69 | test_y = np.asarray(test_y) 70 | print(train_image_x.shape) 71 | print(train_embedding.shape) 72 | print(train_y.shape) 73 | print(test_image_x.shape) 74 | print(test_embedding_x.shape) 75 | print(test_y.shape) 76 | test_y_num = [np.sum(split) - np.sum(split[:-1]) - input_length] 77 | return train_image_x, train_embedding, train_y, test_image_x, test_embedding_x, test_y, test_y_num 78 | 79 | 80 | def main(): 81 | parse = argparse.ArgumentParser() 82 | parse.add_argument('-gpu', '--gpu', type=str, default='0', help='which gpu to use: 0 or 1') 83 | parse.add_argument('-dataset', '--dataset', type=str, default='didi', help='datasets: didi, taxi') 84 | # parse.add_argument('-dataset', '--dataset', type=str, default='taxi') 85 | # 86 | parse.add_argument('-model_save', '--model_save', type=str, default='', help='path to save model') 87 | parse.add_argument('-predict_steps', '--predict_steps', type=int, default=1, help='prediction steps') 88 | parse.add_argument('-input_steps', '--input_steps', type=int, default=6, help='number of input steps') 89 | parse.add_argument('-dim', '--dim', type=int, default=0, help='dim of data to be processed') 90 | parse.add_argument('-trainable', '--trainable', type=int, default=1, help='if to train (1) or to test (0)') 91 | parse.add_argument('-batch_size', '--batch_size', type=int, default=64) 92 | # 93 | args = parse.parse_args() 94 | os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu 95 | # 96 | data_folder = '../../datasets/' + args.dataset + '-data/data/' 97 | if args.dim > 0: 98 | embedding_file = os.path.join('./data', args.dataset, 'dim1', 'embedding.txt') 99 | output_folder = os.path.join('./data', args.dataset, 'dim1', 'model_save', args.model_save) 100 | else: 101 | embedding_file = os.path.join('./data', args.dataset, 'embedding.txt') 102 | output_folder = os.path.join('./data', args.dataset, 'model_save', args.model_save) 103 | # 104 | if not os.path.exists(output_folder): 105 | os.makedirs(output_folder) 106 | # 107 | print('load train, test data...') 108 | if 'taxi' in args.dataset: 109 | #split = [11640, 744, 720] 110 | split = [11640+744, 720] 111 | data_folder = '../../datasets/' + args.dataset + '-data/graph-data/' 112 | data, train_data, val_data, test_data = load_npy_data(filename=[data_folder + 'nyc_taxi_data.npy'], split=split) 113 | data = np.reshape(data, [data.shape[0], 20, 10, -1]) 114 | train_data = np.reshape(train_data, [train_data.shape[0], 20, 10, -1]) 115 | elif 'didi' in args.dataset: 116 | #split = [2400, 192, 288] 117 | split = [2400+192, 288] 118 | data, train_data, val_data, test_data = load_npy_data(filename=[data_folder + 'cd_didi_data.npy'], split=split) 119 | data = np.reshape(data, [data.shape[0], 20, 20, -1]) 120 | train_data = np.reshape(train_data, [train_data.shape[0], 20, 20, -1]) 121 | 122 | # 123 | print(data.shape) 124 | #data = data[..., args.dim] 125 | #data = np.expand_dims(data, axis=-1) 126 | print(data.shape) 127 | minMax = MinMaxNormalization01() 128 | minMax.fit(data[:split[0]]) 129 | data = minMax.transform(data) 130 | print(np.max(data[split[0]:])) 131 | #minMax = MinMax(data) 132 | #data = minMax.transform() 133 | embedding = load_embedding(embedding_file) 134 | # prepare data 135 | train_image, train_embedding, train_y, test_image, test_embedding, test_y, test_num = prepare_data_padding(args.input_steps, 136 | data, embedding, 137 | split, 9, if_padding=True) 138 | print(test_num) 139 | print(np.max(test_y)) 140 | # set gpu config 141 | config = tf.ConfigProto() 142 | config.gpu_options.allow_growth = True 143 | ktf.set_session(tf.Session(config=config)) 144 | # train model 145 | prediction = build_model(train_y, test_y, train_image, test_image, train_embedding, test_embedding, 64, minMax, 146 | seq_len=args.input_steps, 147 | batch_size=args.batch_size, 148 | trainable=args.trainable, 149 | model_path=output_folder) 150 | test_target = np.reshape(test_y, test_num+[-1]) 151 | test_prediction = np.reshape(prediction, test_num+[-1]) 152 | np.save(os.path.join(output_folder, 'test_target.npy'), test_target) 153 | np.save(os.path.join(output_folder, 'test_prediction.npy'), test_prediction) 154 | 155 | 156 | if __name__ == '__main__': 157 | main() 158 | -------------------------------------------------------------------------------- /baseline/STDN/main.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import sys 3 | import os 4 | import argparse 5 | import tensorflow as tf 6 | import keras.backend.tensorflow_backend as ktf 7 | from keras.callbacks import EarlyStopping, ModelCheckpoint 8 | currentPath = os.path.abspath(os.path.curdir) 9 | sys.path.append(currentPath) 10 | from model_2 import build_model 11 | from utils import * 12 | from minMax import * 13 | 14 | 15 | 16 | def prepare_data_padding(input_length, map_data, flow_data, split, nb_size=7): 17 | padding = int(nb_size / 2) 18 | # 19 | raw_map_shape = map_data.shape 20 | padding_map_data = np.zeros((raw_map_shape[0], raw_map_shape[1]+2*padding, raw_map_shape[2]+2*padding, raw_map_shape[3]), dtype=np.float32) 21 | padding_map_data[:, padding:-padding, padding:-padding, :] = map_data 22 | # 23 | height = padding_map_data.shape[1] 24 | width = padding_map_data.shape[2] 25 | # 26 | index = np.ones((height, width), dtype=np.int32) * (-1) 27 | index[padding:-padding, padding:-padding] = np.reshape(np.arange(raw_map_shape[1]*raw_map_shape[2]), (raw_map_shape[1], raw_map_shape[2])) 28 | # 29 | train_image_x = [] 30 | train_y = [] 31 | train_flow = [] 32 | test_image_x = [] 33 | test_y = [] 34 | test_flow = [] 35 | for t in range(input_length, split[0]): 36 | for i in range(padding, height - padding): 37 | for j in range(padding, width - padding): 38 | train_image_x.append(padding_map_data[t-input_length:t, i - padding:i + padding+1, j - padding:j + padding+1, :]) 39 | train_y.append(padding_map_data[t, i, j, :]) 40 | # 41 | ij_index = (i-padding) * raw_map_shape[2] + (j-padding) 42 | nb_index = index[i-padding: i+padding+1, j-padding:j+padding+1].flatten() 43 | ij_flow = flow_data[t-input_length:t, ij_index, nb_index] 44 | ij_flow[:, np.nonzero(nb_index<0)[0]] = 0 45 | ij_flow = np.expand_dims(np.reshape(ij_flow, (-1, nb_size, nb_size)), -1) 46 | train_flow.append(ij_flow) 47 | for t in range(np.sum(split[:-1]) + input_length, np.sum(split)): 48 | for i in range(padding, height - padding): 49 | for j in range(padding, width - padding): 50 | test_image_x.append(padding_map_data[t-input_length:t, i - padding:i + padding+1, j - padding:j + padding+1, :]) 51 | test_y.append(padding_map_data[t, i, j, :]) 52 | # 53 | ij_index = (i-padding) * raw_map_shape[2] + (j-padding) 54 | nb_index = index[i - padding: i + padding + 1, j - padding:j + padding + 1].flatten() 55 | ij_flow = flow_data[t-input_length:t, ij_index, nb_index] 56 | ij_flow[:, np.nonzero(nb_index<0)[0]] = 0 57 | ij_flow = np.expand_dims(np.reshape(ij_flow, (-1, nb_size, nb_size)), -1) 58 | test_flow.append(ij_flow) 59 | train_image_x = np.asarray(train_image_x) 60 | train_flow = np.asarray(train_flow) 61 | train_y = np.asarray(train_y) 62 | test_flow = np.asarray(test_flow) 63 | test_image_x = np.asarray(test_image_x) 64 | test_y = np.asarray(test_y) 65 | print(train_image_x.shape) 66 | print(train_flow.shape) 67 | print(train_y.shape) 68 | print(test_image_x.shape) 69 | print(test_flow.shape) 70 | print(test_y.shape) 71 | test_y_num = [np.sum(split) - np.sum(split[:-1]) - input_length] 72 | return train_image_x, train_flow, train_y, test_image_x, test_flow, test_y, test_y_num 73 | 74 | 75 | def main(): 76 | parse = argparse.ArgumentParser() 77 | parse.add_argument('-gpu', '--gpu', type=str, default='0', help='which gpu to use: 0 or 1') 78 | parse.add_argument('-dataset', '--dataset', type=str, default='didi', help='datasets: didi, taxi') 79 | # parse.add_argument('-dataset', '--dataset', type=str, default='taxi') 80 | # 81 | parse.add_argument('-model_save', '--model_save', type=str, default='', help='path to save model') 82 | parse.add_argument('-predict_steps', '--predict_steps', type=int, default=1, help='prediction steps') 83 | parse.add_argument('-input_steps', '--input_steps', type=int, default=6, help='number of input steps') 84 | parse.add_argument('-dim', '--dim', type=int, default=0, help='dim of data to be processed') 85 | parse.add_argument('-trainable', '--trainable', type=int, default=1, help='if to train (1) or to test (0)') 86 | parse.add_argument('-batch_size', '--batch_size', type=int, default=64) 87 | # 88 | args = parse.parse_args() 89 | os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu 90 | # 91 | data_folder = '../../datasets/' + args.dataset + '-data/data/' 92 | output_folder = os.path.join('./data', args.dataset, 'model_save', args.model_save) 93 | # 94 | if not os.path.exists(output_folder): 95 | os.makedirs(output_folder) 96 | # 97 | print('load train, test data...') 98 | if 'taxi' in args.dataset: 99 | #split = [11640, 744, 720] 100 | split = [11640+744, 720] 101 | data_folder = '../../datasets/' + args.dataset + '-data/graph-data/' 102 | data, train_data, val_data, test_data = load_npy_data(filename=[data_folder + 'nyc_taxi_data.npy'], split=split) 103 | data = np.reshape(data, [data.shape[0], 20, 10, -1]) 104 | train_data = np.reshape(train_data, [train_data.shape[0], 20, 10, -1]) 105 | # 106 | f_data, train_f_data, val_f_data, test_f_data = load_npy_data([data_folder + 'nyc_taxi_flow_in.npy'], 107 | split=split) 108 | elif 'didi' in args.dataset: 109 | #split = [2400, 192, 288] 110 | split = [2400+192, 288] 111 | data, train_data, val_data, test_data = load_npy_data(filename=[data_folder + 'cd_didi_data.npy'], split=split) 112 | data = np.reshape(data, [data.shape[0], 20, 20, -1]) 113 | train_data = np.reshape(train_data, [train_data.shape[0], 20, 20, -1]) 114 | # 115 | f_data, train_f_data, val_f_data, test_f_data = load_npy_data([data_folder + 'cd_didi_flow_in.npy'], 116 | split=split) 117 | 118 | # 119 | print('preprocess data...') 120 | print('data shape: {}'.format(data.shape)) 121 | minMax = MinMaxNormalization01() 122 | minMax.fit(train_data) 123 | data = minMax.transform(data) 124 | # 125 | print('preprocess f_data...') 126 | print('f_data shape: {}'.format(f_data.shape)) 127 | f_minMax = MinMaxNormalization01() 128 | f_minMax.fit(train_f_data) 129 | f_data = f_minMax.transform(f_data) 130 | # prepare data 131 | train_x, train_flow, train_y, test_x, test_flow, test_y, test_num = prepare_data_padding(args.input_steps, 132 | data, f_data, 133 | split, 7) 134 | print(test_num) 135 | # set gpu config 136 | config = tf.ConfigProto() 137 | config.gpu_options.allow_growth = True 138 | ktf.set_session(tf.Session(config=config)) 139 | # train model 140 | prediction = build_model(train_y, test_y, train_x, test_x, train_flow, test_flow, minMax, 141 | seq_len=args.input_steps, 142 | batch_size=args.batch_size, 143 | trainable=args.trainable, 144 | model_path=output_folder) 145 | test_target = np.reshape(test_y, test_num+[-1]) 146 | test_prediction = np.reshape(prediction, test_num+[-1]) 147 | np.save(os.path.join(output_folder, 'test_target.npy'), test_target) 148 | np.save(os.path.join(output_folder, 'test_prediction.npy'), test_prediction) 149 | 150 | 151 | if __name__ == '__main__': 152 | main() 153 | -------------------------------------------------------------------------------- /baseline/STDN/model.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from utils import * 3 | import os 4 | from keras import backend as K, losses 5 | import numpy as np 6 | # import pandas as pd 7 | # import argparse 8 | import keras 9 | from keras.models import Sequential, Model 10 | from keras import activations 11 | from keras.engine.topology import Layer, InputSpec 12 | from keras.utils import conv_utils 13 | from keras.layers import Dense, Activation, concatenate, Input, Conv2D, Reshape, Flatten, Dropout, BatchNormalization, Concatenate, LSTM 14 | from keras.callbacks import EarlyStopping, ModelCheckpoint 15 | from sklearn.preprocessing import MinMaxScaler 16 | from sklearn.metrics import mean_squared_error 17 | from keras.optimizers import Adam 18 | from keras import metrics 19 | from keras.layers.normalization import BatchNormalization 20 | from random import randint 21 | import pickle 22 | 23 | 24 | #batch_size = 64 25 | mean_label = 0.0 26 | label_max = 0 27 | label_min = 0 28 | max_epoch = 100 29 | num_feature = 100 30 | # seq_len = 8 31 | cnn_flat_size = 128 32 | hidden_dim = 64 33 | threshold = 10.0 34 | maxtruey = 0 35 | mintruey = 0 36 | #eps = 1e-5 37 | eps = 1e-6 38 | loss_lambda = 10.0 39 | feature_len = 0 40 | nbhd_size = 7 41 | cnn_hidden_dim_first = 32 42 | len_valid_id = 0 43 | toponet_len = 32 44 | 45 | config = tf.ConfigProto() 46 | config.gpu_options.allow_growth = True 47 | sess = tf.Session(config=config) 48 | K.set_session(sess) 49 | 50 | 51 | def build_model(trainY, testY, trainX, testX, train_flow, test_flow, 52 | minMax, 53 | seq_len=6, 54 | batch_size=64, 55 | trainable=True, 56 | name="MODEL", 57 | model_path='./'): 58 | print(testX.shape) 59 | # 60 | #nbhd_inputs = [Input(shape=(nbhd_size, nbhd_size, 2,), name="nbhd_volume_input_time_{0}".format(ts + 1)) for ts in range(seq_len)] 61 | #flow_inputs = [Input(shape=(nbhd_size, nbhd_size, 1,), name="flow_volume_input_time_{0}".format(ts + 1)) for ts in range(seq_len)] 62 | nbhd_inputs = Input(shape=(seq_len, nbhd_size, nbhd_size, 2,), name="nbhd_volume_input") 63 | flow_inputs = Input(shape=(seq_len, nbhd_size, nbhd_size, 1,), name="flow_volume_input") 64 | 65 | # 1st level gate 66 | # nbhd cnn 67 | nbhd_convs = [Conv2D(filters=64, kernel_size=(3, 3), padding="same", name="nbhd_convs_time0_{0}".format(ts + 1))( 68 | nbhd_inputs[ts]) for ts in range(seq_len)] 69 | nbhd_convs = [Activation("relu", name="nbhd_convs_activation_time0_{0}".format(ts + 1))(nbhd_convs[ts]) for ts in 70 | range(seq_len)] 71 | # flow cnn 72 | flow_convs = [Conv2D(filters=64, kernel_size=(3, 3), padding="same", name="flow_convs_time0_{0}".format(ts + 1))( 73 | flow_inputs[ts]) for ts in range(seq_len)] 74 | flow_convs = [Activation("relu", name="flow_convs_activation_time0_{0}".format(ts + 1))(flow_convs[ts]) for ts in 75 | range(seq_len)] 76 | # flow gate 77 | flow_gates = [Activation("sigmoid", name="flow_gate0_{0}".format(ts + 1))(flow_convs[ts]) for ts in 78 | range(seq_len)] 79 | nbhd_convs = [keras.layers.Multiply()([nbhd_convs[ts], flow_gates[ts]]) for ts in range(seq_len)] 80 | 81 | # 2nd level gate 82 | nbhd_convs = [Conv2D(filters=64, kernel_size=(3, 3), padding="same", name="nbhd_convs_time1_{0}".format(ts + 1))( 83 | nbhd_convs[ts]) for ts in range(seq_len)] 84 | nbhd_convs = [Activation("relu", name="nbhd_convs_activation_time1_{0}".format(ts + 1))(nbhd_convs[ts]) for ts in 85 | range(seq_len)] 86 | flow_convs = [Conv2D(filters=64, kernel_size=(3, 3), padding="same", name="flow_convs_time1_{0}".format(ts + 1))( 87 | flow_inputs[ts]) for ts in range(seq_len)] 88 | flow_convs = [Activation("relu", name="flow_convs_activation_time1_{0}".format(ts + 1))(flow_convs[ts]) for ts in 89 | range(seq_len)] 90 | flow_gates = [Activation("sigmoid", name="flow_gate1_{0}".format(ts + 1))(flow_convs[ts]) for ts in 91 | range(seq_len)] 92 | nbhd_convs = [keras.layers.Multiply()([nbhd_convs[ts], flow_gates[ts]]) for ts in range(seq_len)] 93 | 94 | # 3rd level gate 95 | nbhd_convs = [Conv2D(filters=64, kernel_size=(3, 3), padding="same", name="nbhd_convs_time2_{0}".format(ts + 1))( 96 | nbhd_convs[ts]) for ts in range(seq_len)] 97 | nbhd_convs = [Activation("relu", name="nbhd_convs_activation_time2_{0}".format(ts + 1))(nbhd_convs[ts]) for ts in 98 | range(seq_len)] 99 | flow_convs = [Conv2D(filters=64, kernel_size=(3, 3), padding="same", name="flow_convs_time2_{0}".format(ts + 1))( 100 | flow_inputs[ts]) for ts in range(seq_len)] 101 | flow_convs = [Activation("relu", name="flow_convs_activation_time2_{0}".format(ts + 1))(flow_convs[ts]) for ts in 102 | range(seq_len)] 103 | flow_gates = [Activation("sigmoid", name="flow_gate2_{0}".format(ts + 1))(flow_convs[ts]) for ts in 104 | range(seq_len)] 105 | nbhd_convs = [keras.layers.Multiply()([nbhd_convs[ts], flow_gates[ts]]) for ts in range(seq_len)] 106 | 107 | # dense part 108 | nbhd_vecs = [Flatten(name="nbhd_flatten_time_{0}".format(ts + 1))(nbhd_convs[ts]) for ts in range(seq_len)] 109 | nbhd_vecs = [Dense(units=cnn_flat_size, name="nbhd_dense_time_{0}".format(ts + 1))(nbhd_vecs[ts]) for ts in 110 | range(seq_len)] 111 | nbhd_vecs = [Activation("relu", name="nbhd_dense_activation_time_{0}".format(ts + 1))(nbhd_vecs[ts]) for ts in 112 | range(seq_len)] 113 | 114 | # concatenate 115 | nbhd_vec = Concatenate(axis=-1)(nbhd_vecs) 116 | nbhd_vec = Reshape(target_shape=(seq_len, cnn_flat_size))(nbhd_vec) 117 | # lstm 118 | lstm = LSTM(units=64, return_sequences=False, dropout=0.1, recurrent_dropout=0.1)(nbhd_vec) 119 | lstm_all = Dense(units=2)(lstm) 120 | pred_volume = Activation('tanh')(lstm_all) 121 | 122 | model = Model(inputs=[nbhd_inputs, flow_inputs], 123 | outputs=pred_volume) 124 | sgd = Adam(lr=0.0002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=1e-6) 125 | model.compile(loss=losses.mse, optimizer=sgd, metrics=[metrics.mse]) 126 | earlyStopping = EarlyStopping( 127 | monitor='val_loss', patience=5, verbose=0, mode='min') 128 | # model.fit([trainimage, trainX, traintopo], trainY, batch_size=batch_size, epochs=max_epoch, validation_split=0.1, 129 | # callbacks=[earlyStopping]) 130 | 131 | fname_param = os.path.join(model_path, 'log', name) 132 | if not os.path.exists(fname_param): 133 | os.makedirs(fname_param) 134 | fname_param = os.path.join(fname_param, 'STDN.best.h5') 135 | # early_stopping = EarlyStopping(monitor='val_rmse', patience=5, mode='min') 136 | model_checkpoint = ModelCheckpoint( 137 | fname_param, monitor='val_loss', verbose=0, save_best_only=True, mode='min') 138 | if trainable: 139 | model.fit([trainX, train_flow], trainY, batch_size=batch_size, epochs=max_epoch, validation_split=0.1, 140 | callbacks=[earlyStopping, model_checkpoint]) 141 | else: 142 | model.load_weights(fname_param) 143 | # testLoss = model.evaluate([testimage, testtopo], testY) 144 | score = model.evaluate([trainX, train_flow], trainY, batch_size=batch_size, verbose=0) 145 | # print('Train score: %.6f rmse (norm): %.6f rmse (real): %.6f' % 146 | # (score[0], score[1], minMax.inverse(np.sqrt(score[1])))) 147 | print('Train score: %.6f se (norm): %.6f se (real): %.6f' % 148 | (score[0], score[1], minMax.inverse(minMax.inverse(score[1])))) 149 | 150 | score = model.evaluate( 151 | [testX, test_flow], testY, batch_size=batch_size, verbose=0) 152 | # print('Test score: %.6f rmse (norm): %.6f rmse (real): %.6f' % 153 | # (score[0], score[1], minMax.inverse(np.sqrt(score[1])))) 154 | print('Test score: %.6f se (norm): %.6f se (real): %.6f' % 155 | (score[0], score[1], minMax.inverse(minMax.inverse(score[1])))) 156 | # model.save('local_conv_lstm_total_embed.h5') 157 | 158 | prediction = model.predict([testX, test_flow], batch_size=batch_size, verbose=0) 159 | print(prediction.shape) 160 | test_mse = minMax.inverse(minMax.inverse(np.mean(np.square(prediction - testY)))) 161 | print('test mse is %.6f, and rmse : %.6f' % (test_mse, np.sqrt(test_mse))) 162 | return prediction 163 | 164 | # return model 165 | -------------------------------------------------------------------------------- /baseline/STGCN/data_loader/data_utils.py: -------------------------------------------------------------------------------- 1 | # @Time : Jan. 10, 2019 15:26 2 | # @Author : Veritas YIN 3 | # @FileName : data_utils.py 4 | # @Version : 1.0 5 | # @IDE : PyCharm 6 | # @Github : https://github.com/VeritasYin/Project_Orion 7 | 8 | from utils.math_utils import z_score 9 | 10 | import numpy as np 11 | import pandas as pd 12 | import tensorflow as tf 13 | 14 | 15 | class Dataset(object): 16 | def __init__(self, data, stats): 17 | self.__data = data 18 | self.mean = stats['mean'] 19 | self.std = stats['std'] 20 | 21 | def get_data(self, type): 22 | return self.__data[type] 23 | 24 | def get_stats(self): 25 | return {'mean': self.mean, 'std': self.std} 26 | 27 | def get_len(self, type): 28 | return len(self.__data[type]) 29 | 30 | def z_inverse(self, type): 31 | return self.__data[type] * self.std + self.mean 32 | 33 | 34 | def seq_gen(len_seq, data_seq, offset, n_frame, n_route, day_slot, C_0=1): 35 | ''' 36 | Generate data in the form of standard sequence unit. 37 | :param len_seq: int, the length of target date sequence. 38 | :param data_seq: np.ndarray, source data / time-series. 39 | :param offset: int, the starting index of different dataset type. 40 | :param n_frame: int, the number of frame within a standard sequence unit, 41 | which contains n_his = 12 and n_pred = 9 (3 /15 min, 6 /30 min & 9 /45 min). 42 | :param n_route: int, the number of routes in the graph. 43 | :param day_slot: int, the number of time slots per day, controlled by the time window (5 min as default). 44 | :param C_0: int, the size of input channel. 45 | :return: np.ndarray, [len_seq, n_frame, n_route, C_0]. 46 | ''' 47 | n_slot = day_slot - n_frame + 1 48 | 49 | tmp_seq = np.zeros((len_seq * n_slot, n_frame, n_route, C_0)) 50 | for i in range(len_seq): 51 | for j in range(n_slot): 52 | sta = (i + offset) * day_slot + j 53 | end = sta + n_frame 54 | tmp_seq[i * n_slot + j, :, :, :] = np.reshape(data_seq[sta:end, :], [n_frame, n_route, C_0]) 55 | return tmp_seq 56 | 57 | 58 | def data_gen(file_path, data_config, n_route, n_frame=21, day_slot=288): 59 | ''' 60 | Source file load and dataset generation. 61 | :param file_path: str, the file path of data source. 62 | :param data_config: tuple, the configs of dataset in train, validation, test. 63 | :param n_route: int, the number of routes in the graph. 64 | :param n_frame: int, the number of frame within a standard sequence unit, 65 | which contains n_his = 12 and n_pred = 9 (3 /15 min, 6 /30 min & 9 /45 min). 66 | :param day_slot: int, the number of time slots per day, controlled by the time window (5 min as default). 67 | :return: dict, dataset that contains training, validation and test with stats. 68 | ''' 69 | n_train, n_val, n_test = data_config 70 | # generate training, validation and test data 71 | try: 72 | data_seq = pd.read_csv(file_path, header=None).values 73 | except FileNotFoundError: 74 | print(f'ERROR: input file was not found in {file_path}.') 75 | 76 | seq_train = seq_gen(n_train, data_seq, 0, n_frame, n_route, day_slot) 77 | seq_val = seq_gen(n_val, data_seq, n_train, n_frame, n_route, day_slot) 78 | seq_test = seq_gen(n_test, data_seq, n_train + n_val, n_frame, n_route, day_slot) 79 | 80 | # x_stats: dict, the stats for the train dataset, including the value of mean and standard deviation. 81 | x_stats = {'mean': np.mean(seq_train), 'std': np.std(seq_train)} 82 | 83 | # x_train, x_val, x_test: np.array, [sample_size, n_frame, n_route, channel_size]. 84 | x_train = z_score(seq_train, x_stats['mean'], x_stats['std']) 85 | x_val = z_score(seq_val, x_stats['mean'], x_stats['std']) 86 | x_test = z_score(seq_test, x_stats['mean'], x_stats['std']) 87 | 88 | x_data = {'train': x_train, 'val': x_val, 'test': x_test} 89 | dataset = Dataset(x_data, x_stats) 90 | return dataset 91 | 92 | 93 | 94 | 95 | def cheb_poly_approx_tf(L, Ks, n): 96 | ''' 97 | Chebyshev polynomials approximation function. 98 | :param L: tensor, [n_route, n_route], graph Laplacian. 99 | :param Ks: int, kernel size of spatial convolution. 100 | :param n: int, number of routes / size of graph. 101 | :return: tensor, [n_route, Ks*n_route]. 102 | ''' 103 | L0 = tf.Variable(initial_value=tf.eye(n)) 104 | L1 = tf.Variable(initial_value=L) 105 | if Ks > 1: 106 | L_list = [L0, L1] 107 | for i in range(Ks-2): 108 | Ln = 2 * tf.matmul(L, L1) - L0 109 | L_list.append(tf.identity(Ln)) 110 | # 111 | tf.assign(L0, L1) 112 | tf.assign(L1, L) 113 | return tf.concat(L_list, axis=-1) 114 | elif Ks == 1: 115 | return L0 116 | else: 117 | raise ValueError('ERROR: the size of spatial kernel must be greater than 1, but received "{0}".'.format(Ks)) 118 | # L0, L1 = np.mat(np.identity(n)), np.mat(np.copy(L)) 119 | # if Ks > 1: 120 | # L_list = [np.copy(L0), np.copy(L1)] 121 | # for i in range(Ks - 2): 122 | # Ln = np.mat(2 * L * L1 - L0) 123 | # L_list.append(np.copy(Ln)) 124 | # L0, L1 = np.matrix(np.copy(L1)), np.matrix(np.copy(Ln)) 125 | # # L_lsit [Ks, n*n], Lk [n, Ks*n] 126 | # return np.concatenate(L_list, axis=-1) 127 | # elif Ks == 1: 128 | # return np.asarray(L0) 129 | # else: 130 | # raise ValueError('ERROR: the size of spatial kernel must be greater than 1, but received "{0}".'.format(Ks)) 131 | 132 | def get_rescaled_W(w, delta=1e7, epsilon=0.8): 133 | w2 = np.exp(-w / delta, dtype=np.float32) 134 | zero_index = np.eye(len(w2)) + np.array(w2 < epsilon, np.int32) 135 | W = w2 + zero_index * (-w2) 136 | return W 137 | 138 | 139 | def seq_gen_2(data_seq, n_frame, n_route, channels): 140 | num = len(data_seq) - n_frame + 1 141 | tmp_seq = np.zeros((num, n_frame, n_route, channels)) 142 | for i in range(num): 143 | tmp_seq[i] = data_seq[i: i+n_frame] 144 | return tmp_seq 145 | 146 | def data_gen_2(filename, split, n_frame=21): 147 | # generate training, validation and test data 148 | # 149 | if len(filename) == 2: 150 | d1 = np.load(filename[0]) 151 | d2 = np.load(filename[1]) 152 | data = np.concatenate((np.expand_dims(d1, axis=-1), np.expand_dims(d2, axis=-1)), axis=-1) 153 | elif len(filename) == 1: 154 | data = np.load(filename[0]) 155 | train = data[0:split[0]] 156 | validate = data[split[0]:(split[0] + split[1])] 157 | test = data[(split[0]+split[1]):(split[0]+split[1]+split[2])] 158 | 159 | n_route = train.shape[1] 160 | 161 | seq_train = seq_gen_2(train, n_frame, n_route, train.shape[-1]) 162 | seq_val = seq_gen_2(validate, n_frame, n_route, validate.shape[-1]) 163 | seq_test = seq_gen_2(test, n_frame, n_route, test.shape[-1]) 164 | 165 | # x_stats: dict, the stats for the train dataset, including the value of mean and standard deviation. 166 | x_stats = {'mean': np.mean(seq_train), 'std': np.std(seq_train)} 167 | 168 | # x_train, x_val, x_test: np.array, [sample_size, n_frame, n_route, channel_size]. 169 | x_train = z_score(seq_train, x_stats['mean'], x_stats['std']) 170 | x_val = z_score(seq_val, x_stats['mean'], x_stats['std']) 171 | x_test = z_score(seq_test, x_stats['mean'], x_stats['std']) 172 | 173 | x_data = {'train': x_train, 'val': x_val, 'test': x_test} 174 | dataset = Dataset(x_data, x_stats) 175 | return dataset, n_route 176 | 177 | def gen_batch(inputs, batch_size, dynamic_batch=False, shuffle=False): 178 | ''' 179 | Data iterator in batch. 180 | :param inputs: np.ndarray, [len_seq, n_frame, n_route, C_0], standard sequence units. 181 | :param batch_size: int, the size of batch. 182 | :param dynamic_batch: bool, whether changes the batch size in the last batch if its length is less than the default. 183 | :param shuffle: bool, whether shuffle the batches. 184 | ''' 185 | len_inputs = len(inputs) 186 | 187 | if shuffle: 188 | idx = np.arange(len_inputs) 189 | np.random.shuffle(idx) 190 | 191 | for start_idx in range(0, len_inputs, batch_size): 192 | end_idx = start_idx + batch_size 193 | if end_idx > len_inputs: 194 | if dynamic_batch: 195 | end_idx = len_inputs 196 | else: 197 | break 198 | if shuffle: 199 | slide = idx[start_idx:end_idx] 200 | else: 201 | slide = slice(start_idx, end_idx) 202 | 203 | yield inputs[slide] 204 | -------------------------------------------------------------------------------- /baseline/STResNet/solver.py: -------------------------------------------------------------------------------- 1 | #from __future__ import division 2 | import numpy as np 3 | import time 4 | import os 5 | from sklearn.model_selection import train_test_split 6 | from progressbar import * 7 | import tensorflow as tf 8 | from utils import * 9 | from op_utils import * 10 | 11 | class ModelSolver(object): 12 | def __init__(self, model, data, val_data, preprocessing, **kwargs): 13 | self.model = model 14 | self.data = data 15 | self.val_data = val_data 16 | self.preprocessing = preprocessing 17 | self.cross_val = kwargs.pop('cross_val', False) 18 | self.cpt_ext = kwargs.pop('cpt_ext', False) 19 | self.weighted_loss = kwargs.pop('weighted_loss', False) 20 | self.n_epochs = kwargs.pop('n_epochs', 10) 21 | self.batch_size = kwargs.pop('batch_size', 32) 22 | self.learning_rate = kwargs.pop('learning_rate', 0.000001) 23 | self.update_rule = kwargs.pop('update_rule', 'adam') 24 | self.model_path = kwargs.pop('model_path', './model/') 25 | self.save_every = kwargs.pop('save_every', 1) 26 | self.log_path = kwargs.pop('log_path', './log/') 27 | self.pretrained_model = kwargs.pop('pretrained_model', None) 28 | self.test_model = kwargs.pop('test_model', './model/lstm/model-1') 29 | 30 | if self.update_rule == 'adam': 31 | self.optimizer = tf.train.AdamOptimizer 32 | elif self.update_rule == 'momentum': 33 | self.optimizer = tf.train.MomentumOptimizer 34 | elif self.update_rule == 'rmsprop': 35 | self.optimizer = tf.train.RMSPropOptimizer 36 | 37 | if not os.path.exists(self.model_path): 38 | os.makedirs(self.model_path) 39 | if not os.path.exists(self.log_path): 40 | os.makedirs(self.log_path) 41 | 42 | def train(self, test_data): 43 | x = self.data['x'] 44 | y = self.data['y'] 45 | x_val = self.val_data['x'] 46 | y_val = self.val_data['y'] 47 | # build graphs 48 | y_, loss = self.model.build_model() 49 | grad_loss = loss 50 | # train op 51 | with tf.variable_scope('optimizer', reuse=tf.AUTO_REUSE): 52 | optimizer = self.optimizer(learning_rate=self.learning_rate) 53 | grads = tf.gradients(grad_loss, tf.trainable_variables()) 54 | grads_and_vars = list(zip(grads, tf.trainable_variables())) 55 | train_op = optimizer.apply_gradients(grads_and_vars=grads_and_vars) 56 | 57 | gpu_options = tf.GPUOptions(allow_growth=True) 58 | tf.get_variable_scope().reuse_variables() 59 | 60 | with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: 61 | tf.global_variables_initializer().run() 62 | #summary_writer = tf.summary.FileWriter(self.log_path, graph=sess.graph) 63 | saver = tf.train.Saver(tf.global_variables()) 64 | if self.pretrained_model is not None: 65 | print("Start training with pretrained model...") 66 | pretrained_model_path = os.path.join(self.model_path, self.pretrained_model) 67 | saver.restore(sess, pretrained_model_path) 68 | #curr_loss = 0 69 | start_t = time.time() 70 | for e in range(self.n_epochs): 71 | # =============================== train =================================== 72 | curr_loss = 0 73 | widgets = ['Train: ', Percentage(), ' ', Bar('#'), ' ', ETA()] 74 | pbar = ProgressBar(widgets=widgets, maxval=len(x)).start() 75 | for i in range(len(x)): 76 | pbar.update(i) 77 | feed_dict = {self.model.x_c: np.array(x[i][0]), self.model.x_p: np.array(x[i][1]), self.model.x_t: np.array(x[i][2]), 78 | self.model.x_ext: np.array(x[i][3]), 79 | self.model.y: np.array(y[i])} 80 | _, l = sess.run([train_op, loss], feed_dict) 81 | curr_loss += l 82 | pbar.finish() 83 | # compute counts of all regions 84 | t_count = 0 85 | for c in range(len(y)): 86 | t_count += np.prod(np.array(y[c]).shape) 87 | t_rmse = np.sqrt(curr_loss/t_count) 88 | #print("at epoch " + str(e) + ", train loss is " + str(curr_loss) + ' , ' + str(t_rmse) + ' , ' + str(self.preprocessing.real_loss(t_rmse))) 89 | # ================================= validate ================================= 90 | print('validate for val data...') 91 | val_loss = 0 92 | widgets = ['Validation: ', Percentage(), ' ', Bar('#'), ' ', ETA()] 93 | pbar = ProgressBar(widgets=widgets, maxval=len(x_val)).start() 94 | for i in range(len(y_val)): 95 | pbar.update(i) 96 | feed_dict = {self.model.x_c: np.array(x_val[i][0]), self.model.x_p: np.array(x_val[i][1]), self.model.x_t: np.array(x_val[i][2]), 97 | self.model.x_ext: np.array(x_val[i][3]), 98 | self.model.y: np.array(y_val[i])} 99 | y_val_, l = sess.run([y_, loss], feed_dict=feed_dict) 100 | val_loss += l 101 | pbar.finish() 102 | # y_val : [batches, batch_size, seq_length, row, col, channel] 103 | #print(np.array(y_val).shape) 104 | v_count = 0 105 | for v in range(len(y_val)): 106 | v_count += np.prod(np.array(y_val[v]).shape) 107 | v_rmse = np.sqrt(val_loss/v_count) 108 | #print("at epoch " + str(e) + ", validate loss is " + str(val_loss) + ' , ' + str(rmse) + ' , ' + str(self.preprocessing.real_loss(v_rmse))) 109 | #print("elapsed time: ", time.time() - start_t) 110 | if (e+1)%self.save_every == 0: 111 | save_name = os.path.join(self.model_path, 'model') 112 | saver.save(sess, save_name, global_step=e+1) 113 | print("model-%s saved." % (e+1)) 114 | # ============================ for test data =============================== 115 | print('test for test data...') 116 | x_test = test_data['x'] 117 | y_test = test_data['y'] 118 | t_loss = 0 119 | y_pre_test = [] 120 | widgets = ['Test: ', Percentage(), ' ', Bar('#'), ' ', ETA()] 121 | pbar = ProgressBar(widgets=widgets, maxval=len(x_test)).start() 122 | for i in range(len(y_test)): 123 | pbar.update(i) 124 | feed_dict = {self.model.x_c: np.array(x_test[i][0]), self.model.x_p: np.array(x_test[i][1]), 125 | self.model.x_t: np.array(x_test[i][2]), 126 | self.model.x_ext: np.array(x_test[i][3]), 127 | self.model.y: np.array(y_test[i])} 128 | y_pre_i, l = sess.run([y_, loss], feed_dict=feed_dict) 129 | t_loss += l 130 | y_pre_test.append(y_pre_i) 131 | pbar.finish() 132 | # y_val : [batches, batch_size, seq_length, row, col, channel] 133 | # print(np.array(y_test).shape) 134 | y_true = self.preprocessing.inverse_transform(np.array(y_test)) 135 | y_prediction = self.preprocessing.inverse_transform(np.array(y_pre_test)) 136 | rmse, mae, mape = RMSE(y_prediction, y_true), MAE(y_prediction, y_true), MAPE(y_prediction, y_true) 137 | text = 'at epoch %d, test loss is %.6f, test prediction rmse/mae/mape is %.6f/%.6f/%.6f \n' % ( 138 | e, t_loss, rmse, mae, mape) 139 | print("at epoch " + str(e) + ", train loss is " + str(curr_loss) + ' , ' + str(t_rmse) + ' , ' + str(self.preprocessing.real_loss(t_rmse))) 140 | print("at epoch " + str(e) + ", validate loss is " + str(val_loss) + ' , ' + str(rmse) + ' , ' + str(self.preprocessing.real_loss(v_rmse))) 141 | print(text) 142 | return np.array(y_prediction) 143 | 144 | def test(self, data): 145 | x = data['x'] 146 | y = data['y'] 147 | # build graphs 148 | y_, loss = self.model.build_model() 149 | tf.get_variable_scope().reuse_variables() 150 | y_pred_all = [] 151 | 152 | with tf.Session() as sess: 153 | saver = tf.train.Saver() 154 | saver.restore(sess, self.test_model) 155 | start_t = time.time() 156 | #y_pred_all = np.ndarray(y.shape) 157 | t_loss = 0 158 | for i in range(len(y)): 159 | feed_dict = {self.model.x_c: np.array(x[i][0]), self.model.x_p: np.array(x[i][1]), self.model.x_t: np.array(x[i][2]), 160 | self.model.x_ext: np.array(x[i][3]), 161 | self.model.y: np.array(y[i])} 162 | y_p, l = sess.run([y_, loss], feed_dict=feed_dict) 163 | t_loss += l 164 | y_pred_all.append(y_p) 165 | 166 | # y : [batches, batch_size, seq_length, row, col, channel] 167 | print(np.array(y).shape) 168 | y_true = self.preprocessing.inverse_transform(np.array(y_test)) 169 | y_prediction = self.preprocessing.inverse_transform(np.array(y_pre_test)) 170 | rmse, mae, mape = RMSE(y_prediction, y_true), MAE(y_prediction, y_true), MAPE(y_prediction, y_true) 171 | text = 'test loss is %.6f, test prediction rmse/mae/mape is %.6f/%.6f/%.6f \n' % (t_loss, rmse, mae, mape) 172 | print(text) 173 | return y_pred_all 174 | # if save_outputs: 175 | # np.save('test_outputs.npy',y_pred_all) 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | -------------------------------------------------------------------------------- /baseline/DMVSTNet/model_2.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from utils import * 3 | import os 4 | from keras import backend as K, losses 5 | import numpy as np 6 | import pandas as pd 7 | import argparse 8 | from keras.models import Sequential, Model 9 | from keras import activations 10 | from keras.engine.topology import Layer, InputSpec 11 | from keras.utils import conv_utils 12 | from keras.layers import LSTM, InputLayer, Dense, Input, Flatten, concatenate, Reshape 13 | from keras.callbacks import EarlyStopping, ModelCheckpoint 14 | from sklearn.preprocessing import MinMaxScaler 15 | from sklearn.metrics import mean_squared_error 16 | from keras.optimizers import Adam 17 | from keras import metrics 18 | from keras.layers.normalization import BatchNormalization 19 | from random import randint 20 | import pickle 21 | 22 | # project_path = "/home/pengshunfeng/project/DMVSTNet" 23 | # project_path = "E:/pyCharm/DMVSTNet" 24 | 25 | #batch_size = 64 26 | mean_label = 0.0 27 | label_max = 0 28 | label_min = 0 29 | max_epoch = 100 30 | num_feature = 100 31 | # seq_len = 8 32 | hidden_dim = 64 33 | threshold = 10.0 34 | maxtruey = 0 35 | mintruey = 0 36 | #eps = 1e-5 37 | eps = 1e-6 38 | loss_lambda = 10.0 39 | feature_len = 0 40 | local_image_size = 9 41 | cnn_hidden_dim_first = 32 42 | len_valid_id = 0 43 | toponet_len = 32 44 | 45 | config = tf.ConfigProto() 46 | config.gpu_options.allow_growth = True 47 | sess = tf.Session(config=config) 48 | K.set_session(sess) 49 | 50 | 51 | class Local_Seq_Conv(Layer): 52 | 53 | def __init__(self, output_dim, seq_len, feature_size, kernel_size, activation=None, 54 | kernel_initializer='glorot_uniform', 55 | bias_initializer='zeros', padding='same', strides=(1, 1), **kwargs): 56 | super(Local_Seq_Conv, self).__init__(**kwargs) 57 | self.output_dim = output_dim 58 | self.seq_len = seq_len 59 | self.bias_initializer = bias_initializer 60 | self.kernel_size = kernel_size 61 | self.kernel_initializer = kernel_initializer 62 | self.padding = padding 63 | self.strides = strides 64 | self.activation = activations.get(activation) 65 | 66 | def build(self, input_shape): 67 | batch_size = input_shape[0] 68 | self.kernel = [] 69 | self.bias = [] 70 | for eachlen in range(self.seq_len): 71 | self.kernel += [self.add_weight(shape=self.kernel_size, 72 | initializer=self.kernel_initializer, 73 | trainable=True, name='kernel_{0}'.format(eachlen))] 74 | 75 | self.bias += [self.add_weight(shape=(self.kernel_size[-1],), 76 | initializer=self.bias_initializer, 77 | trainable=True, name='bias_{0}'.format(eachlen))] 78 | self.build = True 79 | 80 | def call(self, inputs): 81 | output = [] 82 | for eachlen in range(self.seq_len): 83 | 84 | tmp = K.bias_add(K.conv2d(inputs[:, eachlen, :, :, :], self.kernel[eachlen], 85 | strides=self.strides, padding=self.padding), self.bias[eachlen]) 86 | 87 | if self.activation is not None: 88 | output += [self.activation(tmp)] 89 | 90 | output = tf.stack(output, axis=1) 91 | return output 92 | 93 | def compute_output_shape(self, input_shape): 94 | return (input_shape[0], input_shape[1], input_shape[2], input_shape[3], self.output_dim) 95 | 96 | def squared_error(y_true, y_pred): 97 | return K.sum(K.square(y_pred - y_true)) 98 | 99 | def build_model(trainY, testY, trainimage, testimage, 100 | feature_len, 101 | minMax, 102 | seq_len=8, 103 | batch_size=64, 104 | trainable=True, 105 | name="MODEL", 106 | model_path='./'): 107 | print(testimage.shape) 108 | # X_train, Y_train, X_test, Y_test = Featureset_get() 109 | image_input = Input(shape=(seq_len, local_image_size, 110 | local_image_size, None), name='cnn_input') 111 | spatial = Local_Seq_Conv(output_dim=cnn_hidden_dim_first, seq_len=seq_len, feature_size=feature_len, 112 | kernel_size=(3, 3, 1, cnn_hidden_dim_first), activation='relu', 113 | kernel_initializer='glorot_uniform', bias_initializer='zeros', padding='same', 114 | strides=(1, 1))(image_input) 115 | spatial = BatchNormalization()(spatial) 116 | # spatial = Local_Seq_Pooling(seq_len=seq_len)(spatial) 117 | spatial = Local_Seq_Conv(output_dim=cnn_hidden_dim_first, seq_len=seq_len, feature_size=feature_len, 118 | kernel_size=(3, 3, cnn_hidden_dim_first, cnn_hidden_dim_first), activation='relu', 119 | kernel_initializer='glorot_uniform', bias_initializer='zeros', padding='same', 120 | strides=(1, 1))(spatial) 121 | spatial = BatchNormalization()(spatial) 122 | # spatial = Local_Seq_Pooling(seq_len=seq_len)(spatial) 123 | spatial = Local_Seq_Conv(output_dim=cnn_hidden_dim_first, seq_len=seq_len, feature_size=feature_len, 124 | kernel_size=(3, 3, cnn_hidden_dim_first, cnn_hidden_dim_first), activation='relu', 125 | kernel_initializer='glorot_uniform', bias_initializer='zeros', padding='same', 126 | strides=(1, 1))(spatial) 127 | # spatial = Local_Seq_Pooling(seq_len=seq_len)(spatial) 128 | # spatial = BatchNormalization()(spatial) 129 | # spatial = Local_Seq_Conv(output_dim=cnn_hidden_dim_first, seq_len=seq_len, feature_size=feature_len, kernel_size=(3, 3, cnn_hidden_dim_first, cnn_hidden_dim_first), activation='relu', kernel_initializer='glorot_uniform', bias_initializer='zeros', padding='same', strides=(1, 1))(spatial) 130 | # spatial = BatchNormalization()(spatial) 131 | # spatial = Local_Seq_Pooling(seq_len=seq_len)(spatial) 132 | # spatial = Local_Seq_Conv(output_dim=cnn_hidden_dim_first, seq_len=seq_len, feature_size=feature_len, kernel_size=(3, 3, cnn_hidden_dim_first, cnn_hidden_dim_first), activation='relu', kernel_initializer='glorot_uniform', bias_initializer='zeros', padding='same', strides=(1, 1))(spatial) 133 | spatial = Flatten()(spatial) 134 | spatial = Reshape(target_shape=(seq_len, -1))(spatial) 135 | spatial_out = Dense(units=64, activation='relu')(spatial) 136 | 137 | # lstm_input = Input(shape=(seq_len, feature_len), 138 | # dtype='float32', name='lstm_input') 139 | # 140 | # x = concatenate([lstm_input, spatial_out], axis=-1) 141 | x = spatial_out 142 | # lstm_out = Dense(units=128, activation=relu)(x) 143 | lstm_out = LSTM(units=hidden_dim, return_sequences=False, dropout=0)(x) 144 | 145 | #topo_input = Input(shape=(toponet_len,), dtype='float32', name='topo_input') 146 | #topo_emb = Dense(units=6, activation='tanh')(topo_input) 147 | #static_dynamic_concate = concatenate([lstm_out, topo_emb], axis=-1) 148 | 149 | #res = Dense(units=1, activation='sigmoid')(static_dynamic_concate) 150 | #res = Dense(units=2, activation='sigmoid')(static_dynamic_concate) 151 | res = Dense(units=2, activation='sigmoid')(lstm_out) 152 | # model = Model(inputs=[image_input, lstm_input, topo_input], 153 | # outputs=res) 154 | model = Model(inputs=image_input, 155 | outputs=res) 156 | #sgd = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=1e-6) 157 | sgd = Adam(lr=0.0002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=1e-6) 158 | #model.compile(loss=losses.mse, optimizer=sgd, metrics=[metrics.mse]) 159 | model.compile(loss=squared_error, optimizer=sgd, metrics=[metrics.mse]) 160 | earlyStopping = EarlyStopping( 161 | monitor='val_loss', patience=5, verbose=0, mode='min') 162 | # model.fit([trainimage, trainX, traintopo], trainY, batch_size=batch_size, epochs=max_epoch, validation_split=0.1, 163 | # callbacks=[earlyStopping]) 164 | 165 | fname_param = os.path.join(model_path, 'log', name) 166 | if not os.path.exists(fname_param): 167 | os.makedirs(fname_param) 168 | fname_param = os.path.join(fname_param, 'DMVSTNet.best.h5') 169 | # early_stopping = EarlyStopping(monitor='val_rmse', patience=5, mode='min') 170 | model_checkpoint = ModelCheckpoint( 171 | fname_param, monitor='val_loss', verbose=0, save_best_only=True, mode='min') 172 | if trainable: 173 | model.fit(trainimage, trainY, batch_size=batch_size, epochs=max_epoch, validation_split=0.1, 174 | callbacks=[earlyStopping, model_checkpoint]) 175 | else: 176 | model.load_weights(fname_param) 177 | # testLoss = model.evaluate([testimage, testtopo], testY) 178 | score = model.evaluate(trainimage, trainY, batch_size=batch_size, verbose=0) 179 | # print('Train score: %.6f rmse (norm): %.6f rmse (real): %.6f' % 180 | # (score[0], score[1], minMax.inverse(np.sqrt(score[1])))) 181 | print('Train score: %.6f se (norm): %.6f se (real): %.6f' % 182 | (score[0], score[1], minMax.inverse(minMax.inverse(score[1])))) 183 | 184 | score = model.evaluate(testimage, testY, batch_size=batch_size, verbose=0) 185 | # print('Test score: %.6f rmse (norm): %.6f rmse (real): %.6f' % 186 | # (score[0], score[1], minMax.inverse(np.sqrt(score[1])))) 187 | print('Test score: %.6f se (norm): %.6f se (real): %.6f' % 188 | (score[0], score[1], minMax.inverse(minMax.inverse(score[1])))) 189 | # model.save('local_conv_lstm_total_embed.h5') 190 | 191 | prediction = model.predict(testimage, batch_size=batch_size, verbose=0) 192 | print(prediction.shape) 193 | test_mse = minMax.inverse(minMax.inverse(np.mean(np.square(prediction - testY)))) 194 | print('test mse is %.6f, and rmse : %.6f' % (test_mse, np.sqrt(test_mse))) 195 | return prediction 196 | 197 | # return model 198 | -------------------------------------------------------------------------------- /baseline/DMVSTNet/model.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from utils import * 3 | import os 4 | from keras import backend as K, losses 5 | import numpy as np 6 | import pandas as pd 7 | import argparse 8 | from keras.models import Sequential, Model 9 | from keras import activations 10 | from keras.engine.topology import Layer, InputSpec 11 | from keras.utils import conv_utils 12 | from keras.layers import LSTM, InputLayer, Dense, Input, Flatten, concatenate, Reshape 13 | from keras.callbacks import EarlyStopping, ModelCheckpoint 14 | from sklearn.preprocessing import MinMaxScaler 15 | from sklearn.metrics import mean_squared_error 16 | from keras.optimizers import Adam 17 | from keras import metrics 18 | from keras.layers.normalization import BatchNormalization 19 | from random import randint 20 | import pickle 21 | 22 | # project_path = "/home/pengshunfeng/project/DMVSTNet" 23 | # project_path = "E:/pyCharm/DMVSTNet" 24 | 25 | #batch_size = 64 26 | mean_label = 0.0 27 | label_max = 0 28 | label_min = 0 29 | max_epoch = 100 30 | num_feature = 100 31 | # seq_len = 8 32 | hidden_dim = 512 33 | threshold = 10.0 34 | maxtruey = 0 35 | mintruey = 0 36 | #eps = 1e-5 37 | eps = 1e-6 38 | loss_lambda = 10.0 39 | feature_len = 0 40 | local_image_size = 9 41 | cnn_hidden_dim_first = 32 42 | len_valid_id = 0 43 | toponet_len = 32 44 | 45 | config = tf.ConfigProto() 46 | config.gpu_options.allow_growth = True 47 | sess = tf.Session(config=config) 48 | K.set_session(sess) 49 | 50 | 51 | class Local_Seq_Conv(Layer): 52 | 53 | def __init__(self, output_dim, seq_len, feature_size, kernel_size, activation=None, 54 | kernel_initializer='glorot_uniform', 55 | bias_initializer='zeros', padding='same', strides=(1, 1), **kwargs): 56 | super(Local_Seq_Conv, self).__init__(**kwargs) 57 | self.output_dim = output_dim 58 | self.seq_len = seq_len 59 | self.bias_initializer = bias_initializer 60 | self.kernel_size = kernel_size 61 | self.kernel_initializer = kernel_initializer 62 | self.padding = padding 63 | self.strides = strides 64 | self.activation = activations.get(activation) 65 | 66 | def build(self, input_shape): 67 | batch_size = input_shape[0] 68 | self.kernel = [] 69 | self.bias = [] 70 | for eachlen in range(self.seq_len): 71 | self.kernel += [self.add_weight(shape=self.kernel_size, 72 | initializer=self.kernel_initializer, 73 | trainable=True, name='kernel_{0}'.format(eachlen))] 74 | 75 | self.bias += [self.add_weight(shape=(self.kernel_size[-1],), 76 | initializer=self.bias_initializer, 77 | trainable=True, name='bias_{0}'.format(eachlen))] 78 | self.build = True 79 | 80 | def call(self, inputs): 81 | output = [] 82 | for eachlen in range(self.seq_len): 83 | 84 | tmp = K.bias_add(K.conv2d(inputs[:, eachlen, :, :, :], self.kernel[eachlen], 85 | strides=self.strides, padding=self.padding), self.bias[eachlen]) 86 | 87 | if self.activation is not None: 88 | output += [self.activation(tmp)] 89 | 90 | output = tf.stack(output, axis=1) 91 | return output 92 | 93 | def compute_output_shape(self, input_shape): 94 | return (input_shape[0], input_shape[1], input_shape[2], input_shape[3], self.output_dim) 95 | 96 | def squared_error(y_true, y_pred): 97 | return K.sum(K.square(y_pred - y_true)) 98 | 99 | def build_model(trainY, testY, trainimage, testimage, traintopo, testtopo, 100 | feature_len, 101 | minMax, 102 | seq_len=8, 103 | batch_size=64, 104 | trainable=True, 105 | name="MODEL", 106 | model_path='./'): 107 | print(testimage.shape) 108 | # X_train, Y_train, X_test, Y_test = Featureset_get() 109 | image_input = Input(shape=(seq_len, local_image_size, 110 | local_image_size, None), name='cnn_input') 111 | spatial = Local_Seq_Conv(output_dim=cnn_hidden_dim_first, seq_len=seq_len, feature_size=feature_len, 112 | kernel_size=(3, 3, 1, cnn_hidden_dim_first), activation='relu', 113 | kernel_initializer='glorot_uniform', bias_initializer='zeros', padding='same', 114 | strides=(1, 1))(image_input) 115 | spatial = BatchNormalization()(spatial) 116 | # spatial = Local_Seq_Pooling(seq_len=seq_len)(spatial) 117 | spatial = Local_Seq_Conv(output_dim=cnn_hidden_dim_first, seq_len=seq_len, feature_size=feature_len, 118 | kernel_size=(3, 3, cnn_hidden_dim_first, cnn_hidden_dim_first), activation='relu', 119 | kernel_initializer='glorot_uniform', bias_initializer='zeros', padding='same', 120 | strides=(1, 1))(spatial) 121 | spatial = BatchNormalization()(spatial) 122 | # spatial = Local_Seq_Pooling(seq_len=seq_len)(spatial) 123 | spatial = Local_Seq_Conv(output_dim=cnn_hidden_dim_first, seq_len=seq_len, feature_size=feature_len, 124 | kernel_size=(3, 3, cnn_hidden_dim_first, cnn_hidden_dim_first), activation='relu', 125 | kernel_initializer='glorot_uniform', bias_initializer='zeros', padding='same', 126 | strides=(1, 1))(spatial) 127 | # spatial = Local_Seq_Pooling(seq_len=seq_len)(spatial) 128 | # spatial = BatchNormalization()(spatial) 129 | # spatial = Local_Seq_Conv(output_dim=cnn_hidden_dim_first, seq_len=seq_len, feature_size=feature_len, kernel_size=(3, 3, cnn_hidden_dim_first, cnn_hidden_dim_first), activation='relu', kernel_initializer='glorot_uniform', bias_initializer='zeros', padding='same', strides=(1, 1))(spatial) 130 | # spatial = BatchNormalization()(spatial) 131 | # spatial = Local_Seq_Pooling(seq_len=seq_len)(spatial) 132 | # spatial = Local_Seq_Conv(output_dim=cnn_hidden_dim_first, seq_len=seq_len, feature_size=feature_len, kernel_size=(3, 3, cnn_hidden_dim_first, cnn_hidden_dim_first), activation='relu', kernel_initializer='glorot_uniform', bias_initializer='zeros', padding='same', strides=(1, 1))(spatial) 133 | spatial = Flatten()(spatial) 134 | spatial = Reshape(target_shape=(seq_len, -1))(spatial) 135 | spatial_out = Dense(units=64, activation='relu')(spatial) 136 | 137 | # lstm_input = Input(shape=(seq_len, feature_len), 138 | # dtype='float32', name='lstm_input') 139 | # 140 | # x = concatenate([lstm_input, spatial_out], axis=-1) 141 | x = spatial_out 142 | # lstm_out = Dense(units=128, activation=relu)(x) 143 | lstm_out = LSTM(units=hidden_dim, return_sequences=False, dropout=0)(x) 144 | 145 | topo_input = Input(shape=(toponet_len,), dtype='float32', name='topo_input') 146 | topo_emb = Dense(units=6, activation='tanh')(topo_input) 147 | static_dynamic_concate = concatenate([lstm_out, topo_emb], axis=-1) 148 | 149 | #res = Dense(units=1, activation='sigmoid')(static_dynamic_concate) 150 | res = Dense(units=2, activation='sigmoid')(static_dynamic_concate) 151 | # model = Model(inputs=[image_input, lstm_input, topo_input], 152 | # outputs=res) 153 | model = Model(inputs=[image_input, topo_input], 154 | outputs=res) 155 | #sgd = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=1e-6) 156 | sgd = Adam(lr=0.0002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=1e-6) 157 | #model.compile(loss=losses.mse, optimizer=sgd, metrics=[metrics.mse]) 158 | model.compile(loss=squared_error, optimizer=sgd, metrics=[metrics.mse]) 159 | earlyStopping = EarlyStopping( 160 | monitor='val_loss', patience=5, verbose=0, mode='min') 161 | # model.fit([trainimage, trainX, traintopo], trainY, batch_size=batch_size, epochs=max_epoch, validation_split=0.1, 162 | # callbacks=[earlyStopping]) 163 | 164 | fname_param = os.path.join(model_path, 'log', name) 165 | if not os.path.exists(fname_param): 166 | os.makedirs(fname_param) 167 | fname_param = os.path.join(fname_param, 'DMVSTNet.best.h5') 168 | # early_stopping = EarlyStopping(monitor='val_rmse', patience=5, mode='min') 169 | model_checkpoint = ModelCheckpoint( 170 | fname_param, monitor='val_loss', verbose=0, save_best_only=True, mode='min') 171 | if trainable: 172 | model.fit([trainimage, traintopo], trainY, batch_size=batch_size, epochs=max_epoch, validation_split=0.1, 173 | callbacks=[earlyStopping, model_checkpoint]) 174 | else: 175 | model.load_weights(fname_param) 176 | # testLoss = model.evaluate([testimage, testtopo], testY) 177 | score = model.evaluate([trainimage, traintopo], trainY, batch_size=batch_size, verbose=0) 178 | # print('Train score: %.6f rmse (norm): %.6f rmse (real): %.6f' % 179 | # (score[0], score[1], minMax.inverse(np.sqrt(score[1])))) 180 | print('Train score: %.6f se (norm): %.6f se (real): %.6f' % 181 | (score[0], score[1], minMax.inverse(minMax.inverse(score[1])))) 182 | 183 | score = model.evaluate( 184 | [testimage, testtopo], testY, batch_size=batch_size, verbose=0) 185 | # print('Test score: %.6f rmse (norm): %.6f rmse (real): %.6f' % 186 | # (score[0], score[1], minMax.inverse(np.sqrt(score[1])))) 187 | print('Test score: %.6f se (norm): %.6f se (real): %.6f' % 188 | (score[0], score[1], minMax.inverse(minMax.inverse(score[1])))) 189 | # model.save('local_conv_lstm_total_embed.h5') 190 | 191 | prediction = model.predict([testimage, testtopo], batch_size=batch_size, verbose=0) 192 | print(prediction.shape) 193 | test_mse = minMax.inverse(minMax.inverse(np.mean(np.square(prediction - testY)))) 194 | print('test mse is %.6f, and rmse : %.6f' % (test_mse, np.sqrt(test_mse))) 195 | return prediction 196 | 197 | # return model 198 | -------------------------------------------------------------------------------- /baseline/STResNet/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pickle 3 | import scipy.io as sio 4 | import h5py 5 | import time 6 | import os 7 | 8 | def load_data(filename, split): 9 | if len(filename)==2: 10 | d1 = sio.loadmat(filename[0])['p_map'] 11 | d2 = sio.loadmat(filename[1])['d_map'] 12 | data = np.concatenate((d1[:,:,:,np.newaxis], d2[:,:,:,np.newaxis]), axis=3) 13 | train = data[0:split[0],:,:,:] 14 | validate = data[split[0]:split[0]+split[1],:,:,:] 15 | test = data[split[0]+split[1]:split[0]+split[1]+split[2],:,:,:] 16 | return data, train, validate, test 17 | 18 | 19 | def load_npy_data(filename, split): 20 | if len(filename) == 2: 21 | d1 = np.load(filename[0]) 22 | d2 = np.load(filename[1]) 23 | data = np.concatenate((np.expand_dims(d1, axis=-1), np.expand_dims(d2, axis=-1)), axis=-1) 24 | elif len(filename) == 1: 25 | data = np.load(filename[0]) 26 | train = data[0:split[0]] 27 | if len(split) > 2: 28 | validate = data[split[0]:(split[0] + split[1])] 29 | test = data[(split[0]+split[1]):(split[0]+split[1]+split[2])] 30 | else: 31 | validate = None 32 | test = data[split[0]:(split[0] + split[1])] 33 | return data, train, validate, test 34 | 35 | 36 | def load_h5data(fname): 37 | f = h5py.File(fname, 'r') 38 | data = f['data'].value 39 | data = np.asarray(data) 40 | data = np.transpose(np.asarray(data), (0,2,3,1)) 41 | timestamps = f['date'].value 42 | f.close() 43 | return data, timestamps 44 | 45 | def batch_data(data, batch_size=32, input_steps=10, output_steps=10): 46 | # data: [num, row, col, channel] 47 | num = data.shape[0] 48 | # x: [batches, batch_size, input_steps, row, col, channel] 49 | # y: [batches, batch_size, output_steps, row, col, channel] 50 | x = [] 51 | y = [] 52 | i = 0 53 | while i= num: 71 | # break 72 | # for d in range(len(depends)): 73 | # x_.append(data[i+b-np.array(depends[d]), :, :, :]) 74 | # x_.append(ext[i]) 75 | # y_b.append(data[i+b, :, :, :]) 76 | # x_b.append(x_) 77 | # x.append(x_b) 78 | # y.append(y_b) 79 | # i += batch_size 80 | 81 | def batch_data_cpt_ext(data, timestamps, batch_size=32, close=3, period=4, trend=4): 82 | # data: [num, row, col, channel] 83 | num = data.shape[0] 84 | #flow = data.shape[1] 85 | # x: [batches, 86 | #[ 87 | #[batch_size, row, col, close*flow], 88 | #[batch_size, row, col, period*flow], 89 | #[batch_size, row, col, trend*flow], 90 | #[batch_size, external_dim] 91 | #] 92 | #] 93 | c = 1 94 | p = 24 95 | t = 24*7 96 | depends = [ [c*j for j in range(1, close+1)], 97 | [p*j for j in range(1, period+1)], 98 | [t*j for j in range(1, trend+1)] ] 99 | depends = np.asarray(depends) 100 | i = max(c*close, p*period, t*trend) 101 | # external feature 102 | ext = external_feature(timestamps) 103 | # ext plus c p t 104 | # x: [batches, 4, batch_size] 105 | # y: [batches, batch_size] 106 | x = [] 107 | y = [] 108 | while i= num: 114 | break 115 | x_.append(np.transpose(np.vstack(np.transpose(data[i+b-np.array(depends[d]), :, :, :],[0,3,1,2])), [1,2,0])) 116 | x_ = np.array(x_) 117 | x_b[d] = x_ 118 | #x_b.append(x_) 119 | # external features 120 | x_b[-1] = ext[i:min(i+batch_size, num)] 121 | # y 122 | y_b = data[i:min(i+batch_size, num), :, :, :] 123 | x.append(x_b) 124 | #print(y_b.shape) 125 | y.append(y_b) 126 | i += batch_size 127 | return x, y 128 | 129 | def external_feature(timestamps): 130 | vec = [time.strptime(t[:8], '%Y%m%d').tm_wday for t in timestamps] 131 | ext = [] 132 | for j in vec: 133 | v = [0 for _ in range(7)] 134 | v[j] = 1 135 | if j >= 5: 136 | v.append(0) # weekend 137 | else: 138 | v.append(1) # weekday 139 | ext.append(v) 140 | ext = np.asarray(ext) 141 | return ext 142 | 143 | def gen_timestamps_for_year(year): 144 | month1 = ['0'+str(e) for e in range(1,10)] 145 | month2 = [str(e) for e in range(10,13)] 146 | month = month1+month2 147 | day1 = ['0'+str(e) for e in range(1,10)] 148 | day2 = [str(e) for e in range(10,32)] 149 | day = day1+day2 150 | if year=='2012' or year=='2016': 151 | day_sum = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] 152 | else: 153 | day_sum = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] 154 | timestamps = [] 155 | for m in range(len(month)): 156 | for d in range(day_sum[m]): 157 | t = [year+month[m]+day[d]] 158 | t_d = t*24 159 | timestamps.append(t_d[:]) 160 | timestamps = np.hstack(np.array(timestamps)) 161 | return timestamps 162 | 163 | def gen_timestamps_intervals(year, month, intervals): 164 | num_repeat = int(60/intervals)*24 165 | if year=='2012' or year=='2016': 166 | day_sum = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] 167 | else: 168 | day_sum = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] 169 | day1 = ['0' + str(e) for e in range(1, 10)] 170 | day2 = [str(e) for e in range(10, 32)] 171 | day = day1 + day2 172 | timestamps = [] 173 | for d in range(day_sum[int(month)-1]): 174 | t = [year+str(month)+day[d]] 175 | t_d = t*num_repeat 176 | timestamps.append(t_d[:]) 177 | timestamps = np.hstack(np.array(timestamps)) 178 | return timestamps 179 | 180 | def gen_timestamps(years, gen_timestamps_for_year=gen_timestamps_for_year): 181 | timestamps = [] 182 | for y in years: 183 | timestamps.append(gen_timestamps_for_year(y)) 184 | timestamps = np.hstack(np.array(timestamps)) 185 | return timestamps 186 | 187 | def gen_timestamps_for_year_ymdh(year): 188 | month1 = ['0'+str(e) for e in range(1,10)] 189 | month2 = [str(e) for e in range(10,13)] 190 | month = month1+month2 191 | day1 = ['0'+str(e) for e in range(1,10)] 192 | day2 = [str(e) for e in range(10,32)] 193 | day = day1+day2 194 | if year=='2012' or year=='2016': 195 | day_sum = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] 196 | else: 197 | day_sum = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] 198 | hour1 = ['0'+str(e) for e in range(0,10)] 199 | hour2 = [str(e) for e in range(10,24)] 200 | hour = hour1+hour2 201 | timestamps = [] 202 | for m in range(len(month)): 203 | for d in range(day_sum[m]): 204 | #t = [year+month[m]+day[d]] 205 | t_d = [] 206 | for h in range(24): 207 | t_d.append(year+month[m]+day[d]+hour[h]) 208 | timestamps.append(t_d[:]) 209 | timestamps = np.hstack(np.array(timestamps)) 210 | return timestamps 211 | 212 | def gen_timestamps_for_year_ymdhm(year): 213 | month1 = ['0'+str(e) for e in range(1,10)] 214 | month2 = [str(e) for e in range(10,13)] 215 | month = month1+month2 216 | day1 = ['0'+str(e) for e in range(1,10)] 217 | day2 = [str(e) for e in range(10,32)] 218 | day = day1+day2 219 | if year=='2012' or year=='2016': 220 | day_sum = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] 221 | else: 222 | day_sum = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] 223 | hour1 = ['0'+str(e) for e in range(0,10)] 224 | hour2 = [str(e) for e in range(10,24)] 225 | hour = hour1+hour2 226 | minute = ['00', '10', '20', '30', '40', '50'] 227 | timestamps = [] 228 | for m in range(len(month)): 229 | for d in range(day_sum[m]): 230 | #t = [year+month[m]+day[d]] 231 | t_d = [] 232 | for h in range(24): 233 | a = [year+month[m]+day[d]+hour[h]+e for e in minute] 234 | #t_d = [t_d.append(year+month[m]+day[d]+hour[h]+e) for e in minute] 235 | t_d.append(a) 236 | t_d = np.hstack(np.array(t_d)) 237 | timestamps.append(t_d[:]) 238 | timestamps = np.hstack(np.array(timestamps)) 239 | return timestamps 240 | 241 | def shuffle_batch_data(data, batch_size=32, input_steps=10, output_steps=10): 242 | num = data.shape[0] 243 | # shuffle 244 | data = data[np.random.shuffle(np.arange(num)), :, :, :] 245 | 246 | x = [] 247 | y = [] 248 | i = 0 249 | while i 2: 71 | cells = [first_cell] + [cell] * (num_layers-2) + [last_cell] 72 | else: 73 | cells = [first_cell, last_cell] 74 | #cells = [first_cell, last_cell] 75 | 76 | self.cells = tf.contrib.rnn.MultiRNNCell(cells, state_is_tuple=True) 77 | 78 | self.x = tf.placeholder(tf.float32, [self.batch_size, self.input_steps, self.num_nodes, 2]) 79 | self.f = tf.placeholder(tf.float32, [self.batch_size, self.input_steps, self.num_nodes, self.num_nodes]) 80 | self.y = tf.placeholder(tf.float32, [self.batch_size, self.input_steps, self.num_nodes, 2]) 81 | 82 | 83 | def build_easy_model(self): 84 | x = tf.transpose(tf.reshape(self.x, (self.batch_size, self.input_steps, -1)), [1, 0, 2]) 85 | #inputs = tf.unstack(x, axis=0) 86 | f_all = tf.transpose(tf.reshape(self.f, (self.batch_size, self.input_steps, -1)), [1, 0, 2]) 87 | inputs = tf.concat([x, f_all], axis=-1) 88 | inputs = tf.unstack(inputs, axis=0) 89 | # 90 | outputs, _ = tf.contrib.rnn.static_rnn(self.cells, inputs, dtype=tf.float32) 91 | outputs = tf.stack(outputs) 92 | # 93 | # temporal attention 94 | outputs = tf.reshape(outputs, (self.input_steps, self.batch_size, self.num_nodes, -1)) 95 | # outputs: [input_steps, batch_size, n, -] 96 | if self.multi_loss: 97 | if self.dy_temporal: 98 | with tf.variable_scope('temporal_attention', reuse=tf.AUTO_REUSE): 99 | h_states = tf.transpose(outputs, (1, 0, 2, 3)) 100 | att_states = [] 101 | for t in range(self.input_steps): 102 | att_state, _ = self.temporal_attention_layer(outputs[t], h_states, self.att_units, 103 | reuse=tf.AUTO_REUSE) 104 | att_states.append(att_state) 105 | att_states = tf.stack(att_states) 106 | outputs = tf.concat([outputs, att_states], -1) 107 | # projection 108 | outputs = tf.layers.dense(outputs, units=2, activation=None, 109 | kernel_initializer=self.weight_initializer) 110 | outputs = tf.transpose(outputs, [1, 0, 2, 3]) 111 | loss = 2 * tf.nn.l2_loss(self.y - outputs) 112 | return outputs, loss 113 | else: 114 | if self.dy_temporal: 115 | with tf.variable_scope('temporal_attention', reuse=tf.AUTO_REUSE): 116 | h_states = tf.transpose(outputs[:-1], (1, 0, 2, 3)) 117 | att_states, _ = self.temporal_attention_layer(outputs[-1], h_states, self.att_units, 118 | reuse=tf.AUTO_REUSE) 119 | output = tf.concat([outputs[-1], att_states], -1) 120 | else: 121 | output = outputs[-1] 122 | # projection 123 | output = tf.layers.dense(output, units=2, activation=None, 124 | kernel_initializer=self.weight_initializer) 125 | loss = 2 * tf.nn.l2_loss(self.y[:, -1, :, :] - output) 126 | return tf.expand_dims(output, 1), loss 127 | ''' 128 | # projection 129 | with tf.variable_scope('dense', reuse=tf.AUTO_REUSE): 130 | outputs = tf.layers.dense(tf.reshape(outputs, (-1, self.num_units)), units=2, activation=None, kernel_initializer=self.weight_initializer) 131 | # 132 | outputs = tf.reshape(outputs, (self.input_steps, self.batch_size, self.num_nodes, -1)) 133 | outputs = tf.transpose(outputs, [1, 0, 2, 3]) 134 | loss = 2 * tf.nn.l2_loss(self.y - outputs) 135 | return outputs, loss 136 | ''' 137 | 138 | def temporal_attention_layer(self, o_state, h_states, att_units, reuse=True): 139 | # o_state: [batch_size, n, channel] 140 | # h_state: [batch_size, input_steps, n, channel] 141 | o_shape = o_state.get_shape().as_list() 142 | h_shape = h_states.get_shape().as_list() 143 | with tf.variable_scope('att', reuse=reuse): 144 | with tf.variable_scope('att_o_state', reuse=reuse): 145 | w = tf.get_variable('att_o_w', [np.prod(o_shape[1:]), att_units], initializer=self.weight_initializer) 146 | o_att = tf.matmul(tf.reshape(o_state, (-1, np.prod(o_shape[1:]))), w) 147 | # o_att: [batch_size, att_units] 148 | with tf.variable_scope('att_h_state', reuse=reuse): 149 | w = tf.get_variable('att_h_w', [np.prod(h_shape[2:]), att_units], initializer=self.weight_initializer) 150 | h_att = tf.matmul(tf.reshape(h_states, [-1, np.prod(h_shape[2:])]), w) 151 | # h_att: [batch_size*input_steps, att_units] 152 | h_att = tf.reshape(h_att, [-1, h_shape[1], att_units]) 153 | # h_att: [batch_size, input_steps, att_units] 154 | b = tf.get_variable('att_b', [att_units], initializer=self.const_initializer) 155 | o_h_att_plus = tf.nn.relu(h_att + tf.expand_dims(o_att, 1) + b) 156 | mlp_w = tf.get_variable('mlp_w', [att_units, 1], initializer=self.weight_initializer) 157 | out_att = tf.reshape(tf.matmul(tf.reshape(o_h_att_plus, [-1, att_units]), mlp_w), 158 | [-1, h_shape[1]]) 159 | # out_att: [batch_size, input_steps] 160 | alpha = tf.nn.softmax(out_att) 161 | context = tf.reduce_sum(tf.reshape(h_states, (-1, h_shape[1], np.prod(h_shape[2:]))) * tf.expand_dims(alpha, -1), 162 | 1, name='context') 163 | att_context = tf.reshape(context, 164 | [-1, h_shape[2], h_shape[3]]) 165 | # att_context: [batch_size, n, channel] 166 | return att_context, alpha 167 | 168 | ''' 169 | def build_easy_model(self): 170 | x = tf.transpose(tf.reshape(self.x, (self.batch_size, self.input_steps, -1)), [1, 0, 2]) 171 | #inputs = tf.unstack(x, axis=0) 172 | f_all = tf.transpose(tf.reshape(self.f, (self.batch_size, self.input_steps, -1)), [1, 0, 2]) 173 | inputs = tf.concat([x, f_all], axis=-1) 174 | inputs = tf.unstack(inputs, axis=0) 175 | # 176 | outputs, _ = tf.contrib.rnn.static_rnn(self.cells, inputs, dtype=tf.float32) 177 | outputs = tf.stack(outputs) 178 | # 179 | # projection 180 | with tf.variable_scope('dense', reuse=tf.AUTO_REUSE): 181 | outputs = tf.layers.dense(tf.reshape(outputs, (-1, self.num_units)), units=2, activation=None, kernel_initializer=self.weight_initializer) 182 | # 183 | outputs = tf.reshape(outputs, (self.input_steps, self.batch_size, self.num_nodes, -1)) 184 | outputs = tf.transpose(outputs, [1, 0, 2, 3]) 185 | loss = 2 * tf.nn.l2_loss(self.y - outputs) 186 | return outputs, loss 187 | ''' 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | -------------------------------------------------------------------------------- /baseline/STGCN/models/layers.py: -------------------------------------------------------------------------------- 1 | # @Time : Jan. 12, 2019 17:45 2 | # @Author : Veritas YIN 3 | # @FileName : layers.py 4 | # @Version : 1.0 5 | # @IDE : PyCharm 6 | # @Github : https://github.com/VeritasYin/Project_Orion 7 | 8 | import tensorflow as tf 9 | 10 | 11 | def gconv(x, theta, Ks, c_in, c_out): 12 | ''' 13 | Spectral-based graph convolution function. 14 | :param x: tensor, [batch_size, n_route, c_in]. 15 | :param theta: tensor, [Ks*c_in, c_out], trainable kernel parameters. 16 | :param Ks: int, kernel size of graph convolution. 17 | :param c_in: int, size of input channel. 18 | :param c_out: int, size of output channel. 19 | :return: tensor, [batch_size, n_route, c_out]. 20 | ''' 21 | # graph kernel: tensor, [n_route, Ks*n_route] 22 | kernel = tf.get_collection('graph_kernel')[0] 23 | n = tf.shape(kernel)[0] 24 | # x -> [batch_size, c_in, n_route] -> [batch_size*c_in, n_route] 25 | x_tmp = tf.reshape(tf.transpose(x, [0, 2, 1]), [-1, n]) 26 | # x_mul = x_tmp * ker -> [batch_size*c_in, Ks*n_route] -> [batch_size, c_in, Ks, n_route] 27 | x_mul = tf.reshape(tf.matmul(x_tmp, kernel), [-1, c_in, Ks, n]) 28 | # x_ker -> [batch_size, n_route, c_in, K_s] -> [batch_size*n_route, c_in*Ks] 29 | x_ker = tf.reshape(tf.transpose(x_mul, [0, 3, 1, 2]), [-1, c_in * Ks]) 30 | # x_gconv -> [batch_size*n_route, c_out] -> [batch_size, n_route, c_out] 31 | x_gconv = tf.reshape(tf.matmul(x_ker, theta), [-1, n, c_out]) 32 | return x_gconv 33 | 34 | 35 | def layer_norm(x, scope): 36 | ''' 37 | Layer normalization function. 38 | :param x: tensor, [batch_size, time_step, n_route, channel]. 39 | :param scope: str, variable scope. 40 | :return: tensor, [batch_size, time_step, n_route, channel]. 41 | ''' 42 | _, _, N, C = x.get_shape().as_list() 43 | mu, sigma = tf.nn.moments(x, axes=[2, 3], keep_dims=True) 44 | 45 | with tf.variable_scope(scope): 46 | gamma = tf.get_variable('gamma', initializer=tf.ones([1, 1, N, C])) 47 | beta = tf.get_variable('beta', initializer=tf.zeros([1, 1, N, C])) 48 | _x = (x - mu) / tf.sqrt(sigma + 1e-6) * gamma + beta 49 | return _x 50 | 51 | 52 | def temporal_conv_layer(x, Kt, c_in, c_out, act_func='relu'): 53 | ''' 54 | Temporal convolution layer. 55 | :param x: tensor, [batch_size, time_step, n_route, c_in]. 56 | :param Kt: int, kernel size of temporal convolution. 57 | :param c_in: int, size of input channel. 58 | :param c_out: int, size of output channel. 59 | :param act_func: str, activation function. 60 | :return: tensor, [batch_size, time_step-Kt+1, n_route, c_out]. 61 | ''' 62 | _, T, n, _ = x.get_shape().as_list() 63 | 64 | if c_in > c_out: 65 | w_input = tf.get_variable('wt_input', shape=[1, 1, c_in, c_out], dtype=tf.float32) 66 | tf.add_to_collection(name='weight_decay', value=tf.nn.l2_loss(w_input)) 67 | x_input = tf.nn.conv2d(x, w_input, strides=[1, 1, 1, 1], padding='SAME') 68 | elif c_in < c_out: 69 | # if the size of input channel is less than the output, 70 | # padding x to the same size of output channel. 71 | # Note, _.get_shape() cannot convert a partially known TensorShape to a Tensor. 72 | x_input = tf.concat([x, tf.zeros([tf.shape(x)[0], T, n, c_out - c_in])], axis=3) 73 | else: 74 | x_input = x 75 | 76 | # keep the original input for residual connection. 77 | x_input = x_input[:, Kt - 1:T, :, :] 78 | 79 | if act_func == 'GLU': 80 | # gated liner unit 81 | wt = tf.get_variable(name='wt', shape=[Kt, 1, c_in, 2 * c_out], dtype=tf.float32) 82 | tf.add_to_collection(name='weight_decay', value=tf.nn.l2_loss(wt)) 83 | bt = tf.get_variable(name='bt', initializer=tf.zeros([2 * c_out]), dtype=tf.float32) 84 | x_conv = tf.nn.conv2d(x, wt, strides=[1, 1, 1, 1], padding='VALID') + bt 85 | return (x_conv[:, :, :, 0:c_out] + x_input) * tf.nn.sigmoid(x_conv[:, :, :, -c_out:]) 86 | else: 87 | wt = tf.get_variable(name='wt', shape=[Kt, 1, c_in, c_out], dtype=tf.float32) 88 | tf.add_to_collection(name='weight_decay', value=tf.nn.l2_loss(wt)) 89 | bt = tf.get_variable(name='bt', initializer=tf.zeros([c_out]), dtype=tf.float32) 90 | x_conv = tf.nn.conv2d(x, wt, strides=[1, 1, 1, 1], padding='VALID') + bt 91 | if act_func == 'linear': 92 | return x_conv 93 | elif act_func == 'sigmoid': 94 | return tf.nn.sigmoid(x_conv) 95 | elif act_func == 'relu': 96 | return tf.nn.relu(x_conv + x_input) 97 | else: 98 | raise ValueError(f'ERROR: activation function "{act_func}" is not defined.') 99 | 100 | 101 | def spatio_conv_layer(x, Ks, c_in, c_out): 102 | ''' 103 | Spatial graph convolution layer. 104 | :param x: tensor, [batch_size, time_step, n_route, c_in]. 105 | :param Ks: int, kernel size of spatial convolution. 106 | :param c_in: int, size of input channel. 107 | :param c_out: int, size of output channel. 108 | :return: tensor, [batch_size, time_step, n_route, c_out]. 109 | ''' 110 | _, T, n, _ = x.get_shape().as_list() 111 | 112 | if c_in > c_out: 113 | # bottleneck down-sampling 114 | w_input = tf.get_variable('ws_input', shape=[1, 1, c_in, c_out], dtype=tf.float32) 115 | tf.add_to_collection(name='weight_decay', value=tf.nn.l2_loss(w_input)) 116 | x_input = tf.nn.conv2d(x, w_input, strides=[1, 1, 1, 1], padding='SAME') 117 | elif c_in < c_out: 118 | # if the size of input channel is less than the output, 119 | # padding x to the same size of output channel. 120 | # Note, _.get_shape() cannot convert a partially known TensorShape to a Tensor. 121 | x_input = tf.concat([x, tf.zeros([tf.shape(x)[0], T, n, c_out - c_in])], axis=3) 122 | else: 123 | x_input = x 124 | 125 | ws = tf.get_variable(name='ws', shape=[Ks * c_in, c_out], dtype=tf.float32) 126 | tf.add_to_collection(name='weight_decay', value=tf.nn.l2_loss(ws)) 127 | variable_summaries(ws, 'theta') 128 | bs = tf.get_variable(name='bs', initializer=tf.zeros([c_out]), dtype=tf.float32) 129 | # x -> [batch_size*time_step, n_route, c_in] -> [batch_size*time_step, n_route, c_out] 130 | x_gconv = gconv(tf.reshape(x, [-1, n, c_in]), ws, Ks, c_in, c_out) + bs 131 | # x_g -> [batch_size, time_step, n_route, c_out] 132 | x_gc = tf.reshape(x_gconv, [-1, T, n, c_out]) 133 | return tf.nn.relu(x_gc[:, :, :, 0:c_out] + x_input) 134 | 135 | 136 | def st_conv_block(x, Ks, Kt, channels, scope, keep_prob, act_func='GLU'): 137 | ''' 138 | Spatio-temporal convolutional block, which contains two temporal gated convolution layers 139 | and one spatial graph convolution layer in the middle. 140 | :param x: tensor, batch_size, time_step, n_route, c_in]. 141 | :param Ks: int, kernel size of spatial convolution. 142 | :param Kt: int, kernel size of temporal convolution. 143 | :param channels: list, channel configs of a single st_conv block. 144 | :param scope: str, variable scope. 145 | :param keep_prob: placeholder, prob of dropout. 146 | :param act_func: str, activation function. 147 | :return: tensor, [batch_size, time_step, n_route, c_out]. 148 | ''' 149 | c_si, c_t, c_oo = channels 150 | 151 | with tf.variable_scope(f'stn_block_{scope}_in'): 152 | x_s = temporal_conv_layer(x, Kt, c_si, c_t, act_func=act_func) 153 | x_t = spatio_conv_layer(x_s, Ks, c_t, c_t) 154 | with tf.variable_scope(f'stn_block_{scope}_out'): 155 | x_o = temporal_conv_layer(x_t, Kt, c_t, c_oo) 156 | x_ln = layer_norm(x_o, f'layer_norm_{scope}') 157 | return tf.nn.dropout(x_ln, keep_prob) 158 | 159 | 160 | def fully_con_layer(x, n, channel, scope): 161 | ''' 162 | Fully connected layer: maps multi-channels to one. 163 | :param x: tensor, [batch_size, 1, n_route, channel]. 164 | :param n: int, number of route / size of graph. 165 | :param channel: channel size of input x. 166 | :param scope: str, variable scope. 167 | #:return: tensor, [batch_size, 1, n_route, 1]. 168 | :return: tensor, [batch_size, 1, n_route, 2]. 169 | ''' 170 | w = tf.get_variable(name=f'w_{scope}', shape=[1, 1, channel, 2], dtype=tf.float32) 171 | tf.add_to_collection(name='weight_decay', value=tf.nn.l2_loss(w)) 172 | b = tf.get_variable(name=f'b_{scope}', initializer=tf.zeros([n, 2]), dtype=tf.float32) 173 | return tf.nn.conv2d(x, w, strides=[1, 1, 1, 1], padding='SAME') + b 174 | 175 | 176 | def output_layer(x, T, scope, act_func='GLU'): 177 | ''' 178 | Output layer: temporal convolution layers attach with one fully connected layer, 179 | which map outputs of the last st_conv block to a single-step prediction. 180 | :param x: tensor, [batch_size, time_step, n_route, channel]. 181 | :param T: int, kernel size of temporal convolution. 182 | :param scope: str, variable scope. 183 | :param act_func: str, activation function. 184 | :return: tensor, [batch_size, 1, n_route, 1]. 185 | ''' 186 | _, _, n, channel = x.get_shape().as_list() 187 | 188 | # maps multi-steps to one. 189 | with tf.variable_scope(f'{scope}_in'): 190 | x_i = temporal_conv_layer(x, T, channel, channel, act_func=act_func) 191 | x_ln = layer_norm(x_i, f'layer_norm_{scope}') 192 | with tf.variable_scope(f'{scope}_out'): 193 | x_o = temporal_conv_layer(x_ln, 1, channel, channel, act_func='sigmoid') 194 | # maps multi-channels to one. 195 | x_fc = fully_con_layer(x_o, n, channel, scope) 196 | return x_fc 197 | 198 | 199 | def variable_summaries(var, v_name): 200 | ''' 201 | Attach summaries to a Tensor (for TensorBoard visualization). 202 | Ref: https://zhuanlan.zhihu.com/p/33178205 203 | :param var: tf.Variable(). 204 | :param v_name: str, name of the variable. 205 | ''' 206 | with tf.name_scope('summaries'): 207 | mean = tf.reduce_mean(var) 208 | tf.summary.scalar(f'mean_{v_name}', mean) 209 | 210 | with tf.name_scope(f'stddev_{v_name}'): 211 | stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean))) 212 | tf.summary.scalar(f'stddev_{v_name}', stddev) 213 | 214 | tf.summary.scalar(f'max_{v_name}', tf.reduce_max(var)) 215 | tf.summary.scalar(f'min_{v_name}', tf.reduce_min(var)) 216 | 217 | tf.summary.histogram(f'histogram_{v_name}', var) 218 | -------------------------------------------------------------------------------- /citibike.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | import numpy as np 4 | import tensorflow as tf 5 | #from gensim.models import Word2Vec 6 | from model.FC_LSTM import FC_LSTM 7 | from model.FC_GRU import FC_GRU 8 | from model.GCN import GCN 9 | from model.Coupled_GCN import Coupled_GCN 10 | from model.flow_GCN import flow_GCN 11 | from solver import ModelSolver 12 | from preprocessing import * 13 | from utils import * 14 | from dataloader import * 15 | # import scipy.io as sio 16 | 17 | 18 | def main(): 19 | parse = argparse.ArgumentParser() 20 | # ---------- environment setting: which gpu ------- 21 | parse.add_argument('-gpu', '--gpu', type=str, default='0', help='which gpu to use: 0 or 1') 22 | parse.add_argument('-folder_name', '--folder_name', type=str, default='datasets/citibike-data/data/') 23 | parse.add_argument('-output_folder_name', '--output_folder_name', type=str, default='output/citibike-data/data/') 24 | # ---------- input/output settings ------- 25 | parse.add_argument('-input_steps', '--input_steps', type=int, default=6, 26 | help='number of input steps') 27 | # ---------- model ---------- 28 | parse.add_argument('-model', '--model', type=str, default='GCN', help='model: DyST, GCN, AttGCN') 29 | parse.add_argument('-num_layers', '--num_layers', type=int, default=2, help='number of layers in model') 30 | parse.add_argument('-num_units', '--num_units', type=int, default=64, help='dim of hidden states') 31 | parse.add_argument('-trained_adj_mx', '--trained_adj_mx', type=int, default=0, help='if training adjacent matrix') 32 | parse.add_argument('-filter_type', '--filter_type', type=str, default='dual_random_walk', help='laplacian, random_walk, or dual_random_walk') 33 | parse.add_argument('-delta', '--delta', type=int, default=1e7, help='delta to calculate rescaled weighted matrix') 34 | parse.add_argument('-epsilon', '--epsilon', type=float, default=0.8, help='epsilon to calculate rescaled weighted matrix') 35 | # 36 | parse.add_argument('-dy_temporal', '--dy_temporal', type=int, default=0, 37 | help='whether to use temporal attention module before output layer') 38 | parse.add_argument('-multi_loss', '--multi_loss', type=int, default=0, 39 | help='whether to only consider last prediction into loss function.') 40 | parse.add_argument('-att_units', '--att_units', type=int, default=64, help='dim of hidden states') 41 | # 42 | parse.add_argument('-dy_adj', '--dy_adj', type=int, default=1, 43 | help='whether to use dynamic adjacent matrix for lower feature extraction layer') 44 | parse.add_argument('-dy_filter', '--dy_filter', type=int, default=0, 45 | help='whether to use dynamic filter generate region-specific filter ') 46 | #parse.add_argument('-att_dynamic_adj', '--att_dynamic_adj', type=int, default=1, help='whether to use dynamic adjacent matrix in attention parts') 47 | parse.add_argument('-model_save', '--model_save', type=str, default='gcn', help='folder name to save model') 48 | parse.add_argument('-pretrained_model', '--pretrained_model_path', type=str, default=None, 49 | help='path to the pretrained model') 50 | # ---------- params for CNN ------------ 51 | parse.add_argument('-num_filters', '--num_filters', type=int, 52 | default=32, help='number of filters in CNN') 53 | parse.add_argument('-pooling_units', '--pooling_units', type=int, 54 | default=64, help='number of pooling units') 55 | parse.add_argument('-dropout_keep_prob', '--dropout_keep_prob', type=float, 56 | default=0.5, help='keep probability in dropout layer') 57 | # ---------- training parameters -------- 58 | parse.add_argument('-n_epochs', '--n_epochs', type=int, default=20, help='number of epochs') 59 | parse.add_argument('-batch_size', '--batch_size', type=int, default=8, help='batch size for training') 60 | parse.add_argument('-show_batches', '--show_batches', type=int, 61 | default=100, help='show how many batches have been processed.') 62 | parse.add_argument('-lr', '--learning_rate', type=float, default=0.0002, help='learning rate') 63 | parse.add_argument('-update_rule', '--update_rule', type=str, default='adam', help='update rule') 64 | # ---------- train or predict ------- 65 | parse.add_argument('-train', '--train', type=int, default=1, help='whether to train') 66 | parse.add_argument('-test', '--test', type=int, default=0, help='if test') 67 | # 68 | args = parse.parse_args() 69 | 70 | os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu 71 | print('load train, test data...') 72 | # train: 20140401 - 20140831 73 | # validate: 20140901 - 20140910 74 | # test: 20140911 - 20140930 75 | split = [3672, 240, 480] 76 | #split = [3912, 480] 77 | data, train_data, val_data, test_data = load_npy_data( 78 | filename=[args.folder_name+'d_station.npy', args.folder_name+'p_station.npy'], split=split) 79 | # data: [num, station_num, 2] 80 | #f_data, train_f_data, val_f_data, test_f_data = load_pkl_data(args.folder_name + 'f_data_list.pkl', split=split) 81 | f_data, train_f_data, val_f_data, test_f_data = load_npy_data(filename=[args.folder_name + 'citibike_flow_data.npy'], split=split) 82 | print(len(f_data)) 83 | print('preprocess train/val/test flow data...') 84 | #f_preprocessing = StandardScaler() 85 | f_preprocessing = MinMaxNormalization01() 86 | f_preprocessing.fit(train_f_data) 87 | train_f_data = f_preprocessing.transform(train_f_data) 88 | if val_f_data is not None: 89 | val_f_data = f_preprocessing.transform(val_f_data) 90 | test_f_data = f_preprocessing.transform(test_f_data) 91 | print('preprocess train/val/test data...') 92 | pre_process = MinMaxNormalization01() 93 | #pre_process = StandardScaler() 94 | pre_process.fit(train_data) 95 | train_data = pre_process.transform(train_data) 96 | if val_data is not None: 97 | val_data = pre_process.transform(val_data) 98 | test_data = pre_process.transform(test_data) 99 | # 100 | num_station = data.shape[1] 101 | print('number of station: %d' % num_station) 102 | # 103 | train_loader = DataLoader_graph(train_data, train_f_data, 104 | args.input_steps, flow_format='identity') 105 | if val_data is not None: 106 | val_loader = DataLoader_graph(val_data, val_f_data, 107 | args.input_steps, flow_format='identity') 108 | else: 109 | val_loader = None 110 | test_loader = DataLoader_graph(test_data, test_f_data, 111 | args.input_steps, flow_format='identity') 112 | # f_adj_mx = None 113 | if os.path.isfile(args.folder_name + 'f_adj_mx.npy'): 114 | f_adj_mx = np.load(args.folder_name + 'f_adj_mx.npy') 115 | else: 116 | f_adj_mx = train_loader.get_flow_adj_mx() 117 | np.save(args.folder_name + 'f_adj_mx.npy', f_adj_mx) 118 | # 119 | # 120 | if args.filter_type == 'laplacian': 121 | w = np.load(args.folder_name + 'w.npy') 122 | # w = np.array(w, dtype=np.float32) 123 | W = get_rescaled_W(w, delta=args.delta, epsilon=args.epsilon) 124 | # Calculate graph kernel 125 | L = scaled_laplacian(W) 126 | # 127 | f_adj_mx = L 128 | 129 | if args.model == 'FC_LSTM': 130 | model = FC_LSTM(num_station, args.input_steps, 131 | num_layers=args.num_layers, num_units=args.num_units, 132 | batch_size=args.batch_size) 133 | if args.model == 'FC_GRU': 134 | model = FC_GRU(num_station, args.input_steps, 135 | num_layers=args.num_layers, num_units=args.num_units, 136 | batch_size=args.batch_size) 137 | if args.model == 'GCN': 138 | model = GCN(num_station, args.input_steps, 139 | num_layers=args.num_layers, num_units=args.num_units, 140 | dy_adj=args.dy_adj, dy_filter=args.dy_filter, 141 | f_adj_mx=f_adj_mx, trained_adj_mx=args.trained_adj_mx, 142 | filter_type=args.filter_type, 143 | batch_size=args.batch_size) 144 | if args.model == 'flow_GCN': 145 | model = flow_GCN(num_station, args.input_steps, 146 | num_layers=args.num_layers, num_units=args.num_units, 147 | f_adj_mx=f_adj_mx, trained_adj_mx=args.trained_adj_mx, 148 | filter_type=args.filter_type, 149 | batch_size=args.batch_size) 150 | if args.model == 'Coupled_GCN': 151 | model = Coupled_GCN(num_station, args.input_steps, 152 | num_layers=args.num_layers, num_units=args.num_units, 153 | f_adj_mx=f_adj_mx, trained_adj_mx=args.trained_adj_mx, 154 | filter_type=args.filter_type, 155 | dy_temporal=args.dy_temporal, att_units=args.att_units, 156 | multi_loss=args.multi_loss, 157 | batch_size=args.batch_size) 158 | # 159 | model_path = os.path.join(args.output_folder_name, 'model_save', args.model_save) 160 | if not os.path.exists(model_path): 161 | os.makedirs(model_path) 162 | #model_path = os.path.join(args.folder_name, 'model_save', args.model_save) 163 | solver = ModelSolver(model, train_loader, val_loader, test_loader, pre_process, 164 | batch_size=args.batch_size, 165 | show_batches=args.show_batches, 166 | n_epochs=args.n_epochs, 167 | pretrained_model=args.pretrained_model_path, 168 | update_rule=args.update_rule, 169 | learning_rate=args.learning_rate, 170 | model_path=model_path, 171 | ) 172 | results_path = os.path.join(model_path, 'results') 173 | if not os.path.exists(results_path): 174 | os.makedirs(results_path) 175 | if args.train: 176 | print('==================== begin training ======================') 177 | test_target, test_prediction = solver.train(os.path.join(model_path, 'out')) 178 | np.save(os.path.join(results_path, 'test_target.npy'), test_target) 179 | np.save(os.path.join(results_path, 'test_prediction.npy'), test_prediction) 180 | if args.test: 181 | print('==================== begin test ==========================') 182 | test_target, test_prediction = solver.test() 183 | np.save(os.path.join(results_path, 'test_target.npy'), test_target) 184 | np.save(os.path.join(results_path, 'test_prediction.npy'), test_prediction) 185 | 186 | 187 | if __name__ == "__main__": 188 | main() 189 | -------------------------------------------------------------------------------- /taxi_graph.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | import numpy as np 4 | import tensorflow as tf 5 | #from gensim.models import Word2Vec 6 | #from model.FC_LSTM import FC_LSTM 7 | from model.FC_GRU import FC_GRU 8 | from model.FC_LSTM import FC_LSTM 9 | from model.GCN import GCN 10 | from model.ConvGRU import ConvGRU 11 | from model.ConvLSTM import ConvLSTM 12 | #from model.flow_ConvGRU import flow_ConvGRU 13 | from model.flow_ConvGRU_2 import flow_ConvGRU_2 14 | from model.Stack_ConvGRU import Stack_ConvGRU 15 | from model.Coupled_ConvGRU import CoupledConvGRU 16 | from solver import ModelSolver 17 | from preprocessing import * 18 | from utils import * 19 | from dataloader import * 20 | # import scipy.io as sio 21 | 22 | 23 | def main(): 24 | parse = argparse.ArgumentParser() 25 | # ---------- environment setting: which gpu ------- 26 | parse.add_argument('-gpu', '--gpu', type=str, default='0', help='which gpu to use: 0 or 1') 27 | parse.add_argument('-folder_name', '--folder_name', type=str, default='datasets/taxi-data/graph-data/') 28 | parse.add_argument('-output_folder_name', '--output_folder_name', type=str, default='output/taxi-data/graph-data/') 29 | # ---------- input/output settings ------- 30 | parse.add_argument('-input_steps', '--input_steps', type=int, default=6, 31 | help='number of input steps') 32 | # ---------- model ---------- 33 | parse.add_argument('-model', '--model', type=str, default='GCN', help='model: GCN, ConvLSTM, flow_ConvLSTM') 34 | parse.add_argument('-num_layers', '--num_layers', type=int, default=2, help='number of layers in model') 35 | parse.add_argument('-num_units', '--num_units', type=int, default=64, help='dim of hidden states') 36 | parse.add_argument('-kernel_size', '--kernel_size', type=int, default=3, help='kernel size in convolutional operations') 37 | # 38 | parse.add_argument('-dy_temporal', '--dy_temporal', type=int, default=0, 39 | help='whether to use temporal attention module before output layer') 40 | parse.add_argument('-multi_loss', '--multi_loss', type=int, default=0, 41 | help='whether to only consider last prediction into loss function.') 42 | parse.add_argument('-att_units', '--att_units', type=int, default=64, help='dim of hidden states') 43 | # 44 | parse.add_argument('-dy_adj', '--dy_adj', type=int, default=1, 45 | help='whether to use dynamic adjacent matrix for lower feature extraction layer') 46 | parse.add_argument('-dy_filter', '--dy_filter', type=int, default=0, 47 | help='whether to use dynamic filter generate region-specific filter ') 48 | #parse.add_argument('-att_dynamic_adj', '--att_dynamic_adj', type=int, default=0, help='whether to use dynamic adjacent matrix in attention parts') 49 | # 50 | parse.add_argument('-model_save', '--model_save', type=str, default='gcn', help='folder name to save model') 51 | parse.add_argument('-pretrained_model', '--pretrained_model_path', type=str, default=None, 52 | help='path to the pretrained model') 53 | # ---------- params for CNN ------------ 54 | parse.add_argument('-num_filters', '--num_filters', type=int, 55 | default=32, help='number of filters in CNN') 56 | parse.add_argument('-pooling_units', '--pooling_units', type=int, 57 | default=64, help='number of pooling units') 58 | parse.add_argument('-dropout_keep_prob', '--dropout_keep_prob', type=float, 59 | default=0.5, help='keep probability in dropout layer') 60 | # ---------- training parameters -------- 61 | parse.add_argument('-n_epochs', '--n_epochs', type=int, default=20, help='number of epochs') 62 | parse.add_argument('-batch_size', '--batch_size', type=int, default=8, help='batch size for training') 63 | parse.add_argument('-show_batches', '--show_batches', type=int, 64 | default=100, help='show how many batches have been processed.') 65 | parse.add_argument('-lr', '--learning_rate', type=float, default=0.0002, help='learning rate') 66 | parse.add_argument('-update_rule', '--update_rule', type=str, default='adam', help='update rule') 67 | # ---------- train or predict ------- 68 | parse.add_argument('-train', '--train', type=int, default=1, help='whether to train') 69 | parse.add_argument('-test', '--test', type=int, default=0, help='if test') 70 | # 71 | args = parse.parse_args() 72 | 73 | os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu 74 | print('load train, test data...') 75 | # train: 20140101 - 20150430 76 | # validate: 20150501 - 20150531 77 | # test: 20150601 - 20150630 78 | split = [11640, 744, 720] 79 | data, train_data, val_data, test_data = load_npy_data(filename=[args.folder_name + 'nyc_taxi_data.npy'], split=split) 80 | # data: [num, station_num, 2] 81 | print(data.shape) 82 | # 83 | if 'GCN' in args.model or 'FC' in args.model: 84 | dataloader = DataLoader_graph 85 | else: 86 | data = np.reshape(data, (-1, 20, 10, 2)) 87 | train_data = np.reshape(train_data, (-1, 20, 10, 2)) 88 | val_data = np.reshape(val_data, (-1, 20, 10, 2)) 89 | test_data = np.reshape(test_data, (-1, 20, 10, 2)) 90 | # data: [num, height, width, 2] 91 | print(data.shape) 92 | # 93 | dataloader = DataLoader_map 94 | # 95 | map_size = data.shape[1:-1] 96 | input_dim = data.shape[-1] 97 | num_station = np.prod(data.shape[1:-1]) 98 | # 99 | f_data, train_f_data, val_f_data, test_f_data = load_npy_data([args.folder_name + 'nyc_taxi_flow_in.npy'], split=split) 100 | print(len(f_data)) 101 | print('preprocess train/val/test flow data...') 102 | #f_preprocessing = StandardScaler() 103 | f_preprocessing = MinMaxNormalization01() 104 | f_preprocessing.fit(train_f_data) 105 | train_f_data = f_preprocessing.transform(train_f_data) 106 | val_f_data = f_preprocessing.transform(val_f_data) 107 | test_f_data = f_preprocessing.transform(test_f_data) 108 | print('preprocess train/val/test data...') 109 | # pre_process = StandardScaler() 110 | pre_process = MinMaxNormalization01() 111 | pre_process.fit(train_data) 112 | train_data = pre_process.transform(train_data) 113 | val_data = pre_process.transform(val_data) 114 | test_data = pre_process.transform(test_data) 115 | # 116 | 117 | print('number of station: %d' % num_station) 118 | # 119 | train_loader = dataloader(train_data, train_f_data, 120 | args.input_steps, flow_format='identity') 121 | val_loader = dataloader(val_data, val_f_data, 122 | args.input_steps, flow_format='identity') 123 | test_loader = dataloader(test_data, test_f_data, 124 | args.input_steps, flow_format='identity') 125 | # f_adj_mx = None 126 | if os.path.isfile(args.folder_name + 'f_adj_mx.npy'): 127 | f_adj_mx = np.load(args.folder_name + 'f_adj_mx.npy') 128 | else: 129 | f_adj_mx = train_loader.get_flow_adj_mx() 130 | np.save(args.folder_name + 'f_adj_mx.npy', f_adj_mx) 131 | 132 | if args.model == 'FC_LSTM': 133 | model = FC_LSTM(num_station, args.input_steps, 134 | num_layers=args.num_layers, num_units=args.num_units, 135 | batch_size=args.batch_size) 136 | if args.model == 'FC_GRU': 137 | model = FC_GRU(num_station, args.input_steps, 138 | num_layers=args.num_layers, num_units=args.num_units, 139 | batch_size=args.batch_size) 140 | if args.model == 'GCN': 141 | model = GCN(num_station, args.input_steps, 142 | num_layers=args.num_layers, num_units=args.num_units, 143 | dy_adj=args.dy_adj, dy_filter=args.dy_filter, 144 | f_adj_mx=f_adj_mx, 145 | batch_size=args.batch_size) 146 | if args.model == 'ConvGRU': 147 | model = ConvGRU(input_shape=[map_size[0], map_size[1], input_dim], input_steps=args.input_steps, 148 | num_layers=args.num_layers, num_units=args.num_units, kernel_shape=[args.kernel_size, args.kernel_size], 149 | batch_size=args.batch_size) 150 | if args.model == 'ConvLSTM': 151 | model = ConvLSTM(input_shape=[map_size[0], map_size[1], input_dim], input_steps=args.input_steps, 152 | num_layers=args.num_layers, num_units=args.num_units, kernel_shape=[args.kernel_size, args.kernel_size], 153 | batch_size=args.batch_size) 154 | # if args.model == 'flow_ConvGRU': 155 | # model = flow_ConvGRU(input_shape=[20, 10, input_dim], input_steps=args.input_steps, 156 | # num_layers=args.num_layers, num_units=args.num_units,kernel_shape=[args.kernel_size, args.kernel_size], 157 | # f_adj_mx=f_adj_mx, 158 | # batch_size=args.batch_size) 159 | if args.model == 'Coupled_ConvGRU': 160 | model = CoupledConvGRU(input_shape=[20, 10, input_dim], input_steps=args.input_steps, 161 | num_layers=args.num_layers, num_units=args.num_units, kernel_shape=[args.kernel_size, args.kernel_size], 162 | dy_temporal=args.dy_temporal, att_units=args.att_units, 163 | multi_loss=args.multi_loss, 164 | batch_size=args.batch_size) 165 | 166 | # 167 | model_path = os.path.join(args.output_folder_name, 'model_save', args.model_save) 168 | if not os.path.exists(model_path): 169 | os.makedirs(model_path) 170 | #model_path = os.path.join(args.folder_name, 'model_save', args.model_save) 171 | solver = ModelSolver(model, train_loader, val_loader, test_loader, pre_process, 172 | batch_size=args.batch_size, 173 | show_batches=args.show_batches, 174 | n_epochs=args.n_epochs, 175 | pretrained_model=args.pretrained_model_path, 176 | update_rule=args.update_rule, 177 | learning_rate=args.learning_rate, 178 | model_path=model_path, 179 | ) 180 | results_path = os.path.join(model_path, 'results') 181 | if not os.path.exists(results_path): 182 | os.makedirs(results_path) 183 | if args.train: 184 | print('==================== begin training ======================') 185 | test_target, test_prediction = solver.train(os.path.join(model_path, 'out')) 186 | np.save(os.path.join(results_path, 'test_target.npy'), test_target) 187 | np.save(os.path.join(results_path, 'test_prediction.npy'), test_prediction) 188 | if args.test: 189 | print('==================== begin test ==========================') 190 | test_target, test_prediction = solver.test() 191 | np.save(os.path.join(results_path, 'test_target.npy'), test_target) 192 | np.save(os.path.join(results_path, 'test_prediction.npy'), test_prediction) 193 | 194 | 195 | if __name__ == "__main__": 196 | main() 197 | -------------------------------------------------------------------------------- /model/Coupled_ConvGRU.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import numpy as np 3 | import pickle 4 | import tensorflow as tf 5 | 6 | sys.path.append('./util/') 7 | from utils import * 8 | from model.dcrnn_cell import DCGRUCell 9 | from model.coupled_convgru_cell import Coupled_Conv2DGRUCell 10 | 11 | 12 | class CoupledConvGRU(): 13 | def __init__(self, input_shape=[20,10,2], input_steps=6, 14 | num_layers=2, num_units=64, kernel_shape=[3,3], 15 | dy_temporal=0, att_units=64, 16 | dy_adj=0, 17 | dy_filter=0, 18 | multi_loss=0, 19 | batch_size=32): 20 | self.input_shape = input_shape 21 | self.input_steps = input_steps 22 | self.num_layers = num_layers 23 | self.num_units = num_units 24 | self.kernel_shape = kernel_shape 25 | # 26 | self.num_nodes = np.prod(self.input_shape[:-1]) 27 | self.dy_temporal = dy_temporal 28 | self.att_units = att_units 29 | # self.dy_adj = dy_adj 30 | # self.dy_filter = dy_filter 31 | self.multi_loss = multi_loss 32 | self.batch_size = batch_size 33 | 34 | self.weight_initializer = tf.contrib.layers.xavier_initializer() 35 | self.const_initializer = tf.constant_initializer() 36 | 37 | 38 | first_cell = Coupled_Conv2DGRUCell(num_units=self.num_units, 39 | input_shape=self.input_shape, 40 | kernel_shape=self.kernel_shape, 41 | num_proj=None, 42 | input_dim=self.input_shape[-1], 43 | output_dy_adj=1) 44 | cell = Coupled_Conv2DGRUCell(num_units=self.num_units, 45 | input_shape=[self.input_shape[0], self.input_shape[1], self.num_units], 46 | kernel_shape=self.kernel_shape, 47 | num_proj=None, 48 | input_dim=self.num_units, 49 | output_dy_adj=1) 50 | last_cell = Coupled_Conv2DGRUCell(num_units=self.num_units, 51 | input_shape=[self.input_shape[0], self.input_shape[1], self.num_units], 52 | kernel_shape=self.kernel_shape, 53 | num_proj=None, 54 | input_dim=self.num_units, 55 | output_dy_adj=0) 56 | ## for only one layer 57 | one_cell = Coupled_Conv2DGRUCell(num_units=self.num_units, 58 | input_shape=self.input_shape, 59 | kernel_shape=self.kernel_shape, 60 | num_proj=None, 61 | input_dim=self.input_shape[-1], 62 | output_dy_adj=0) 63 | 64 | if num_layers > 2: 65 | cells = [first_cell] + [cell] * (num_layers-2) + [last_cell] 66 | else: 67 | cells = [first_cell, last_cell] 68 | if num_layers == 1: 69 | cells = [one_cell] 70 | 71 | self.cells = tf.contrib.rnn.MultiRNNCell(cells, state_is_tuple=True) 72 | 73 | self.x = tf.placeholder(tf.float32, [self.batch_size, self.input_steps, self.input_shape[0], self.input_shape[1], self.input_shape[2]]) 74 | self.f = tf.placeholder(tf.float32, 75 | [self.batch_size, self.input_steps, self.input_shape[0] * self.input_shape[1], 76 | self.input_shape[0] * self.input_shape[1]]) 77 | self.y = tf.placeholder(tf.float32, [self.batch_size, self.input_steps, self.input_shape[0], self.input_shape[1], self.input_shape[2]]) 78 | 79 | 80 | 81 | def build_easy_model(self): 82 | x = tf.transpose(tf.reshape(self.x, (self.batch_size, self.input_steps, -1)), [1, 0, 2]) 83 | #inputs = tf.unstack(x, axis=0) 84 | f_all = tf.transpose(tf.reshape(self.f, (self.batch_size, self.input_steps, -1)), [1, 0, 2]) 85 | inputs = tf.concat([x, f_all], axis=-1) 86 | inputs = tf.unstack(inputs, axis=0) 87 | # 88 | outputs, _ = tf.contrib.rnn.static_rnn(self.cells, inputs, dtype=tf.float32) 89 | outputs = tf.stack(outputs) 90 | # temporal attention 91 | outputs = tf.reshape(outputs, (self.input_steps, self.batch_size, self.input_shape[0], self.input_shape[1], -1)) 92 | # outputs: [input_steps, batch_size, -, -, -] 93 | if self.multi_loss: 94 | if self.dy_temporal: 95 | with tf.variable_scope('temporal_attention', reuse=tf.AUTO_REUSE): 96 | h_states = tf.transpose(outputs, (1, 0, 2, 3, 4)) 97 | att_states = [] 98 | for t in range(self.input_steps): 99 | att_state, _ = self.temporal_attention_layer(outputs[t], h_states, self.att_units, reuse=tf.AUTO_REUSE) 100 | att_states.append(att_state) 101 | att_states = tf.stack(att_states) 102 | outputs = tf.concat([outputs, att_states], -1) 103 | # projection 104 | outputs = tf.layers.dense(outputs, units=self.input_shape[-1], activation=None, kernel_initializer=self.weight_initializer) 105 | outputs = tf.transpose(outputs, [1, 0, 2, 3, 4]) 106 | loss = 2 * tf.nn.l2_loss(self.y - outputs) 107 | return outputs, loss 108 | else: 109 | if self.dy_temporal: 110 | with tf.variable_scope('temporal_attention', reuse=tf.AUTO_REUSE): 111 | h_states = tf.transpose(outputs[:-1], (1, 0, 2, 3, 4)) 112 | att_states, _ = self.temporal_attention_layer(outputs[-1], h_states, self.att_units, 113 | reuse=tf.AUTO_REUSE) 114 | output = tf.concat([outputs[-1], att_states], -1) 115 | else: 116 | output = outputs[-1] 117 | # projection 118 | output = tf.layers.dense(output, units=self.input_shape[-1], activation=None, 119 | kernel_initializer=self.weight_initializer) 120 | loss = 2 * tf.nn.l2_loss(self.y[:, -1, :, :, :] - output) 121 | return tf.expand_dims(output, 1), loss 122 | 123 | ''' 124 | # single loss 125 | def build_easy_model(self): 126 | x = tf.transpose(tf.reshape(self.x, (self.batch_size, self.input_steps, -1)), [1, 0, 2]) 127 | #inputs = tf.unstack(x, axis=0) 128 | f_all = tf.transpose(tf.reshape(self.f, (self.batch_size, self.input_steps, -1)), [1, 0, 2]) 129 | inputs = tf.concat([x, f_all], axis=-1) 130 | inputs = tf.unstack(inputs, axis=0) 131 | # 132 | outputs, _ = tf.contrib.rnn.static_rnn(self.cells, inputs, dtype=tf.float32) 133 | outputs = tf.stack(outputs) 134 | # temporal attention 135 | outputs = tf.reshape(outputs, (self.input_steps, self.batch_size, self.input_shape[0], self.input_shape[1], -1)) 136 | # outputs: [input_steps, batch_size, -, -, -] 137 | if self.dy_temporal: 138 | with tf.variable_scope('temporal_attention', reuse=tf.AUTO_REUSE): 139 | h_states = tf.transpose(outputs[:-1], (1,0,2,3,4)) 140 | att_states, _ = self.temporal_attention_layer(outputs[-1], h_states, self.att_units, reuse=tf.AUTO_REUSE) 141 | output = tf.concat([outputs[-1], att_states], -1) 142 | else: 143 | output = outputs[-1] 144 | # projection 145 | output = tf.layers.dense(output, units=self.input_shape[-1], activation=None, kernel_initializer=self.weight_initializer) 146 | loss = 2 * tf.nn.l2_loss(self.y[:, -1, :, :, :] - output) 147 | return tf.expand_dims(output, 1), loss 148 | ''' 149 | 150 | def temporal_attention_layer(self, o_state, h_states, att_units, reuse=True): 151 | # o_state: [batch_size, row, col, channel] 152 | # h_state: [batch_size, input_steps, row, col, channel] 153 | o_shape = o_state.get_shape().as_list() 154 | h_shape = h_states.get_shape().as_list() 155 | with tf.variable_scope('att', reuse=reuse): 156 | with tf.variable_scope('att_o_state', reuse=reuse): 157 | w = tf.get_variable('att_o_w', [np.prod(o_shape[1:]), att_units], initializer=self.weight_initializer) 158 | o_att = tf.matmul(tf.reshape(o_state, (-1, np.prod(o_shape[1:]))), w) 159 | # o_att: [batch_size, att_units] 160 | with tf.variable_scope('att_h_state', reuse=reuse): 161 | w = tf.get_variable('att_h_w', [np.prod(h_shape[2:]), att_units], initializer=self.weight_initializer) 162 | h_att = tf.matmul(tf.reshape(h_states, [-1, np.prod(h_shape[2:])]), w) 163 | # h_att: [batch_size*input_steps, att_units] 164 | h_att = tf.reshape(h_att, [-1, h_shape[1], att_units]) 165 | # encoder_state_att: [batch_size, input_steps, att_units] 166 | b = tf.get_variable('att_b', [att_units], initializer=self.const_initializer) 167 | o_h_att_plus = tf.nn.relu(h_att + tf.expand_dims(o_att, 1) + b) 168 | mlp_w = tf.get_variable('mlp_w', [att_units, 1], initializer=self.weight_initializer) 169 | out_att = tf.reshape(tf.matmul(tf.reshape(o_h_att_plus, [-1, att_units]), mlp_w), 170 | [-1, h_shape[1]]) 171 | # out_att: [batch_size, input_steps] 172 | alpha = tf.nn.softmax(out_att) 173 | context = tf.reduce_sum(tf.reshape(h_states, (-1, h_shape[1], np.prod(h_shape[2:]))) * tf.expand_dims(alpha, -1), 174 | 1, name='context') 175 | att_context = tf.reshape(context, 176 | [-1, h_shape[2], h_shape[3], h_shape[4]]) 177 | # att_context: [batch_size, row, col, channel] 178 | return att_context, alpha 179 | 180 | ''' 181 | def build_easy_model(self): 182 | x = tf.transpose(tf.reshape(self.x, (self.batch_size, self.input_steps, -1)), [1, 0, 2]) 183 | #inputs = tf.unstack(x, axis=0) 184 | f_all = tf.transpose(tf.reshape(self.f, (self.batch_size, self.input_steps, -1)), [1, 0, 2]) 185 | inputs = tf.concat([x, f_all], axis=-1) 186 | inputs = tf.unstack(inputs, axis=0) 187 | # 188 | outputs, _ = tf.contrib.rnn.static_rnn(self.cells, inputs, dtype=tf.float32) 189 | outputs = tf.stack(outputs) 190 | # 191 | # projection 192 | outputs = tf.layers.dense(tf.reshape(outputs, (-1, self.num_units)), units=self.input_shape[-1], 193 | activation=None, kernel_initializer=self.weight_initializer) 194 | # 195 | outputs = tf.reshape(outputs, (self.input_steps, self.batch_size, self.input_shape[0], self.input_shape[1], -1)) 196 | outputs = tf.transpose(outputs, [1, 0, 2, 3, 4]) 197 | loss = 2 * tf.nn.l2_loss(self.y - outputs) 198 | return outputs, loss 199 | ''' 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | --------------------------------------------------------------------------------