├── .gitattributes ├── .gitignore ├── README.md ├── data ├── energydata_complete.txt └── nasdaq100_padding.csv ├── models ├── __init__.py ├── model_runner.py ├── models.py └── optimize.py ├── runEnergy.py ├── runNASDAQ.py ├── runTraffic.py └── utils ├── __init__.py ├── _libs_.py ├── args.py └── data_io.py /.gitattributes: -------------------------------------------------------------------------------- 1 | data/traffic.txt filter=lfs diff=lfs merge=lfs -text 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.pyc 3 | .DS_Store 4 | */.DS_Store 5 | .vscode 6 | *.sh 7 | example.py -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MLCNN for Multivariate Time Series Forecasting 2 | 3 | This repository provides the code for the paper [Towards Better Forecasting by Fusing Near and Distant Future Visions](https://arxiv.org/abs/1912.05122), accepted by AAAI 2020. 4 | 5 | ### Usage 6 | 7 | You can find the `Energy` and `NASDAQ` dataset in the `data/` folder. As For `Traffic` dataset, you can find it in [LSTNet data repository](https://github.com/laiguokun/multivariate-time-series-data/tree/master/traffic). 8 | 9 | Examples with parameter grid search to run different datasets are in `runTraffic.py`, `runEnergy.py` and `runNASDAQ.py`. 10 | 11 | ### Environment 12 | 13 | Python 3.6.7 and Pytorch 1.0.0 14 | 15 | ### Acknowledgements 16 | 17 | This multivariate time series forecasting framework was implemented based on the following two repositories: 18 | 19 | + [LSTNet](https://github.com/laiguokun/LSTNet) 20 | + [SOCNN](https://github.com/mbinkowski/nntimeseries) 21 | 22 | Some codes and design patterns are borrowed from these two excellent frameworks. -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This package implements the MCLNN model and the program for running it 3 | 4 | Author: Jiezhu Cheng 5 | Date: 17/03/2019 6 | """ -------------------------------------------------------------------------------- /models/model_runner.py: -------------------------------------------------------------------------------- 1 | """ 2 | Define ModelRunner class to run the model. 3 | """ 4 | 5 | from utils._libs_ import math, time, torch, nn, np 6 | from utils.data_io import DataGenerator 7 | from models.optimize import Optimize 8 | 9 | class ModelRunner(): 10 | def __init__(self, args, data_gen, model): 11 | """ 12 | Initialization arguments: 13 | args - (object) parameters of model 14 | data_gen - (DataGenerator object) the data generator 15 | model - (torch.nn.Module object) the model to be run 16 | """ 17 | self.args = args 18 | self.data_gen = data_gen 19 | self.model = model 20 | self.best_rmse = None 21 | self.best_rse = None 22 | self.best_mae = None 23 | self.running_times = [] 24 | self.train_losses = [] 25 | 26 | # --------------------------------------------------------------------------------------------------------------------------------------------- 27 | """ 28 | Train the model 29 | """ 30 | def train(self): 31 | self.model.train() 32 | total_loss = 0 33 | n_samples = 0 34 | 35 | for X, Y in self.data_gen.get_batches(self.data_gen.train_set[0], self.data_gen.train_set[1], self.args.batch_size, True): 36 | self.model.zero_grad() 37 | output = self.model(X) 38 | 39 | if self.data_gen.normalize_pattern == 0: 40 | loss = self.criterion(output, Y) 41 | elif self.data_gen.normalize_pattern == 1: 42 | if self.data_gen.mode == "immediate": 43 | maximums = self.data_gen.maximums.expand(output.size(0), self.data_gen.column_num) 44 | loss = self.criterion(output * maximums, Y * maximums) 45 | else: 46 | tmp_maximums = torch.unsqueeze(self.data_gen.maximums, 0) 47 | tmp_maximums = torch.unsqueeze(tmp_maximums, 1) 48 | new_maximums = tmp_maximums.expand(output.size(0), output.size(1), self.data_gen.column_num) 49 | loss = self.criterion(output * new_maximums, Y * new_maximums) 50 | else: 51 | if self.data_gen.mode == "immediate": 52 | means = self.data_gen.means.expand(output.size(0), self.data_gen.column_num) 53 | tmp_stds = self.data_gen.stds + ((self.data_gen.stds == 0).float() * 0.001) 54 | stds = tmp_stds.expand(output.size(0), self.data_gen.column_num) 55 | loss = self.criterion(output * stds + means, Y * stds + means) 56 | else: 57 | tmp_stds = self.data_gen.stds + ((self.data_gen.stds == 0).float() * 0.001) 58 | tmp_means = torch.unsqueeze(self.data_gen.means, 0) 59 | tmp_means = torch.unsqueeze(tmp_means, 1) 60 | tmp_stds = torch.unsqueeze(tmp_stds, 0) 61 | tmp_stds = torch.unsqueeze(tmp_stds, 1) 62 | new_means = tmp_means.expand(output.size(0), output.size(1), self.data_gen.column_num) 63 | new_stds = tmp_stds.expand(output.size(0), output.size(1), self.data_gen.column_num) 64 | loss = self.criterion(output * new_stds + new_means, Y * new_stds + new_means) 65 | 66 | loss.backward() 67 | grad_norm = self.optim.step() 68 | total_loss += loss.item() 69 | if self.data_gen.mode == "immediate": 70 | n_samples += (output.size(0) * self.data_gen.column_num) 71 | else: 72 | n_samples += (output.size(0) * output.size(1) * self.data_gen.column_num) 73 | 74 | return total_loss / n_samples 75 | 76 | # --------------------------------------------------------------------------------------------------------------------------------------------- 77 | """ 78 | Valid the model while training 79 | """ 80 | def evaluate(self, mode='valid'): 81 | """ 82 | Arguments: 83 | mode - (string) 'valid' or 'test' 84 | """ 85 | self.model.eval() 86 | total_loss = 0 87 | total_loss_l1 = 0 88 | n_samples = 0 89 | predict = None 90 | test = None 91 | 92 | if mode == 'valid': 93 | tmp_X = self.data_gen.valid_set[0] 94 | tmp_Y = self.data_gen.valid_set[1] 95 | elif mode == 'test': 96 | tmp_X = self.data_gen.test_set[0] 97 | tmp_Y = self.data_gen.test_set[1] 98 | else: 99 | raise Exception('invalid evaluation mode') 100 | 101 | for X, Y in self.data_gen.get_batches(tmp_X, tmp_Y, self.args.batch_size, False): 102 | output = self.model(X) 103 | 104 | if self.data_gen.normalize_pattern == 0: 105 | L1_loss = self.evaluateL1(output, Y).item() 106 | L2_loss = self.evaluateL2(output, Y).item() 107 | if predict is None: 108 | predict = output 109 | test = Y 110 | else: 111 | predict = torch.cat((predict, output)) 112 | test = torch.cat((test, Y)) 113 | elif self.data_gen.normalize_pattern == 1: 114 | maximums = self.data_gen.maximums.expand(output.size(0), self.data_gen.column_num) 115 | if self.data_gen.mode == "immediate": 116 | L1_loss = self.evaluateL1(output * maximums, Y * maximums).item() 117 | L2_loss = self.evaluateL2(output * maximums, Y * maximums).item() 118 | if predict is None: 119 | predict = output * maximums 120 | test = Y * maximums 121 | else: 122 | predict = torch.cat((predict, output * maximums)) 123 | test = torch.cat((test, Y * maximums)) 124 | else: 125 | L1_loss = self.evaluateL1(output[:, self.data_gen.collaborate_span] * maximums, Y[:, self.data_gen.collaborate_span] * maximums).item() 126 | L2_loss = self.evaluateL2(output[:, self.data_gen.collaborate_span] * maximums, Y[:, self.data_gen.collaborate_span] * maximums).item() 127 | if predict is None: 128 | predict = output[:, self.data_gen.collaborate_span] * maximums 129 | test = Y[:, self.data_gen.collaborate_span] * maximums 130 | else: 131 | predict = torch.cat((predict, output[:, self.data_gen.collaborate_span] * maximums)) 132 | test = torch.cat((test, Y[:, self.data_gen.collaborate_span] * maximums)) 133 | 134 | else: 135 | means = self.data_gen.means.expand(output.size(0), self.data_gen.column_num) 136 | tmp_stds = self.data_gen.stds + ((self.data_gen.stds == 0).float() * 0.001) 137 | stds = tmp_stds.expand(output.size(0), self.data_gen.column_num) 138 | if self.data_gen.mode == "immediate": 139 | L1_loss = self.evaluateL1(output * stds + means, Y * stds + means).item() 140 | L2_loss = self.evaluateL2(output * stds + means, Y * stds + means).item() 141 | if predict is None: 142 | predict = output * stds + means 143 | test = Y * stds + means 144 | else: 145 | predict = torch.cat((predict, output * stds + means)) 146 | test = torch.cat((test, Y * stds + means)) 147 | else: 148 | L1_loss = self.evaluateL1(output[:, self.data_gen.collaborate_span] * stds + means, Y[:, self.data_gen.collaborate_span] * stds + means).item() 149 | L2_loss = self.evaluateL2(output[:, self.data_gen.collaborate_span] * stds + means, Y[:, self.data_gen.collaborate_span] * stds + means).item() 150 | if predict is None: 151 | predict = output[:, self.data_gen.collaborate_span] * stds + means 152 | test = Y[:, self.data_gen.collaborate_span] * stds + means 153 | else: 154 | predict = torch.cat((predict, output[:, self.data_gen.collaborate_span] * stds + means)) 155 | test = torch.cat((test, Y[:, self.data_gen.collaborate_span] * stds + means)) 156 | 157 | total_loss_l1 += L1_loss 158 | total_loss += L2_loss 159 | n_samples += (output.size(0) * self.data_gen.column_num) 160 | 161 | mse = total_loss / n_samples 162 | rse = math.sqrt(total_loss / n_samples) / self.data_gen.rse 163 | mae = total_loss_l1 / n_samples 164 | 165 | #if mode == 'test': 166 | # self.predict = predict.data.cpu().numpy() 167 | # self.ground_truth = test.data.cpu().numpy() 168 | 169 | return mse, rse, mae 170 | 171 | # --------------------------------------------------------------------------------------------------------------------------------------------- 172 | """ 173 | Run the model 174 | """ 175 | def run(self): 176 | #print(self.model) 177 | use_cuda = self.args.gpu is not None 178 | if use_cuda: 179 | if type(self.args.gpu) == list: 180 | self.model = nn.DataParallel(self.model, device_ids=self.args.gpu) 181 | else: 182 | torch.cuda.set_device(self.args.gpu) 183 | torch.manual_seed(self.args.seed) 184 | if torch.cuda.is_available(): torch.cuda.manual_seed(self.args.seed) 185 | if use_cuda: self.model.cuda() 186 | 187 | self.nParams = sum([p.nelement() for p in self.model.parameters()]) 188 | 189 | if self.args.L1Loss: 190 | self.criterion = nn.L1Loss(reduction='sum') 191 | else: 192 | self.criterion = nn.MSELoss(reduction='sum') 193 | self.evaluateL1 = nn.L1Loss(reduction='sum') 194 | self.evaluateL2 = nn.MSELoss(reduction='sum') 195 | if use_cuda: 196 | self.criterion = self.criterion.cuda() 197 | self.evaluateL1 = self.evaluateL1.cuda() 198 | self.evaluateL2 = self.evaluateL2.cuda() 199 | 200 | self.optim = Optimize(self.model.parameters(), self.args.optim, self.args.lr, self.args.clip) 201 | 202 | best_valid_mse = float("inf") 203 | best_valid_rse = float("inf") 204 | best_valid_mae = float("inf") 205 | best_test_mse = float("inf") 206 | best_test_rse = float("inf") 207 | best_test_mae = float("inf") 208 | 209 | tmp_losses = [] 210 | # At any point you can hit Ctrl + C to break out of training early. 211 | try: 212 | for epoch in range(1, self.args.epochs+1): 213 | epoch_start_time = time.time() 214 | train_loss = self.train() 215 | self.running_times.append(time.time() - epoch_start_time) 216 | tmp_losses.append(train_loss) 217 | val_mse, val_rse, val_mae = self.evaluate() 218 | if val_mse < best_valid_mse: 219 | best_valid_mse = val_mse 220 | best_valid_rse = val_rse 221 | best_valid_mae = val_mae 222 | 223 | self.optim.updateLearningRate(val_mse, epoch) 224 | 225 | test_mse, test_rse, test_mae = self.evaluate(mode='test') 226 | if test_mse < best_test_mse: 227 | best_test_mse = test_mse 228 | best_test_rse = test_rse 229 | best_test_mae = test_mae 230 | except KeyboardInterrupt: 231 | pass 232 | 233 | self.train_losses.append(tmp_losses) 234 | # In our experiment, we first use the validation set to select hyper-parameters from the whole hyper-parameter set. 235 | # We filtered out hyper-parameters that had poor performance 236 | # but remained several groups of hyper-parameters that obtained similar good predictive results. 237 | # After determining the remaining groups of hyper-parameters, 238 | # the validation set becomes another test set for testing the best performance of different hyper-parameters. 239 | # Therefore, we average the best valid and test result here to get final best result. 240 | final_best_mse = (best_valid_mse + best_test_mse) / 2.0 241 | final_best_rse = (best_valid_rse + best_test_rse) / 2.0 242 | final_best_mae = (best_valid_mae + best_test_mae) / 2.0 243 | # otherwise, uncomment the following codes to see the best test result only 244 | #final_best_mse = best_test_mse 245 | #final_best_rse = best_test_rse 246 | #final_best_mae = best_test_mae 247 | 248 | self.best_rmse = np.sqrt(final_best_mse) 249 | self.best_rse = final_best_rse 250 | self.best_mae = final_best_mae 251 | 252 | # --------------------------------------------------------------------------------------------------------------------------------------------- 253 | """ 254 | Compute and output the metrics 255 | """ 256 | def getMetrics(self): 257 | print('-' * 100) 258 | print() 259 | 260 | print('* number of parameters: %d' % self.nParams) 261 | for k in self.args.__dict__.keys(): 262 | print(k, ': ', self.args.__dict__[k]) 263 | 264 | running_times = np.array(self.running_times) 265 | #train_losses = np.array(self.train_losses) 266 | 267 | print("time: sum {:8.7f} | mean {:8.7f}".format(np.sum(running_times), np.mean(running_times))) 268 | #print("loss trend: ", self.train_losses) 269 | print("rmse: {:8.7f}".format(self.best_rmse)) 270 | print("rse: {:8.7f}".format(self.best_rse)) 271 | print("mae: {:8.7f}".format(self.best_mae)) 272 | print() -------------------------------------------------------------------------------- /models/models.py: -------------------------------------------------------------------------------- 1 | """ 2 | Define all models. 3 | """ 4 | 5 | from utils._libs_ import torch, nn, F, np 6 | 7 | # ------------------------------------------------------------------------------------------------------------------------------------------------- 8 | """ 9 | Multi-Level Construal Neural Network 10 | """ 11 | class MLCNN(nn.Module): 12 | def __init__(self, args, data): 13 | """ 14 | Initialization arguments: 15 | args - (object) parameters of model 16 | data - (DataGenerator object) the data generator 17 | """ 18 | super(MLCNN, self).__init__() 19 | self.use_cuda = args.cuda 20 | self.input_T = args.input_T 21 | self.idim = data.column_num 22 | self.kernel_size = args.kernel_size 23 | self.hidC = args.hidCNN 24 | self.hidR = args.hidRNN 25 | self.hw = args.highway_window 26 | self.collaborate_span = args.collaborate_span 27 | self.cnn_split_num = int(args.n_CNN / (self.collaborate_span * 2 + 1)) 28 | self.n_CNN = self.cnn_split_num * (self.collaborate_span * 2 + 1) 29 | 30 | self.dropout = nn.Dropout(p = args.dropout) 31 | self.convs = nn.ModuleList([]) 32 | self.bns = nn.ModuleList([]) 33 | for i in range(self.n_CNN): 34 | if i == 0: 35 | tmpconv = nn.Conv2d(1, self.hidC, kernel_size=(self.kernel_size, self.idim), padding=(self.kernel_size//2, 0)) 36 | else: 37 | tmpconv = nn.Conv2d(1, self.hidC, kernel_size=(self.kernel_size, self.hidC), padding=(self.kernel_size//2, 0)) 38 | self.convs.append(tmpconv) 39 | self.bns.append(nn.BatchNorm2d(self.hidC)) 40 | self.shared_lstm = nn.LSTM(self.hidC, self.hidR) 41 | self.target_lstm = nn.LSTM(self.hidC, self.hidR) 42 | self.linears = nn.ModuleList([]) 43 | self.highways = nn.ModuleList([]) 44 | for i in range(self.collaborate_span * 2 + 1): 45 | self.linears.append(nn.Linear(self.hidR, self.idim)) 46 | if (self.hw > 0): 47 | self.highways.append(nn.Linear(self.hw * (i+1), 1)) 48 | 49 | # --------------------------------------------------------------------------------------------------------------------------------------------- 50 | """ 51 | Forward propagation 52 | """ 53 | def forward(self, x): 54 | """ 55 | Arguments: 56 | x - (torch.tensor) input data 57 | Returns: 58 | res - (torch.tensor) result of prediction 59 | """ 60 | regressors = [] 61 | currentR = torch.unsqueeze(x, 1) 62 | for i in range(self.n_CNN): 63 | currentR = self.convs[i](currentR) 64 | currentR = self.bns[i](currentR) 65 | currentR = F.leaky_relu(currentR, negative_slope=0.01) 66 | currentR = torch.squeeze(currentR, 3) 67 | if (i + 1) % self.cnn_split_num == 0: 68 | regressors.append(currentR) 69 | currentR = self.dropout(currentR) 70 | if i < self.n_CNN - 1: 71 | currentR = currentR.permute(0,2,1).contiguous() 72 | currentR = torch.unsqueeze(currentR, 1) 73 | 74 | shared_lstm_results = [] 75 | target_R = None 76 | target_h = None 77 | target_c = None 78 | self.shared_lstm.flatten_parameters() 79 | for i in range(self.collaborate_span * 2 + 1): 80 | cur_R = regressors[i].permute(2,0,1).contiguous() 81 | _, (cur_result, cur_state) = self.shared_lstm(cur_R) 82 | if i == self.collaborate_span: 83 | target_R = cur_R 84 | target_h = cur_result 85 | target_c = cur_state 86 | cur_result = self.dropout(torch.squeeze(cur_result, 0)) 87 | shared_lstm_results.append(cur_result) 88 | 89 | self.target_lstm.flatten_parameters() 90 | _, (target_result, _) = self.target_lstm(target_R, (target_h, target_c)) 91 | target_result = self.dropout(torch.squeeze(target_result, 0)) 92 | 93 | res = None 94 | for i in range(self.collaborate_span * 2 + 1): 95 | if i == self.collaborate_span: 96 | cur_res = self.linears[i](target_result) 97 | else: 98 | cur_res = self.linears[i](shared_lstm_results[i]) 99 | cur_res = torch.unsqueeze(cur_res, 1) 100 | if res is not None: 101 | res = torch.cat((res, cur_res), 1) 102 | else: 103 | res = cur_res 104 | 105 | #highway 106 | if (self.hw > 0): 107 | highway = None 108 | for i in range(self.collaborate_span * 2 + 1): 109 | z = x[:, -(self.hw * (i+1)):, :] 110 | z = z.permute(0,2,1).contiguous().view(-1, self.hw * (i+1)) 111 | z = self.highways[i](z) 112 | z = z.view(-1, self.idim) 113 | z = torch.unsqueeze(z, 1) 114 | if highway is not None: 115 | highway = torch.cat((highway, z), 1) 116 | else: 117 | highway = z 118 | res = res + highway 119 | 120 | return res -------------------------------------------------------------------------------- /models/optimize.py: -------------------------------------------------------------------------------- 1 | """ 2 | Implements optimization algorithms. 3 | """ 4 | 5 | from utils._libs_ import math, optim 6 | 7 | # ------------------------------------------------------------------------------------------------------------------------------------------------- 8 | class Optimize(): 9 | def __init__(self, params, method, lr, max_grad_norm, lr_decay=0.1, start_decay_at=None, patience=5, decay_nb=3): 10 | """ 11 | Initialization arguments: 12 | params - (torch.nn.Module.parameters()) parameters of model (may be a generator) 13 | method - (string) optimization method 14 | lr - (float) learning rate 15 | max_grad_norm - (float) norm for gradient clipping 16 | lr_decay - (float) decay scale of learning rate when validation performance does not improve or we hit epoch 17 | start_decay_at - (int) decay the learning rate at i-th epoch referenced by 18 | patience - (int) number of epoch after which learning rate will decay if no improvement 19 | decay_nb - (int) number of learning rate decay 20 | """ 21 | self.params = list(params) 22 | self.last_ppl = None 23 | self.lr = lr 24 | self.max_grad_norm = max_grad_norm 25 | self.method = method 26 | self.lr_decay = lr_decay 27 | self.start_decay_at = start_decay_at 28 | self.start_decay = False 29 | self.patience = patience 30 | self.decay_nb = decay_nb 31 | self.wait = 0 32 | self.already_decay_nb = 0 33 | 34 | self._makeOptimizer() 35 | 36 | # --------------------------------------------------------------------------------------------------------------------------------------------- 37 | """ 38 | Select the optimizer 39 | """ 40 | def _makeOptimizer(self): 41 | if self.method == 'sgd': 42 | self.optimizer = optim.SGD(self.params, lr=self.lr) 43 | elif self.method == 'adagrad': 44 | self.optimizer = optim.Adagrad(self.params, lr=self.lr) 45 | elif self.method == 'adadelta': 46 | self.optimizer = optim.Adadelta(self.params, lr=self.lr) 47 | elif self.method == 'adam': 48 | self.optimizer = optim.Adam(self.params, lr=self.lr) 49 | else: 50 | raise RuntimeError("Invalid optim method: " + self.method) 51 | 52 | # --------------------------------------------------------------------------------------------------------------------------------------------- 53 | """ 54 | Run the gradient optimization 55 | """ 56 | def step(self): 57 | # Compute gradients norm. 58 | grad_norm = 0 59 | for param in self.params: 60 | if param.grad is not None: 61 | grad_norm += math.pow(param.grad.data.norm(), 2) 62 | else: 63 | grad_norm += 0. 64 | 65 | grad_norm = math.sqrt(grad_norm) 66 | if grad_norm > 0: 67 | shrinkage = self.max_grad_norm / grad_norm 68 | else: 69 | shrinkage = 1. 70 | 71 | for param in self.params: 72 | if shrinkage < 1 and param.grad is not None: 73 | param.grad.data.mul_(shrinkage) 74 | 75 | self.optimizer.step() 76 | return grad_norm 77 | 78 | # --------------------------------------------------------------------------------------------------------------------------------------------- 79 | """ 80 | decay learning rate if validation performance does not improve or we hit the start_decay_at epoch 81 | """ 82 | def updateLearningRate(self, ppl, epoch): 83 | """ 84 | Arguments: 85 | ppl - (float) the loss value 86 | epoch - (int) the number of trained epoch 87 | """ 88 | if self.already_decay_nb < self.decay_nb: 89 | if self.start_decay_at is not None and epoch >= self.start_decay_at: 90 | self.wait += 1 91 | if self.wait >= self.patience: 92 | self.start_decay = True 93 | self.wait = 0 94 | else: 95 | self.start_decay = False 96 | if self.last_ppl is not None and ppl > self.last_ppl: 97 | self.wait += 1 98 | if self.wait >= self.patience: 99 | self.start_decay = True 100 | self.wait = 0 101 | else: 102 | self.start_decay = False 103 | 104 | if self.start_decay: 105 | self.already_decay_nb += 1 106 | self.lr = self.lr * self.lr_decay 107 | #print("Decaying learning rate to %g" % self.lr) 108 | else: 109 | if self.last_ppl is not None and ppl > self.last_ppl: 110 | self.wait += 1 111 | if self.wait >= self.patience: 112 | raise KeyboardInterrupt 113 | #only decay for one epoch 114 | self.start_decay = False 115 | 116 | self.last_ppl = ppl 117 | 118 | self._makeOptimizer() -------------------------------------------------------------------------------- /runEnergy.py: -------------------------------------------------------------------------------- 1 | from utils.data_io import getGenerator 2 | from utils.args import Args, list_of_param_dicts 3 | from models.models import MLCNN 4 | from models.model_runner import ModelRunner 5 | import gc 6 | 7 | param_dict = dict( 8 | data = ['data/energydata_complete.txt'], 9 | mode = ['continuous'], 10 | collaborate_span = [2], 11 | collaborate_stride = [1], 12 | train_share = [(0.8, 0.1)], 13 | input_T = [144], 14 | n_CNN = [10], 15 | kernel_size = [5, 7], 16 | hidCNN = [5, 10, 100], 17 | hidRNN = [25, 100], 18 | dropout = [0.2], 19 | highway_window = [8], 20 | clip = [10.], 21 | epochs = [100], 22 | batch_size = [128], 23 | seed = [54321], 24 | gpu = [0], 25 | cuda = [True], 26 | optim = ['adam'], 27 | lr = [0.001], 28 | output_T = [3, 6, 12], 29 | L1Loss = [True], 30 | normalize = [2] 31 | ) 32 | 33 | if __name__ == '__main__': 34 | params = list_of_param_dicts(param_dict) 35 | for param in params: 36 | cur_args = Args(param) 37 | generator = getGenerator(cur_args.data) 38 | data_gen = generator(cur_args.data, cur_args.mode, 39 | train_share=cur_args.train_share, input_T=cur_args.input_T, output_T=cur_args.output_T, 40 | collaborate_span=cur_args.collaborate_span, collaborate_stride=cur_args.collaborate_stride, 41 | cuda=cur_args.cuda, normalize_pattern=cur_args.normalize) 42 | runner = ModelRunner(cur_args, data_gen, None) 43 | runner.model = MLCNN(cur_args, data_gen) 44 | runner.run() 45 | runner.getMetrics() 46 | del runner 47 | gc.collect() -------------------------------------------------------------------------------- /runNASDAQ.py: -------------------------------------------------------------------------------- 1 | from utils.data_io import getGenerator 2 | from utils.args import Args, list_of_param_dicts 3 | from models.models import MLCNN 4 | from models.model_runner import ModelRunner 5 | import gc 6 | 7 | param_dict = dict( 8 | data = ['data/nasdaq100_padding.csv'], 9 | mode = ['continuous'], 10 | collaborate_span = [2], 11 | collaborate_stride = [1], 12 | train_share = [(0.9, 0.05)], 13 | input_T = [60, 180], 14 | n_CNN = [5], 15 | kernel_size = [3, 5], 16 | hidCNN = [10, 100], 17 | hidRNN = [25, 100], 18 | dropout = [0.2, 0.3, 0.5], 19 | highway_window = [8], 20 | clip = [10.], 21 | epochs = [100], 22 | batch_size = [128], 23 | seed = [54321], 24 | gpu = [0], 25 | cuda = [True], 26 | optim = ['adam'], 27 | lr = [0.001], 28 | output_T = [3, 6, 12], 29 | L1Loss = [False], 30 | normalize = [2] 31 | ) 32 | 33 | if __name__ == '__main__': 34 | params = list_of_param_dicts(param_dict) 35 | for param in params: 36 | cur_args = Args(param) 37 | generator = getGenerator(cur_args.data) 38 | data_gen = generator(cur_args.data, cur_args.mode, 39 | train_share=cur_args.train_share, input_T=cur_args.input_T, output_T=cur_args.output_T, 40 | collaborate_span=cur_args.collaborate_span, collaborate_stride=cur_args.collaborate_stride, 41 | cuda=cur_args.cuda, normalize_pattern=cur_args.normalize) 42 | runner = ModelRunner(cur_args, data_gen, None) 43 | runner.model = MLCNN(cur_args, data_gen) 44 | runner.run() 45 | runner.getMetrics() 46 | del runner 47 | gc.collect() -------------------------------------------------------------------------------- /runTraffic.py: -------------------------------------------------------------------------------- 1 | from utils.data_io import getGenerator 2 | from utils.args import Args, list_of_param_dicts 3 | from models.models import MLCNN 4 | from models.model_runner import ModelRunner 5 | import gc 6 | 7 | param_dict = dict( 8 | data = ['data/traffic.txt'], 9 | mode = ['continuous'], 10 | collaborate_span = [2], 11 | collaborate_stride = [1], 12 | train_share = [(0.6, 0.2)], 13 | input_T = [24 * 7], 14 | n_CNN = [5, 10], 15 | kernel_size = [3], 16 | hidCNN = [50], 17 | hidRNN = [100], 18 | dropout = [0.2, 0.3], 19 | highway_window = [8], 20 | clip = [10.], 21 | epochs = [100], 22 | batch_size = [128], 23 | seed = [54321], 24 | gpu = [0], 25 | cuda = [True], 26 | optim = ['adam'], 27 | lr = [0.001], 28 | output_T = [3, 6, 12], 29 | L1Loss = [True], 30 | normalize = [1, 2] 31 | ) 32 | 33 | if __name__ == '__main__': 34 | params = list_of_param_dicts(param_dict) 35 | for param in params: 36 | cur_args = Args(param) 37 | generator = getGenerator(cur_args.data) 38 | data_gen = generator(cur_args.data, cur_args.mode, 39 | train_share=cur_args.train_share, input_T=cur_args.input_T, output_T=cur_args.output_T, 40 | collaborate_span=cur_args.collaborate_span, collaborate_stride=cur_args.collaborate_stride, 41 | cuda=cur_args.cuda, normalize_pattern=cur_args.normalize) 42 | runner = ModelRunner(cur_args, data_gen, None) 43 | runner.model = MLCNN(cur_args, data_gen) 44 | runner.run() 45 | runner.getMetrics() 46 | del runner 47 | gc.collect() -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This package imports necessary libraries and defines useful objects 3 | 4 | Author: Jiezhu Cheng 5 | Date: 17/03/2019 6 | """ -------------------------------------------------------------------------------- /utils/_libs_.py: -------------------------------------------------------------------------------- 1 | """ 2 | Import necessary libraries. 3 | """ 4 | 5 | # pyTorch 6 | import torch 7 | import torch.optim as optim 8 | import torch.nn as nn 9 | from torch.nn import Module 10 | import torch.nn.functional as F 11 | from torch.nn.parameter import Parameter 12 | from torch.nn.utils.rnn import PackedSequence 13 | from torch.autograd import Variable 14 | 15 | # Computation 16 | import numpy as np 17 | import pandas as pd 18 | import math 19 | import time 20 | 21 | # Tools 22 | from itertools import product as prod -------------------------------------------------------------------------------- /utils/args.py: -------------------------------------------------------------------------------- 1 | """ 2 | Specify the arguments 3 | """ 4 | 5 | from utils._libs_ import prod 6 | 7 | # ------------------------------------------------------------------------------------------------------------------------------------------------- 8 | """ 9 | Function to convert dictionary of lists to list of dictionaries of all combinations of listed variables. 10 | Example: 11 | list_of_param_dicts({'a': [1, 2], 'b': [3, 4]}) ---> [{'a': 1, 'b': 3}, {'a': 1, 'b': 4}, {'a': 2, 'b': 3}, {'a': 2, 'b': 4}] 12 | """ 13 | def list_of_param_dicts(param_dict): 14 | """ 15 | Arguments: 16 | param_dict -(dict) dictionary of parameters 17 | """ 18 | vals = list(prod(*[v for k, v in param_dict.items()])) 19 | keys = list(prod(*[[k]*len(v) for k, v in param_dict.items()])) 20 | return [dict([(k, v) for k, v in zip(key, val)]) for key, val in zip(keys, vals)] 21 | 22 | # ------------------------------------------------------------------------------------------------------------------------------------------------- 23 | """ 24 | Arguments class that fits models 25 | """ 26 | class Args(): 27 | """ 28 | Arguments: 29 | arg_dict -(dict) dictionary of model parameters 30 | """ 31 | def __init__(self, arg_dict): 32 | self.__dict__ = arg_dict -------------------------------------------------------------------------------- /utils/data_io.py: -------------------------------------------------------------------------------- 1 | """ 2 | Implements all io operations of data. 3 | """ 4 | 5 | from utils._libs_ import np, pd, torch, Variable 6 | 7 | # ------------------------------------------------------------------------------------------------------------------------------------------------- 8 | """ 9 | Compute the normal std 10 | """ 11 | def normal_std(x): 12 | """ 13 | Arguments: 14 | x - (torch.tensor) dataset for computation 15 | Returns: 16 | The normal std tensor 17 | """ 18 | return x.std() * np.sqrt((len(x) - 1.) / (len(x))) 19 | 20 | # ------------------------------------------------------------------------------------------------------------------------------------------------- 21 | """ 22 | Get the data generator 23 | """ 24 | def getGenerator(data_name): 25 | """ 26 | Arguments: 27 | data_name - (string) name of data file 28 | Returns: 29 | DataGenerator class that fits data_name 30 | """ 31 | if 'nasdaq' in data_name: 32 | return NasdaqGenerator 33 | else: 34 | return GeneralGenerator 35 | 36 | # ------------------------------------------------------------------------------------------------------------------------------------------------- 37 | """ 38 | DataGenerator class produces data samples for all models 39 | """ 40 | class DataGenerator(): 41 | # --------------------------------------------------------------------------------------------------------------------------------------------- 42 | """ 43 | Initialization arguments: 44 | X - (numpy.array) the whole dataset used for training, validation and testing 45 | mode - (string) define the prediction mode of models: 46 | - "immediate": predict the value at time X_{t+output_T} 47 | - "continuous": predict the value at time X_{t+output_T-collaborate_span}, ..., X_{t+output_T-1}, X_{t+output_T}, X_{t+output_T+1}, ..., X_{t+output_T+collaborate_span} 48 | train_share - (tuple) two numbers in range (0, 1) showing proportion of training and validation samples 49 | input_T - (int) number of timesteps in the input 50 | output_T - (int) number of timesteps in the output 51 | collaborate_span - (int) time span for collaborate prediction, only works when mode == "continuous" 52 | collaborate_stride - (int) stride for collaborate prediction, only works when mode == "continuous" 53 | limit - (int) maximum number of timesteps-rows in the 'X' DataFrame 54 | cuda - (boolean) whether use gpu to train models 55 | normalize_pattern - (int) define the normalization method: 56 | - 0: use the original data without normalization 57 | - 1: normlized by the maximum absolute value of each column. 58 | - 2: normlized by the mean and std value of each column 59 | """ 60 | def __init__(self, X, mode, train_share=(.8, .1), input_T=10, output_T=1, collaborate_span=0, 61 | collaborate_stride=1, limit=np.inf, cuda=False, normalize_pattern = 2): 62 | if mode == "continuous" and (output_T <= collaborate_span or collaborate_span <= 0): 63 | raise Exception("collaborate_span must > 0 and < output_T!") 64 | 65 | self.X = X 66 | if limit < np.inf: self.X = self.X[:limit] 67 | self.train_share = train_share 68 | self.input_T = input_T 69 | self.output_T = output_T 70 | self.collaborate_span = collaborate_span 71 | self.collaborate_stride = collaborate_stride 72 | self.row_num = self.X.shape[0] 73 | self.column_num = self.X.shape[1] 74 | self.n_train = int(self.row_num * train_share[0]) 75 | self.n_valid = int(self.row_num * (train_share[0] + train_share[1])) 76 | self.n_test = self.row_num 77 | self.mode = mode 78 | self.cuda = cuda 79 | self.normalize_pattern = normalize_pattern 80 | 81 | self.normalize() 82 | self.split_data() 83 | self.compute_metrics() 84 | 85 | # --------------------------------------------------------------------------------------------------------------------------------------------- 86 | """ 87 | Normalize data 88 | """ 89 | def normalize(self): 90 | if self.normalize_pattern == 0: 91 | pass 92 | elif self.normalize_pattern == 1: 93 | self.maximums = np.max(np.abs(self.X), axis=0) 94 | self.X = self.X / self.maximums 95 | elif self.normalize_pattern == 2: 96 | self.means = np.mean(self.X[:self.n_train], axis=0) 97 | self.stds = np.std(self.X[:self.n_train], axis=0) 98 | self.X = (self.X - self.means) / (self.stds + (self.stds == 0) * .001) 99 | else: 100 | raise Exception('invalid normalize_pattern') 101 | 102 | # --------------------------------------------------------------------------------------------------------------------------------------------- 103 | """ 104 | Split the training, validation and testing data 105 | """ 106 | def split_data(self): 107 | train_range = range(self.input_T + self.output_T - 1, self.n_train) 108 | valid_range = range(self.n_train, self.n_valid) 109 | test_range = range(self.n_valid, self.row_num) 110 | self.train_set = self.batchify(train_range) 111 | self.valid_set = self.batchify(valid_range) 112 | self.test_set = self.batchify(test_range) 113 | 114 | # --------------------------------------------------------------------------------------------------------------------------------------------- 115 | """ 116 | Get the X and Y for each set 117 | """ 118 | def batchify(self, idx_set): 119 | """ 120 | Arguments: 121 | idx_set - (list) index set of data samples 122 | Returns: 123 | [X, Y] 124 | """ 125 | idx_num = len(idx_set) 126 | if self.mode == "immediate": 127 | X = torch.zeros((idx_num, self.input_T, self.column_num)) 128 | Y = torch.zeros((idx_num, self.column_num)) 129 | for i in range(idx_num): 130 | end = idx_set[i] - self.output_T + 1 131 | start = end - self.input_T 132 | X[i, :, :] = torch.from_numpy(self.X[start:end, :]) 133 | Y[i, :] = torch.from_numpy(self.X[idx_set[i], :]) 134 | elif self.mode == "continuous": 135 | X = torch.zeros((idx_num - self.collaborate_span * self.collaborate_stride, self.input_T, self.column_num)) 136 | Y = torch.zeros((idx_num - self.collaborate_span * self.collaborate_stride, self.collaborate_span * 2 + 1, self.column_num)) 137 | for i in range(idx_num - self.collaborate_span * self.collaborate_stride): 138 | X_end = idx_set[i] - self.output_T + 1 139 | X_start = X_end - self.input_T 140 | Y_sample = [] 141 | tmp_span = self.collaborate_span * self.collaborate_stride 142 | for _ in range(self.collaborate_span * 2 + 1): 143 | Y_sample.append(idx_set[i] - tmp_span) 144 | tmp_span -= self.collaborate_stride 145 | X[i, :, :] = torch.from_numpy(self.X[X_start:X_end, :]) 146 | Y[i, :, :] = torch.from_numpy(self.X[Y_sample, :]) 147 | else: 148 | raise Exception('invalid mode') 149 | 150 | return [X, Y] 151 | 152 | # --------------------------------------------------------------------------------------------------------------------------------------------- 153 | """ 154 | Compute the metrics for evaluating the models 155 | """ 156 | def compute_metrics(self): 157 | if self.normalize_pattern == 0: 158 | if self.mode == "immediate": 159 | tmp = self.test_set[1] 160 | else: 161 | tmp = self.test_set[1][:, self.collaborate_span] 162 | elif self.normalize_pattern == 1: 163 | self.maximums = torch.from_numpy(self.maximums).float() 164 | if self.mode == "immediate": 165 | tmp = self.test_set[1] * self.maximums.expand(self.test_set[1].size(0), self.column_num) 166 | else: 167 | tmp = self.test_set[1][:, self.collaborate_span] * self.maximums.expand(self.test_set[1].size(0), self.column_num) 168 | 169 | if self.cuda: 170 | self.maximums = self.maximums.cuda() 171 | self.maximums = Variable(self.maximums) 172 | else: 173 | self.means = torch.from_numpy(self.means).float() 174 | self.stds = torch.from_numpy(self.stds).float() 175 | tmp_std = self.stds + ((self.stds == 0).float() * 0.001) 176 | if self.mode == "immediate": 177 | tmp = self.test_set[1] * tmp_std.expand(self.test_set[1].size(0), self.column_num) + self.means.expand(self.test_set[1].size(0), self.column_num) 178 | else: 179 | tmp = self.test_set[1][:, self.collaborate_span] * tmp_std.expand(self.test_set[1].size(0), self.column_num) + self.means.expand(self.test_set[1].size(0), self.column_num) 180 | 181 | if self.cuda: 182 | self.means = self.means.cuda() 183 | self.stds = self.stds.cuda() 184 | self.means = Variable(self.means) 185 | self.stds = Variable(self.stds) 186 | 187 | self.rse = normal_std(tmp) 188 | 189 | # --------------------------------------------------------------------------------------------------------------------------------------------- 190 | """ 191 | Get the batch data 192 | """ 193 | def get_batches(self, X, Y, batch_size, shuffle=True): 194 | """ 195 | Arguments: 196 | X - (torch.tensor) input dataset 197 | Y - (torch.tensor) ground-truth dataset 198 | batch_size - (int) batch size 199 | shuffle - (boolean) whether shuffle the dataset 200 | Yields: 201 | (batch_X, batch_Y) 202 | """ 203 | length = len(X) 204 | if shuffle: 205 | index = torch.randperm(length) 206 | else: 207 | index = torch.LongTensor(range(length)) 208 | start_idx = 0 209 | while (start_idx < length): 210 | end_idx = min(length, start_idx + batch_size) 211 | excerpt = index[start_idx:end_idx] 212 | batch_X = X[excerpt] 213 | batch_Y = Y[excerpt] 214 | if (self.cuda): 215 | batch_X = batch_X.cuda() 216 | batch_Y = batch_Y.cuda() 217 | yield Variable(batch_X), Variable(batch_Y) 218 | start_idx += batch_size 219 | 220 | # ------------------------------------------------------------------------------------------------------------------------------------------------- 221 | """ 222 | A GeneralGenerator reads complete data from .txt file without outliers 223 | """ 224 | class GeneralGenerator(DataGenerator): 225 | def __init__(self, data_path, mode, train_share=(.8, .1), input_T=10, output_T=1, collaborate_span=0, 226 | collaborate_stride=1, limit=np.inf, cuda=False, normalize_pattern = 2): 227 | X = np.loadtxt(data_path, delimiter=',') 228 | super(GeneralGenerator, self).__init__(X, mode=mode, 229 | train_share=train_share, 230 | input_T=input_T, 231 | output_T=output_T, 232 | collaborate_span=collaborate_span, 233 | collaborate_stride=collaborate_stride, 234 | limit=limit, 235 | cuda=cuda, 236 | normalize_pattern=normalize_pattern) 237 | 238 | # -------------------------------------------------------------------------------------------------------------------------------- 239 | """ 240 | A NasdaqGenerator reads samples of NASDAQ 100 stock data. 241 | """ 242 | class NasdaqGenerator(DataGenerator): 243 | def __init__(self, data_path, mode, train_share=(.8, .1), input_T=10, output_T=1, collaborate_span=0, 244 | collaborate_stride=1, limit=np.inf, cuda=False, normalize_pattern = 2): 245 | X = pd.read_csv(data_path) 246 | super(NasdaqGenerator, self).__init__(X.values, mode=mode, 247 | train_share=train_share, 248 | input_T=input_T, 249 | output_T=output_T, 250 | collaborate_span=collaborate_span, 251 | collaborate_stride=collaborate_stride, 252 | limit=limit, 253 | cuda=cuda, 254 | normalize_pattern=normalize_pattern) --------------------------------------------------------------------------------